gleep fixes mtls and stuff

app/apps/keys/models.py

@@ -107,7 +107,15 @@ class SSHCertificateAuthority(models.Model):
         self.revoked_at = timezone.now()
 
     def get_private_key(self) -> str:
-        file_payload = read_secret_ref(self.private_key, label="User CA private key")
+        try:
+            file_payload = read_secret_ref(self.private_key, label="User CA private key")
+        except RuntimeError as exc:
+            # Recover gracefully when a secret-file reference points to a
+            # deleted key file (for example after volume resets).
+            if "file missing:" in str(exc):
+                file_payload = ""
+            else:
+                raise
         if file_payload:
             return file_payload
         inline_payload = (self.private_key or "").strip()

app/apps/servers/admin.py

@@ -1,4 +1,5 @@
 from django.contrib import admin
+from django.conf import settings
 from django.utils.html import format_html
 
 try:
@@ -14,7 +15,16 @@ except ImportError:  # Fallback for older Unfold builds without guardian admin s
     class GuardedModelAdmin(GuardianGuardedModelAdmin, UnfoldModelAdmin):
         pass
 
-from .models import AgentCertificateAuthority, EnrollmentToken, Server, ServerAuditLog, ServerLogSource
+from apps.access.models import AccessRequest
+
+from .models import (
+    AgentCertificateAuthority,
+    EnrollmentToken,
+    Server,
+    ServerAccount,
+    ServerAuditLog,
+    ServerLogSource,
+)
 
 
 class ServerLogSourceInline(admin.TabularInline):
@@ -54,6 +64,64 @@ class ServerAdmin(GuardedModelAdmin):
     )
     inlines = (ServerLogSourceInline,)
 
+    def _large_delete_preview_threshold(self) -> int:
+        value = getattr(settings, "KEYWARDEN_ADMIN_DELETE_PREVIEW_MAX_ITEMS", 5000)
+        try:
+            threshold = int(value)
+        except (TypeError, ValueError):
+            threshold = 5000
+        return max(threshold, 1)
+
+    def _related_delete_counts(self, server_ids: list[int]) -> dict[str, int]:
+        return {
+            ServerAuditLog._meta.verbose_name_plural: ServerAuditLog.objects.filter(server_id__in=server_ids).count(),
+            ServerAccount._meta.verbose_name_plural: ServerAccount.objects.filter(server_id__in=server_ids).count(),
+            ServerLogSource._meta.verbose_name_plural: ServerLogSource.objects.filter(server_id__in=server_ids).count(),
+            AccessRequest._meta.verbose_name_plural: AccessRequest.objects.filter(server_id__in=server_ids).count(),
+        }
+
+    def get_deleted_objects(self, objs, request):
+        # Django's default nested object preview scales poorly for very large
+        # cascades (for example, many ServerAuditLog rows) and can cause
+        # worker timeouts/OOM during delete confirmation rendering.
+        if not request.user.is_superuser:
+            return super().get_deleted_objects(objs, request)
+
+        server_ids = list(objs.values_list("id", flat=True))
+        if not server_ids:
+            return super().get_deleted_objects(objs, request)
+
+        related_counts = self._related_delete_counts(server_ids)
+        total_related = sum(related_counts.values())
+        if total_related < self._large_delete_preview_threshold():
+            return super().get_deleted_objects(objs, request)
+
+        model_count = {
+            self.model._meta.verbose_name_plural: len(server_ids),
+        }
+        deleted_objects = [f"{len(server_ids)} {self.model._meta.verbose_name_plural}"]
+        for label, count in related_counts.items():
+            if count:
+                model_count[label] = count
+                deleted_objects.append(f"{count} {label}")
+        deleted_objects.append(
+            "Large cascade detected. Related objects are summarized here to keep delete confirmation responsive."
+        )
+        return deleted_objects, model_count, set(), []
+
+    def _purge_server_audit_logs(self, server_ids: list[int]) -> None:
+        if not server_ids:
+            return
+        ServerAuditLog.objects.filter(server_id__in=server_ids).delete()
+
+    def delete_model(self, request, obj) -> None:
+        self._purge_server_audit_logs([obj.id])
+        super().delete_model(request, obj)
+
+    def delete_queryset(self, request, queryset) -> None:
+        self._purge_server_audit_logs(list(queryset.values_list("id", flat=True)))
+        super().delete_queryset(request, queryset)
+
     def avatar(self, obj: Server):
         if obj.image_url:
             return format_html(

app/apps/servers/models.py

@@ -139,7 +139,15 @@ class AgentCertificateAuthority(models.Model):
         self.revoked_at = timezone.now()
 
     def get_key_pem(self) -> str:
-        file_payload = read_secret_ref(self.key_pem, label="Agent CA private key")
+        try:
+            file_payload = read_secret_ref(self.key_pem, label="Agent CA private key")
+        except RuntimeError as exc:
+            # Recover gracefully when a secret-file reference points to a
+            # deleted key file (for example after volume resets).
+            if "file missing:" in str(exc):
+                file_payload = ""
+            else:
+                raise
         if file_payload:
             return file_payload
         inline_payload = (self.key_pem or "").strip()

app/apps/servers/tests/test_admin_delete.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from django.contrib.admin.sites import AdminSite
+from django.contrib.auth import get_user_model
+from django.test import RequestFactory, TestCase, override_settings
+from django.utils import timezone
+
+from apps.access.models import AccessRequest
+from apps.servers.admin import ServerAdmin
+from apps.servers.models import Server, ServerAccount, ServerAuditLog, ServerLogSource
+
+
+class ServerAdminDeleteTests(TestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        User = get_user_model()
+        self.superuser = User.objects.create_superuser(
+            username="admin-delete",
+            email="admin-delete@example.com",
+            password="pass12345",
+        )
+        self.user = User.objects.create_user(
+            username="member-delete",
+            email="member-delete@example.com",
+            password="pass12345",
+        )
+        self.site = AdminSite()
+        self.model_admin = ServerAdmin(Server, self.site)
+        self.factory = RequestFactory()
+
+    def _request(self):
+        request = self.factory.post("/admin/servers/server/")
+        request.user = self.superuser
+        return request
+
+    @override_settings(KEYWARDEN_ADMIN_DELETE_PREVIEW_MAX_ITEMS=2)
+    def test_get_deleted_objects_summarizes_large_cascade(self) -> None:
+        server = Server.objects.create(display_name="srv-admin-delete")
+        ServerAuditLog.objects.bulk_create(
+            [
+                ServerAuditLog(
+                    server=server,
+                    event_at=timezone.now(),
+                    category="auth",
+                    event_type="login",
+                ),
+                ServerAuditLog(
+                    server=server,
+                    event_at=timezone.now(),
+                    category="auth",
+                    event_type="logout",
+                ),
+                ServerAuditLog(
+                    server=server,
+                    event_at=timezone.now(),
+                    category="auth",
+                    event_type="sudo",
+                ),
+            ]
+        )
+        ServerAccount.objects.create(server=server, user=self.user, system_username="member_delete")
+        ServerLogSource.objects.create(server=server, kind=ServerLogSource.Kind.JOURNAL, name="system-journal")
+        AccessRequest.objects.create(requester=self.user, server=server, request_shell=True)
+
+        deleted_objects, model_count, perms_needed, protected = self.model_admin.get_deleted_objects(
+            Server.objects.filter(id=server.id),
+            self._request(),
+        )
+
+        self.assertEqual(perms_needed, set())
+        self.assertEqual(protected, [])
+        self.assertIn("Large cascade detected", " ".join(str(item) for item in deleted_objects))
+        self.assertEqual(model_count[Server._meta.verbose_name_plural], 1)
+        self.assertEqual(model_count[ServerAuditLog._meta.verbose_name_plural], 3)
+        self.assertEqual(model_count[ServerAccount._meta.verbose_name_plural], 1)
+        self.assertEqual(model_count[ServerLogSource._meta.verbose_name_plural], 1)
+        self.assertEqual(model_count[AccessRequest._meta.verbose_name_plural], 1)
+
+    def test_delete_queryset_prepurges_audit_logs(self) -> None:
+        server = Server.objects.create(display_name="srv-admin-delete-purge")
+        ServerAuditLog.objects.bulk_create(
+            [
+                ServerAuditLog(
+                    server=server,
+                    event_at=timezone.now(),
+                    category="auth",
+                    event_type="login",
+                ),
+                ServerAuditLog(
+                    server=server,
+                    event_at=timezone.now(),
+                    category="auth",
+                    event_type="logout",
+                ),
+            ]
+        )
+
+        self.model_admin.delete_queryset(self._request(), Server.objects.filter(id=server.id))
+
+        self.assertFalse(Server.objects.filter(id=server.id).exists())
+        self.assertFalse(ServerAuditLog.objects.filter(server_id=server.id).exists())

app/apps/servers/tests/test_audit_pipeline.py

@@ -21,7 +21,7 @@ from ninja.errors import HttpError
 from apps.access.models import AccessRequest
 from apps.keys.models import SSHKey
 from apps.keys.models import SSHCertificate, SSHCertificateAuthority, parse_public_key
-from apps.servers.models import AgentCertificateAuthority, Server, ServerAuditLog, ServerLogSource
+from apps.servers.models import AgentCertificateAuthority, EnrollmentToken, Server, ServerAuditLog, ServerLogSource
 from keywarden.api.routers import agent as agent_router
 
 
@@ -472,6 +472,10 @@ class AgentRouterHelpersTests(TestCase):
             SimpleNamespace(auth=agent_router.AgentPrincipal(server_id=self.server.id, mode="mtls")),
             self.server.id,
         )
+        agent_router._require_agent_access(
+            SimpleNamespace(auth=agent_router.AgentPrincipal(server_id=None, mode="global-token")),
+            self.server.id,
+        )
         self.assertNotEqual(agent_router._issue_agent_api_token(), agent_router._issue_agent_api_token())
 
     def test_server_lookup_key_map_and_account_sync_helpers(self) -> None:
@@ -543,6 +547,65 @@ class AgentRouterHelpersTests(TestCase):
         self.assertTrue(fingerprint)
         self.assertTrue(serial)
 
+    def test_enroll_agent_rotates_existing_server_when_server_id_is_provided(self) -> None:
+        ca_cert, _ca_key, ca_pem, key_pem = self._build_ca_material()
+        agent_router.AgentCertificateAuthority.objects.create(
+            name="Enroll CA",
+            cert_pem=ca_pem,
+            key_pem=key_pem,
+            is_active=True,
+        )
+        token = EnrollmentToken.objects.create(token="rotate-token")
+        original_server_count = Server.objects.count()
+        self.server.agent_cert_fingerprint = "deadbeef"
+        self.server.agent_cert_serial = "1"
+        self.server.save(update_fields=["agent_cert_fingerprint", "agent_cert_serial"])
+
+        csr = self._build_csr("rotate-agent.example.test")
+        payload = {
+            "token": token.token,
+            "csr_pem": csr.public_bytes(serialization.Encoding.PEM).decode("utf-8"),
+            "server_id": str(self.server.id),
+            "host": self.server.hostname,
+        }
+        response = self.client.post(
+            "/api/v1/agent/enroll",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 200)
+        body = response.json()
+        self.assertEqual(body["server_id"], str(self.server.id))
+        self.assertIn("BEGIN CERTIFICATE", body["client_cert_pem"])
+        self.assertIn("BEGIN CERTIFICATE", body["ca_cert_pem"])
+        self.assertTrue(body["agent_api_token"])
+        self.assertEqual(Server.objects.count(), original_server_count)
+
+        self.server.refresh_from_db()
+        self.assertNotEqual(self.server.agent_cert_fingerprint, "deadbeef")
+        self.assertNotEqual(self.server.agent_cert_serial, "1")
+        self.assertIsNotNone(self.server.agent_enrolled_at)
+        token.refresh_from_db()
+        self.assertEqual(token.server_id, self.server.id)
+        self.assertIsNotNone(token.used_at)
+
+    def test_enroll_agent_rejects_invalid_server_id(self) -> None:
+        token = EnrollmentToken.objects.create(token="rotate-token-bad")
+        csr = self._build_csr("rotate-agent.example.test")
+        payload = {
+            "token": token.token,
+            "csr_pem": csr.public_bytes(serialization.Encoding.PEM).decode("utf-8"),
+            "server_id": "not-a-number",
+        }
+        response = self.client.post(
+            "/api/v1/agent/enroll",
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 422)
+        self.assertIn("server_id must be numeric", response.content.decode("utf-8"))
+
 
 class KeysModelHelpersTests(TestCase):
     def setUp(self) -> None:

app/apps/servers/tests/test_ca_file_recovery.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import os
+import tempfile
+
+from django.test import TestCase, override_settings
+
+from apps.core.secret_files import parse_secret_ref
+from apps.keys.models import SSHCertificateAuthority
+from apps.servers.models import AgentCertificateAuthority
+
+
+class CASecretFileRecoveryTests(TestCase):
+    def test_user_ca_recovers_when_private_key_file_is_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            missing_path = os.path.join(tmpdir, "missing-user-ca-key.pem")
+            with override_settings(KEYWARDEN_CA_KEY_DIR=tmpdir):
+                ca = SSHCertificateAuthority.objects.create(
+                    name="User CA Recovery",
+                    public_key="ssh-ed25519 AAAAinvalid",
+                    private_key=f"file://{missing_path}",
+                    is_active=True,
+                )
+
+                ca.ensure_material()
+                ca.save()
+
+                self.assertTrue(ca.public_key.startswith("ssh-ed25519 "))
+                self.assertTrue(ca.fingerprint.startswith("SHA256:"))
+                key_path = parse_secret_ref(ca.private_key)
+                self.assertTrue(key_path)
+                self.assertTrue(os.path.exists(key_path))
+                self.assertIn("BEGIN OPENSSH PRIVATE KEY", ca.get_private_key())
+
+    def test_agent_ca_recovers_when_private_key_file_is_missing(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            missing_path = os.path.join(tmpdir, "missing-agent-ca-key.pem")
+            with override_settings(KEYWARDEN_CA_KEY_DIR=tmpdir):
+                ca = AgentCertificateAuthority.objects.create(
+                    name="Agent CA Recovery",
+                    cert_pem="",
+                    key_pem=f"file://{missing_path}",
+                    is_active=True,
+                )
+
+                ca.ensure_material()
+                ca.save()
+
+                self.assertIn("BEGIN CERTIFICATE", ca.cert_pem)
+                self.assertTrue(ca.fingerprint)
+                self.assertTrue(ca.serial)
+                key_path = parse_secret_ref(ca.key_pem)
+                self.assertTrue(key_path)
+                self.assertTrue(os.path.exists(key_path))
+                self.assertIn("BEGIN RSA PRIVATE KEY", ca.get_key_pem())

app/apps/servers/tests/test_consumers.py

@@ -455,6 +455,57 @@ class SecurityAndCertificateHelperTests(TestCase):
         )
         self.assertIsNone(auth.authenticate(bad_fingerprint))
 
+    def test_agent_runtime_auth_falls_back_to_server_token_without_mtls_headers(self):
+        server_token = "server-token-123"
+        server = Server.objects.create(
+            display_name="token-fallback-1",
+            hostname="token-fallback-1.example.test",
+            agent_api_token_hash=api_security.hash_agent_token(server_token),
+        )
+        request = SimpleNamespace(
+            META={"HTTP_AUTHORIZATION": f"Bearer {server_token}"},
+            path="/api/v1/agent/servers/1/log-config",
+        )
+        auth = api_security.AgentRuntimeAuth()
+
+        principal = auth.authenticate(request)
+
+        self.assertIsNotNone(principal)
+        self.assertEqual(principal.mode, "server-token")
+        self.assertEqual(principal.server_id, server.id)
+
+    @override_settings(KEYWARDEN_AGENT_API_TOKEN="global-token-123")
+    def test_agent_runtime_auth_accepts_global_token_without_mtls_headers(self):
+        request = SimpleNamespace(
+            META={"HTTP_AUTHORIZATION": "Bearer global-token-123"},
+            path="/api/v1/agent/servers/1/log-config",
+        )
+        auth = api_security.AgentRuntimeAuth()
+
+        principal = auth.authenticate(request)
+
+        self.assertIsNotNone(principal)
+        self.assertEqual(principal.mode, "global-token")
+        self.assertIsNone(principal.server_id)
+
+    @override_settings(KEYWARDEN_AGENT_RUNTIME_ALLOW_TOKEN_FALLBACK=False)
+    def test_agent_runtime_auth_can_disable_token_fallback(self):
+        server_token = "server-token-strict-123"
+        Server.objects.create(
+            display_name="token-fallback-off-1",
+            hostname="token-fallback-off-1.example.test",
+            agent_api_token_hash=api_security.hash_agent_token(server_token),
+        )
+        request = SimpleNamespace(
+            META={"HTTP_AUTHORIZATION": f"Bearer {server_token}"},
+            path="/api/v1/agent/servers/1/log-config",
+        )
+        auth = api_security.AgentRuntimeAuth()
+
+        principal = auth.authenticate(request)
+
+        self.assertIsNone(principal)
+
     def test_sanitize_username_and_render_system_username(self):
         self.assertEqual(sanitize_username("Alice Smith"), "alice_smith")
         self.assertEqual(sanitize_username("  "), "")