gleep fixes mtls and stuff
This commit is contained in:
1 parent
31b726b439
commit
779db4cd94
16 files changed
+504
-23
No files matched your search
@@ -17,6 +17,10 @@ KEYWARDEN_CA_KEY_DIR=/var/lib/keywarden/ca-keys
|
|||||||
# Optional legacy global fallback token for agents.
|
# Optional legacy global fallback token for agents.
|
||||||
# Preferred model is per-agent token issued during /agent/enroll.
|
# Preferred model is per-agent token issued during /agent/enroll.
|
||||||
# KEYWARDEN_AGENT_API_TOKEN=
|
# KEYWARDEN_AGENT_API_TOKEN=
|
||||||
|
# Runtime compatibility switch:
|
||||||
|
# true = allow bearer fallback when mTLS headers are absent (default)
|
||||||
|
# false = strict mTLS-only runtime auth
|
||||||
|
# KEYWARDEN_AGENT_RUNTIME_ALLOW_TOKEN_FALLBACK=true
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
KEYWARDEN_POSTGRES_DB=keywarden
|
KEYWARDEN_POSTGRES_DB=keywarden
|
||||||
|
|||||||
+10
-2
@@ -13,10 +13,18 @@ go build -o keywarden-agent ./cmd/keywarden-agent
|
|||||||
## Run
|
## Run
|
||||||
|
|
||||||
```
|
```
|
||||||
./keywarden-agent -config /etc/keywarden/agent.json -server-url https://keywarden.example.com -enroll-token <token>
|
./keywarden-agent --config /etc/keywarden/agent.json --server-url https://keywarden.example.com --enroll-token <token>
|
||||||
```
|
```
|
||||||
|
|
||||||
To rotate/re-enroll an existing agent identity, add `-force-enroll` with a fresh token.
|
Short aliases are also supported:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./keywarden-agent -c /etc/keywarden/agent.json -t <token> -f
|
||||||
|
```
|
||||||
|
|
||||||
|
To rotate/re-enroll an existing agent identity in place, use `-f/--force-enroll` with a fresh token.
|
||||||
|
When `server_id` already exists in config, force-enroll rotates the client certificate for that server
|
||||||
|
instead of creating a new server record.
|
||||||
|
|
||||||
You can also pass `KEYWARDEN_SERVER_URL` and `KEYWARDEN_ENROLL_TOKEN` as environment variables.
|
You can also pass `KEYWARDEN_SERVER_URL` and `KEYWARDEN_ENROLL_TOKEN` as environment variables.
|
||||||
|
|
||||||
|
|||||||
@@ -32,9 +32,12 @@ var lastNoEventsNotice time.Time
|
|||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
configPath := flag.String("config", config.DefaultConfigPath, "Path to agent config JSON")
|
configPath := flag.String("config", config.DefaultConfigPath, "Path to agent config JSON")
|
||||||
|
flag.StringVar(configPath, "c", config.DefaultConfigPath, "Path to agent config JSON (shorthand)")
|
||||||
serverURL := flag.String("server-url", "", "Keywarden server URL (first boot)")
|
serverURL := flag.String("server-url", "", "Keywarden server URL (first boot)")
|
||||||
enrollToken := flag.String("enroll-token", "", "Enrollment token (first boot)")
|
enrollToken := flag.String("enroll-token", "", "Enrollment token (first boot)")
|
||||||
|
flag.StringVar(enrollToken, "t", "", "Enrollment token (first boot, shorthand)")
|
||||||
forceEnroll := flag.Bool("force-enroll", false, "Force re-enrollment even if already bootstrapped")
|
forceEnroll := flag.Bool("force-enroll", false, "Force re-enrollment even if already bootstrapped")
|
||||||
|
flag.BoolVar(forceEnroll, "f", false, "Force re-enrollment even if already bootstrapped (shorthand)")
|
||||||
showVersion := flag.Bool("version", false, "Print version and exit")
|
showVersion := flag.Bool("version", false, "Print version and exit")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
@@ -236,17 +239,21 @@ func bootstrapIfNeeded(cfg *config.Config, configPath string, enrollToken string
|
|||||||
previousServerID := strings.TrimSpace(cfg.ServerID)
|
previousServerID := strings.TrimSpace(cfg.ServerID)
|
||||||
if !forceEnroll && cfg.ServerID != "" && fileExists(cfg.ClientCertPath()) && fileExists(cfg.CACertPath()) {
|
if !forceEnroll && cfg.ServerID != "" && fileExists(cfg.ClientCertPath()) && fileExists(cfg.CACertPath()) {
|
||||||
if strings.TrimSpace(enrollToken) != "" {
|
if strings.TrimSpace(enrollToken) != "" {
|
||||||
log.Printf("enrollment token provided but bootstrap is already complete; skipping enrollment (use -force-enroll to rotate identity)")
|
log.Printf("enrollment token provided but bootstrap is already complete; skipping enrollment (use -f/--force-enroll to rotate identity)")
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if enrollToken == "" {
|
if enrollToken == "" {
|
||||||
return fmt.Errorf("missing enrollment token; set KEYWARDEN_ENROLL_TOKEN or -enroll-token")
|
return fmt.Errorf("missing enrollment token; set KEYWARDEN_ENROLL_TOKEN or -t/--enroll-token")
|
||||||
}
|
}
|
||||||
|
rotateServerID := ""
|
||||||
if forceEnroll {
|
if forceEnroll {
|
||||||
log.Printf("force re-enrollment requested; replacing existing server identity")
|
rotateServerID = previousServerID
|
||||||
cfg.ServerID = ""
|
if rotateServerID != "" {
|
||||||
cfg.AgentAPIToken = ""
|
log.Printf("force re-enrollment requested; rotating certificate in-place for server_id=%s", rotateServerID)
|
||||||
|
} else {
|
||||||
|
log.Printf("force re-enrollment requested; no existing server_id, enrolling as new server identity")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
keyPath := cfg.ClientKeyPath()
|
keyPath := cfg.ClientKeyPath()
|
||||||
if !fileExists(keyPath) {
|
if !fileExists(keyPath) {
|
||||||
@@ -266,10 +273,18 @@ func bootstrapIfNeeded(cfg *config.Config, configPath string, enrollToken string
|
|||||||
Host: hostname,
|
Host: hostname,
|
||||||
IPv4: info.IPv4,
|
IPv4: info.IPv4,
|
||||||
IPv6: info.IPv6,
|
IPv6: info.IPv6,
|
||||||
|
ServerID: rotateServerID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if rotateServerID != "" && strings.TrimSpace(resp.ServerID) != rotateServerID {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"re-enrollment returned unexpected server_id=%s (expected %s)",
|
||||||
|
strings.TrimSpace(resp.ServerID),
|
||||||
|
rotateServerID,
|
||||||
|
)
|
||||||
|
}
|
||||||
if err := os.WriteFile(cfg.ClientCertPath(), []byte(resp.ClientCert), 0o600); err != nil {
|
if err := os.WriteFile(cfg.ClientCertPath(), []byte(resp.ClientCert), 0o600); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-3
@@ -1,7 +1,6 @@
|
|||||||
{
|
{
|
||||||
"server_url": "https://keywarden.dev.ntbx.io/api/v1",
|
"server_url": "https://agent.keywarden.dev.ntbx.io/api/v1",
|
||||||
"server_id": "6",
|
"server_id": "13",
|
||||||
"agent_api_token": "HV8MnbysmINShu9wkXzzNrQE5NyRlPJHfP2aU6SX4HERrlac8EQpoMEu02FX1TYv",
|
|
||||||
"sync_interval_seconds": 5,
|
"sync_interval_seconds": 5,
|
||||||
"log_batch_size": 2000,
|
"log_batch_size": 2000,
|
||||||
"state_dir": "/var/lib/keywarden-agent",
|
"state_dir": "/var/lib/keywarden-agent",
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ func New(cfg *config.Config) (*Client, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("load client cert: %w", err)
|
return nil, fmt.Errorf("load client cert: %w", err)
|
||||||
}
|
}
|
||||||
|
clientCert := cert
|
||||||
caPool, err := x509.SystemCertPool()
|
caPool, err := x509.SystemCertPool()
|
||||||
if err != nil || caPool == nil {
|
if err != nil || caPool == nil {
|
||||||
caPool = x509.NewCertPool()
|
caPool = x509.NewCertPool()
|
||||||
@@ -70,6 +71,12 @@ func New(cfg *config.Config) (*Client, error) {
|
|||||||
Certificates: []tls.Certificate{cert},
|
Certificates: []tls.Certificate{cert},
|
||||||
RootCAs: caPool,
|
RootCAs: caPool,
|
||||||
MinVersion: tls.VersionTLS12,
|
MinVersion: tls.VersionTLS12,
|
||||||
|
// Always return the enrolled client certificate when the server
|
||||||
|
// requests mTLS credentials. This avoids edge cases where automatic
|
||||||
|
// certificate selection declines to send a cert.
|
||||||
|
GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||||
|
return &clientCert, nil
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
transport := &http.Transport{
|
transport := &http.Transport{
|
||||||
@@ -127,6 +134,7 @@ type EnrollRequest struct {
|
|||||||
Host string `json:"host"`
|
Host string `json:"host"`
|
||||||
IPv4 string `json:"ipv4,omitempty"`
|
IPv4 string `json:"ipv4,omitempty"`
|
||||||
IPv6 string `json:"ipv6,omitempty"`
|
IPv6 string `json:"ipv6,omitempty"`
|
||||||
|
ServerID string `json:"server_id,omitempty"`
|
||||||
AgentID string `json:"agent_id,omitempty"`
|
AgentID string `json:"agent_id,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
@@ -107,7 +107,15 @@ class SSHCertificateAuthority(models.Model):
|
|||||||
self.revoked_at = timezone.now()
|
self.revoked_at = timezone.now()
|
||||||
|
|
||||||
def get_private_key(self) -> str:
|
def get_private_key(self) -> str:
|
||||||
|
try:
|
||||||
file_payload = read_secret_ref(self.private_key, label="User CA private key")
|
file_payload = read_secret_ref(self.private_key, label="User CA private key")
|
||||||
|
except RuntimeError as exc:
|
||||||
|
# Recover gracefully when a secret-file reference points to a
|
||||||
|
# deleted key file (for example after volume resets).
|
||||||
|
if "file missing:" in str(exc):
|
||||||
|
file_payload = ""
|
||||||
|
else:
|
||||||
|
raise
|
||||||
if file_payload:
|
if file_payload:
|
||||||
return file_payload
|
return file_payload
|
||||||
inline_payload = (self.private_key or "").strip()
|
inline_payload = (self.private_key or "").strip()
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from django.contrib import admin
|
from django.contrib import admin
|
||||||
|
from django.conf import settings
|
||||||
from django.utils.html import format_html
|
from django.utils.html import format_html
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -14,7 +15,16 @@ except ImportError: # Fallback for older Unfold builds without guardian admin s
|
|||||||
class GuardedModelAdmin(GuardianGuardedModelAdmin, UnfoldModelAdmin):
|
class GuardedModelAdmin(GuardianGuardedModelAdmin, UnfoldModelAdmin):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
from .models import AgentCertificateAuthority, EnrollmentToken, Server, ServerAuditLog, ServerLogSource
|
from apps.access.models import AccessRequest
|
||||||
|
|
||||||
|
from .models import (
|
||||||
|
AgentCertificateAuthority,
|
||||||
|
EnrollmentToken,
|
||||||
|
Server,
|
||||||
|
ServerAccount,
|
||||||
|
ServerAuditLog,
|
||||||
|
ServerLogSource,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class ServerLogSourceInline(admin.TabularInline):
|
class ServerLogSourceInline(admin.TabularInline):
|
||||||
@@ -54,6 +64,64 @@ class ServerAdmin(GuardedModelAdmin):
|
|||||||
)
|
)
|
||||||
inlines = (ServerLogSourceInline,)
|
inlines = (ServerLogSourceInline,)
|
||||||
|
|
||||||
|
def _large_delete_preview_threshold(self) -> int:
|
||||||
|
value = getattr(settings, "KEYWARDEN_ADMIN_DELETE_PREVIEW_MAX_ITEMS", 5000)
|
||||||
|
try:
|
||||||
|
threshold = int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
threshold = 5000
|
||||||
|
return max(threshold, 1)
|
||||||
|
|
||||||
|
def _related_delete_counts(self, server_ids: list[int]) -> dict[str, int]:
|
||||||
|
return {
|
||||||
|
ServerAuditLog._meta.verbose_name_plural: ServerAuditLog.objects.filter(server_id__in=server_ids).count(),
|
||||||
|
ServerAccount._meta.verbose_name_plural: ServerAccount.objects.filter(server_id__in=server_ids).count(),
|
||||||
|
ServerLogSource._meta.verbose_name_plural: ServerLogSource.objects.filter(server_id__in=server_ids).count(),
|
||||||
|
AccessRequest._meta.verbose_name_plural: AccessRequest.objects.filter(server_id__in=server_ids).count(),
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_deleted_objects(self, objs, request):
|
||||||
|
# Django's default nested object preview scales poorly for very large
|
||||||
|
# cascades (for example, many ServerAuditLog rows) and can cause
|
||||||
|
# worker timeouts/OOM during delete confirmation rendering.
|
||||||
|
if not request.user.is_superuser:
|
||||||
|
return super().get_deleted_objects(objs, request)
|
||||||
|
|
||||||
|
server_ids = list(objs.values_list("id", flat=True))
|
||||||
|
if not server_ids:
|
||||||
|
return super().get_deleted_objects(objs, request)
|
||||||
|
|
||||||
|
related_counts = self._related_delete_counts(server_ids)
|
||||||
|
total_related = sum(related_counts.values())
|
||||||
|
if total_related < self._large_delete_preview_threshold():
|
||||||
|
return super().get_deleted_objects(objs, request)
|
||||||
|
|
||||||
|
model_count = {
|
||||||
|
self.model._meta.verbose_name_plural: len(server_ids),
|
||||||
|
}
|
||||||
|
deleted_objects = [f"{len(server_ids)} {self.model._meta.verbose_name_plural}"]
|
||||||
|
for label, count in related_counts.items():
|
||||||
|
if count:
|
||||||
|
model_count[label] = count
|
||||||
|
deleted_objects.append(f"{count} {label}")
|
||||||
|
deleted_objects.append(
|
||||||
|
"Large cascade detected. Related objects are summarized here to keep delete confirmation responsive."
|
||||||
|
)
|
||||||
|
return deleted_objects, model_count, set(), []
|
||||||
|
|
||||||
|
def _purge_server_audit_logs(self, server_ids: list[int]) -> None:
|
||||||
|
if not server_ids:
|
||||||
|
return
|
||||||
|
ServerAuditLog.objects.filter(server_id__in=server_ids).delete()
|
||||||
|
|
||||||
|
def delete_model(self, request, obj) -> None:
|
||||||
|
self._purge_server_audit_logs([obj.id])
|
||||||
|
super().delete_model(request, obj)
|
||||||
|
|
||||||
|
def delete_queryset(self, request, queryset) -> None:
|
||||||
|
self._purge_server_audit_logs(list(queryset.values_list("id", flat=True)))
|
||||||
|
super().delete_queryset(request, queryset)
|
||||||
|
|
||||||
def avatar(self, obj: Server):
|
def avatar(self, obj: Server):
|
||||||
if obj.image_url:
|
if obj.image_url:
|
||||||
return format_html(
|
return format_html(
|
||||||
|
|||||||
@@ -139,7 +139,15 @@ class AgentCertificateAuthority(models.Model):
|
|||||||
self.revoked_at = timezone.now()
|
self.revoked_at = timezone.now()
|
||||||
|
|
||||||
def get_key_pem(self) -> str:
|
def get_key_pem(self) -> str:
|
||||||
|
try:
|
||||||
file_payload = read_secret_ref(self.key_pem, label="Agent CA private key")
|
file_payload = read_secret_ref(self.key_pem, label="Agent CA private key")
|
||||||
|
except RuntimeError as exc:
|
||||||
|
# Recover gracefully when a secret-file reference points to a
|
||||||
|
# deleted key file (for example after volume resets).
|
||||||
|
if "file missing:" in str(exc):
|
||||||
|
file_payload = ""
|
||||||
|
else:
|
||||||
|
raise
|
||||||
if file_payload:
|
if file_payload:
|
||||||
return file_payload
|
return file_payload
|
||||||
inline_payload = (self.key_pem or "").strip()
|
inline_payload = (self.key_pem or "").strip()
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from django.contrib.admin.sites import AdminSite
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.test import RequestFactory, TestCase, override_settings
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from apps.access.models import AccessRequest
|
||||||
|
from apps.servers.admin import ServerAdmin
|
||||||
|
from apps.servers.models import Server, ServerAccount, ServerAuditLog, ServerLogSource
|
||||||
|
|
||||||
|
|
||||||
|
class ServerAdminDeleteTests(TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
super().setUp()
|
||||||
|
User = get_user_model()
|
||||||
|
self.superuser = User.objects.create_superuser(
|
||||||
|
username="admin-delete",
|
||||||
|
email="admin-delete@example.com",
|
||||||
|
password="pass12345",
|
||||||
|
)
|
||||||
|
self.user = User.objects.create_user(
|
||||||
|
username="member-delete",
|
||||||
|
email="member-delete@example.com",
|
||||||
|
password="pass12345",
|
||||||
|
)
|
||||||
|
self.site = AdminSite()
|
||||||
|
self.model_admin = ServerAdmin(Server, self.site)
|
||||||
|
self.factory = RequestFactory()
|
||||||
|
|
||||||
|
def _request(self):
|
||||||
|
request = self.factory.post("/admin/servers/server/")
|
||||||
|
request.user = self.superuser
|
||||||
|
return request
|
||||||
|
|
||||||
|
@override_settings(KEYWARDEN_ADMIN_DELETE_PREVIEW_MAX_ITEMS=2)
|
||||||
|
def test_get_deleted_objects_summarizes_large_cascade(self) -> None:
|
||||||
|
server = Server.objects.create(display_name="srv-admin-delete")
|
||||||
|
ServerAuditLog.objects.bulk_create(
|
||||||
|
[
|
||||||
|
ServerAuditLog(
|
||||||
|
server=server,
|
||||||
|
event_at=timezone.now(),
|
||||||
|
category="auth",
|
||||||
|
event_type="login",
|
||||||
|
),
|
||||||
|
ServerAuditLog(
|
||||||
|
server=server,
|
||||||
|
event_at=timezone.now(),
|
||||||
|
category="auth",
|
||||||
|
event_type="logout",
|
||||||
|
),
|
||||||
|
ServerAuditLog(
|
||||||
|
server=server,
|
||||||
|
event_at=timezone.now(),
|
||||||
|
category="auth",
|
||||||
|
event_type="sudo",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
ServerAccount.objects.create(server=server, user=self.user, system_username="member_delete")
|
||||||
|
ServerLogSource.objects.create(server=server, kind=ServerLogSource.Kind.JOURNAL, name="system-journal")
|
||||||
|
AccessRequest.objects.create(requester=self.user, server=server, request_shell=True)
|
||||||
|
|
||||||
|
deleted_objects, model_count, perms_needed, protected = self.model_admin.get_deleted_objects(
|
||||||
|
Server.objects.filter(id=server.id),
|
||||||
|
self._request(),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(perms_needed, set())
|
||||||
|
self.assertEqual(protected, [])
|
||||||
|
self.assertIn("Large cascade detected", " ".join(str(item) for item in deleted_objects))
|
||||||
|
self.assertEqual(model_count[Server._meta.verbose_name_plural], 1)
|
||||||
|
self.assertEqual(model_count[ServerAuditLog._meta.verbose_name_plural], 3)
|
||||||
|
self.assertEqual(model_count[ServerAccount._meta.verbose_name_plural], 1)
|
||||||
|
self.assertEqual(model_count[ServerLogSource._meta.verbose_name_plural], 1)
|
||||||
|
self.assertEqual(model_count[AccessRequest._meta.verbose_name_plural], 1)
|
||||||
|
|
||||||
|
def test_delete_queryset_prepurges_audit_logs(self) -> None:
|
||||||
|
server = Server.objects.create(display_name="srv-admin-delete-purge")
|
||||||
|
ServerAuditLog.objects.bulk_create(
|
||||||
|
[
|
||||||
|
ServerAuditLog(
|
||||||
|
server=server,
|
||||||
|
event_at=timezone.now(),
|
||||||
|
category="auth",
|
||||||
|
event_type="login",
|
||||||
|
),
|
||||||
|
ServerAuditLog(
|
||||||
|
server=server,
|
||||||
|
event_at=timezone.now(),
|
||||||
|
category="auth",
|
||||||
|
event_type="logout",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
self.model_admin.delete_queryset(self._request(), Server.objects.filter(id=server.id))
|
||||||
|
|
||||||
|
self.assertFalse(Server.objects.filter(id=server.id).exists())
|
||||||
|
self.assertFalse(ServerAuditLog.objects.filter(server_id=server.id).exists())
|
||||||
@@ -21,7 +21,7 @@ from ninja.errors import HttpError
|
|||||||
from apps.access.models import AccessRequest
|
from apps.access.models import AccessRequest
|
||||||
from apps.keys.models import SSHKey
|
from apps.keys.models import SSHKey
|
||||||
from apps.keys.models import SSHCertificate, SSHCertificateAuthority, parse_public_key
|
from apps.keys.models import SSHCertificate, SSHCertificateAuthority, parse_public_key
|
||||||
from apps.servers.models import AgentCertificateAuthority, Server, ServerAuditLog, ServerLogSource
|
from apps.servers.models import AgentCertificateAuthority, EnrollmentToken, Server, ServerAuditLog, ServerLogSource
|
||||||
from keywarden.api.routers import agent as agent_router
|
from keywarden.api.routers import agent as agent_router
|
||||||
|
|
||||||
|
|
||||||
@@ -472,6 +472,10 @@ class AgentRouterHelpersTests(TestCase):
|
|||||||
SimpleNamespace(auth=agent_router.AgentPrincipal(server_id=self.server.id, mode="mtls")),
|
SimpleNamespace(auth=agent_router.AgentPrincipal(server_id=self.server.id, mode="mtls")),
|
||||||
self.server.id,
|
self.server.id,
|
||||||
)
|
)
|
||||||
|
agent_router._require_agent_access(
|
||||||
|
SimpleNamespace(auth=agent_router.AgentPrincipal(server_id=None, mode="global-token")),
|
||||||
|
self.server.id,
|
||||||
|
)
|
||||||
self.assertNotEqual(agent_router._issue_agent_api_token(), agent_router._issue_agent_api_token())
|
self.assertNotEqual(agent_router._issue_agent_api_token(), agent_router._issue_agent_api_token())
|
||||||
|
|
||||||
def test_server_lookup_key_map_and_account_sync_helpers(self) -> None:
|
def test_server_lookup_key_map_and_account_sync_helpers(self) -> None:
|
||||||
@@ -543,6 +547,65 @@ class AgentRouterHelpersTests(TestCase):
|
|||||||
self.assertTrue(fingerprint)
|
self.assertTrue(fingerprint)
|
||||||
self.assertTrue(serial)
|
self.assertTrue(serial)
|
||||||
|
|
||||||
|
def test_enroll_agent_rotates_existing_server_when_server_id_is_provided(self) -> None:
|
||||||
|
ca_cert, _ca_key, ca_pem, key_pem = self._build_ca_material()
|
||||||
|
agent_router.AgentCertificateAuthority.objects.create(
|
||||||
|
name="Enroll CA",
|
||||||
|
cert_pem=ca_pem,
|
||||||
|
key_pem=key_pem,
|
||||||
|
is_active=True,
|
||||||
|
)
|
||||||
|
token = EnrollmentToken.objects.create(token="rotate-token")
|
||||||
|
original_server_count = Server.objects.count()
|
||||||
|
self.server.agent_cert_fingerprint = "deadbeef"
|
||||||
|
self.server.agent_cert_serial = "1"
|
||||||
|
self.server.save(update_fields=["agent_cert_fingerprint", "agent_cert_serial"])
|
||||||
|
|
||||||
|
csr = self._build_csr("rotate-agent.example.test")
|
||||||
|
payload = {
|
||||||
|
"token": token.token,
|
||||||
|
"csr_pem": csr.public_bytes(serialization.Encoding.PEM).decode("utf-8"),
|
||||||
|
"server_id": str(self.server.id),
|
||||||
|
"host": self.server.hostname,
|
||||||
|
}
|
||||||
|
response = self.client.post(
|
||||||
|
"/api/v1/agent/enroll",
|
||||||
|
data=json.dumps(payload),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
body = response.json()
|
||||||
|
self.assertEqual(body["server_id"], str(self.server.id))
|
||||||
|
self.assertIn("BEGIN CERTIFICATE", body["client_cert_pem"])
|
||||||
|
self.assertIn("BEGIN CERTIFICATE", body["ca_cert_pem"])
|
||||||
|
self.assertTrue(body["agent_api_token"])
|
||||||
|
self.assertEqual(Server.objects.count(), original_server_count)
|
||||||
|
|
||||||
|
self.server.refresh_from_db()
|
||||||
|
self.assertNotEqual(self.server.agent_cert_fingerprint, "deadbeef")
|
||||||
|
self.assertNotEqual(self.server.agent_cert_serial, "1")
|
||||||
|
self.assertIsNotNone(self.server.agent_enrolled_at)
|
||||||
|
token.refresh_from_db()
|
||||||
|
self.assertEqual(token.server_id, self.server.id)
|
||||||
|
self.assertIsNotNone(token.used_at)
|
||||||
|
|
||||||
|
def test_enroll_agent_rejects_invalid_server_id(self) -> None:
|
||||||
|
token = EnrollmentToken.objects.create(token="rotate-token-bad")
|
||||||
|
csr = self._build_csr("rotate-agent.example.test")
|
||||||
|
payload = {
|
||||||
|
"token": token.token,
|
||||||
|
"csr_pem": csr.public_bytes(serialization.Encoding.PEM).decode("utf-8"),
|
||||||
|
"server_id": "not-a-number",
|
||||||
|
}
|
||||||
|
response = self.client.post(
|
||||||
|
"/api/v1/agent/enroll",
|
||||||
|
data=json.dumps(payload),
|
||||||
|
content_type="application/json",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 422)
|
||||||
|
self.assertIn("server_id must be numeric", response.content.decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
class KeysModelHelpersTests(TestCase):
|
class KeysModelHelpersTests(TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from django.test import TestCase, override_settings
|
||||||
|
|
||||||
|
from apps.core.secret_files import parse_secret_ref
|
||||||
|
from apps.keys.models import SSHCertificateAuthority
|
||||||
|
from apps.servers.models import AgentCertificateAuthority
|
||||||
|
|
||||||
|
|
||||||
|
class CASecretFileRecoveryTests(TestCase):
|
||||||
|
def test_user_ca_recovers_when_private_key_file_is_missing(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
missing_path = os.path.join(tmpdir, "missing-user-ca-key.pem")
|
||||||
|
with override_settings(KEYWARDEN_CA_KEY_DIR=tmpdir):
|
||||||
|
ca = SSHCertificateAuthority.objects.create(
|
||||||
|
name="User CA Recovery",
|
||||||
|
public_key="ssh-ed25519 AAAAinvalid",
|
||||||
|
private_key=f"file://{missing_path}",
|
||||||
|
is_active=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
ca.ensure_material()
|
||||||
|
ca.save()
|
||||||
|
|
||||||
|
self.assertTrue(ca.public_key.startswith("ssh-ed25519 "))
|
||||||
|
self.assertTrue(ca.fingerprint.startswith("SHA256:"))
|
||||||
|
key_path = parse_secret_ref(ca.private_key)
|
||||||
|
self.assertTrue(key_path)
|
||||||
|
self.assertTrue(os.path.exists(key_path))
|
||||||
|
self.assertIn("BEGIN OPENSSH PRIVATE KEY", ca.get_private_key())
|
||||||
|
|
||||||
|
def test_agent_ca_recovers_when_private_key_file_is_missing(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
missing_path = os.path.join(tmpdir, "missing-agent-ca-key.pem")
|
||||||
|
with override_settings(KEYWARDEN_CA_KEY_DIR=tmpdir):
|
||||||
|
ca = AgentCertificateAuthority.objects.create(
|
||||||
|
name="Agent CA Recovery",
|
||||||
|
cert_pem="",
|
||||||
|
key_pem=f"file://{missing_path}",
|
||||||
|
is_active=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
ca.ensure_material()
|
||||||
|
ca.save()
|
||||||
|
|
||||||
|
self.assertIn("BEGIN CERTIFICATE", ca.cert_pem)
|
||||||
|
self.assertTrue(ca.fingerprint)
|
||||||
|
self.assertTrue(ca.serial)
|
||||||
|
key_path = parse_secret_ref(ca.key_pem)
|
||||||
|
self.assertTrue(key_path)
|
||||||
|
self.assertTrue(os.path.exists(key_path))
|
||||||
|
self.assertIn("BEGIN RSA PRIVATE KEY", ca.get_key_pem())
|
||||||
@@ -455,6 +455,57 @@ class SecurityAndCertificateHelperTests(TestCase):
|
|||||||
)
|
)
|
||||||
self.assertIsNone(auth.authenticate(bad_fingerprint))
|
self.assertIsNone(auth.authenticate(bad_fingerprint))
|
||||||
|
|
||||||
|
def test_agent_runtime_auth_falls_back_to_server_token_without_mtls_headers(self):
|
||||||
|
server_token = "server-token-123"
|
||||||
|
server = Server.objects.create(
|
||||||
|
display_name="token-fallback-1",
|
||||||
|
hostname="token-fallback-1.example.test",
|
||||||
|
agent_api_token_hash=api_security.hash_agent_token(server_token),
|
||||||
|
)
|
||||||
|
request = SimpleNamespace(
|
||||||
|
META={"HTTP_AUTHORIZATION": f"Bearer {server_token}"},
|
||||||
|
path="/api/v1/agent/servers/1/log-config",
|
||||||
|
)
|
||||||
|
auth = api_security.AgentRuntimeAuth()
|
||||||
|
|
||||||
|
principal = auth.authenticate(request)
|
||||||
|
|
||||||
|
self.assertIsNotNone(principal)
|
||||||
|
self.assertEqual(principal.mode, "server-token")
|
||||||
|
self.assertEqual(principal.server_id, server.id)
|
||||||
|
|
||||||
|
@override_settings(KEYWARDEN_AGENT_API_TOKEN="global-token-123")
|
||||||
|
def test_agent_runtime_auth_accepts_global_token_without_mtls_headers(self):
|
||||||
|
request = SimpleNamespace(
|
||||||
|
META={"HTTP_AUTHORIZATION": "Bearer global-token-123"},
|
||||||
|
path="/api/v1/agent/servers/1/log-config",
|
||||||
|
)
|
||||||
|
auth = api_security.AgentRuntimeAuth()
|
||||||
|
|
||||||
|
principal = auth.authenticate(request)
|
||||||
|
|
||||||
|
self.assertIsNotNone(principal)
|
||||||
|
self.assertEqual(principal.mode, "global-token")
|
||||||
|
self.assertIsNone(principal.server_id)
|
||||||
|
|
||||||
|
@override_settings(KEYWARDEN_AGENT_RUNTIME_ALLOW_TOKEN_FALLBACK=False)
|
||||||
|
def test_agent_runtime_auth_can_disable_token_fallback(self):
|
||||||
|
server_token = "server-token-strict-123"
|
||||||
|
Server.objects.create(
|
||||||
|
display_name="token-fallback-off-1",
|
||||||
|
hostname="token-fallback-off-1.example.test",
|
||||||
|
agent_api_token_hash=api_security.hash_agent_token(server_token),
|
||||||
|
)
|
||||||
|
request = SimpleNamespace(
|
||||||
|
META={"HTTP_AUTHORIZATION": f"Bearer {server_token}"},
|
||||||
|
path="/api/v1/agent/servers/1/log-config",
|
||||||
|
)
|
||||||
|
auth = api_security.AgentRuntimeAuth()
|
||||||
|
|
||||||
|
principal = auth.authenticate(request)
|
||||||
|
|
||||||
|
self.assertIsNone(principal)
|
||||||
|
|
||||||
def test_sanitize_username_and_render_system_username(self):
|
def test_sanitize_username_and_render_system_username(self):
|
||||||
self.assertEqual(sanitize_username("Alice Smith"), "alice_smith")
|
self.assertEqual(sanitize_username("Alice Smith"), "alice_smith")
|
||||||
self.assertEqual(sanitize_username(" "), "")
|
self.assertEqual(sanitize_username(" "), "")
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ from apps.servers.models import (
|
|||||||
hostname_validator,
|
hostname_validator,
|
||||||
)
|
)
|
||||||
from apps.telemetry.models import TelemetryEvent
|
from apps.telemetry.models import TelemetryEvent
|
||||||
from keywarden.api.security import AgentMTLSAuth, AgentPrincipal, hash_agent_token
|
from keywarden.api.security import AgentPrincipal, AgentRuntimeAuth, hash_agent_token
|
||||||
|
|
||||||
|
|
||||||
class AuthorizedKeyOut(Schema):
|
class AuthorizedKeyOut(Schema):
|
||||||
@@ -80,6 +80,7 @@ class SyncReportOut(Schema):
|
|||||||
class AgentEnrollIn(Schema):
|
class AgentEnrollIn(Schema):
|
||||||
token: str
|
token: str
|
||||||
csr_pem: str
|
csr_pem: str
|
||||||
|
server_id: Optional[str] = None
|
||||||
host: Optional[str] = None
|
host: Optional[str] = None
|
||||||
ipv4: Optional[str] = None
|
ipv4: Optional[str] = None
|
||||||
ipv6: Optional[str] = None
|
ipv6: Optional[str] = None
|
||||||
@@ -137,7 +138,7 @@ class AgentHeartbeatIn(Schema):
|
|||||||
|
|
||||||
def build_router() -> Router:
|
def build_router() -> Router:
|
||||||
router = Router()
|
router = Router()
|
||||||
agent_auth = AgentMTLSAuth()
|
agent_auth = AgentRuntimeAuth()
|
||||||
|
|
||||||
@router.post("/enroll", response=AgentEnrollOut, auth=None)
|
@router.post("/enroll", response=AgentEnrollOut, auth=None)
|
||||||
@csrf_exempt
|
@csrf_exempt
|
||||||
@@ -148,7 +149,8 @@ def build_router() -> Router:
|
|||||||
enrollment completes.
|
enrollment completes.
|
||||||
Inputs: enrollment token + CSR from the agent, optional host/IP hints.
|
Inputs: enrollment token + CSR from the agent, optional host/IP hints.
|
||||||
Behavior:
|
Behavior:
|
||||||
- Creates a Server record (agent is the source of truth for host/IP).
|
- Creates a Server record (agent is the source of truth for host/IP), or
|
||||||
|
rotates credentials in-place when `server_id` is provided.
|
||||||
- Marks the token as used (single-use).
|
- Marks the token as used (single-use).
|
||||||
- Signs the CSR with the active Agent CA and returns client cert + CA.
|
- Signs the CSR with the active Agent CA and returns client cert + CA.
|
||||||
Rationale: this is the only supported server onboarding flow. If this
|
Rationale: this is the only supported server onboarding flow. If this
|
||||||
@@ -175,16 +177,34 @@ def build_router() -> Router:
|
|||||||
hostname = None
|
hostname = None
|
||||||
ipv4 = _normalize_ip(payload.ipv4, 4)
|
ipv4 = _normalize_ip(payload.ipv4, 4)
|
||||||
ipv6 = _normalize_ip(payload.ipv6, 6)
|
ipv6 = _normalize_ip(payload.ipv6, 6)
|
||||||
|
requested_server_id = _parse_requested_server_id(payload.server_id)
|
||||||
|
|
||||||
csr = _load_csr((payload.csr_pem or "").strip())
|
csr = _load_csr((payload.csr_pem or "").strip())
|
||||||
try:
|
try:
|
||||||
with transaction.atomic():
|
with transaction.atomic():
|
||||||
|
if requested_server_id is None:
|
||||||
server = Server.objects.create(
|
server = Server.objects.create(
|
||||||
display_name=display_name,
|
display_name=display_name,
|
||||||
hostname=hostname,
|
hostname=hostname,
|
||||||
ipv4=ipv4,
|
ipv4=ipv4,
|
||||||
ipv6=ipv6,
|
ipv6=ipv6,
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
server = Server.objects.select_for_update().get(id=requested_server_id)
|
||||||
|
except Server.DoesNotExist:
|
||||||
|
raise HttpError(404, "Server not found for re-enrollment")
|
||||||
|
|
||||||
|
# Optional host/IP updates are best-effort metadata refresh.
|
||||||
|
if host and (server.display_name or "").strip().lower() in {"", "server"}:
|
||||||
|
server.display_name = display_name
|
||||||
|
if hostname:
|
||||||
|
server.hostname = hostname
|
||||||
|
if ipv4:
|
||||||
|
server.ipv4 = ipv4
|
||||||
|
if ipv6:
|
||||||
|
server.ipv6 = ipv6
|
||||||
|
|
||||||
token.mark_used(server)
|
token.mark_used(server)
|
||||||
token.save(update_fields=["used_at", "server"])
|
token.save(update_fields=["used_at", "server"])
|
||||||
cert_pem, ca_pem, fingerprint, serial = _issue_client_cert(csr, host, server.id)
|
cert_pem, ca_pem, fingerprint, serial = _issue_client_cert(csr, host, server.id)
|
||||||
@@ -193,16 +213,11 @@ def build_router() -> Router:
|
|||||||
server.agent_cert_fingerprint = fingerprint
|
server.agent_cert_fingerprint = fingerprint
|
||||||
server.agent_cert_serial = serial
|
server.agent_cert_serial = serial
|
||||||
server.agent_api_token_hash = hash_agent_token(agent_api_token)
|
server.agent_api_token_hash = hash_agent_token(agent_api_token)
|
||||||
server.save(
|
server.save()
|
||||||
update_fields=[
|
|
||||||
"agent_enrolled_at",
|
|
||||||
"agent_cert_fingerprint",
|
|
||||||
"agent_cert_serial",
|
|
||||||
"agent_api_token_hash",
|
|
||||||
]
|
|
||||||
)
|
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
|
if requested_server_id is None:
|
||||||
raise HttpError(409, "Server already enrolled")
|
raise HttpError(409, "Server already enrolled")
|
||||||
|
raise HttpError(409, "Server address already in use")
|
||||||
|
|
||||||
return AgentEnrollOut(
|
return AgentEnrollOut(
|
||||||
server_id=str(server.id),
|
server_id=str(server.id),
|
||||||
@@ -440,6 +455,19 @@ def _get_server_or_404(server_id: int) -> Server:
|
|||||||
raise HttpError(404, "Server not found")
|
raise HttpError(404, "Server not found")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_requested_server_id(value: Optional[str]) -> Optional[int]:
|
||||||
|
text = str(value or "").strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
server_id = int(text)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
raise HttpError(422, "server_id must be numeric")
|
||||||
|
if server_id <= 0:
|
||||||
|
raise HttpError(422, "server_id must be positive")
|
||||||
|
return server_id
|
||||||
|
|
||||||
|
|
||||||
def _issue_agent_api_token() -> str:
|
def _issue_agent_api_token() -> str:
|
||||||
# Opaque, high-entropy token; only SHA-256 hash is stored server-side.
|
# Opaque, high-entropy token; only SHA-256 hash is stored server-side.
|
||||||
return secrets.token_urlsafe(48)
|
return secrets.token_urlsafe(48)
|
||||||
@@ -449,6 +477,9 @@ def _require_agent_access(request: HttpRequest, server_id: int) -> None:
|
|||||||
principal = getattr(request, "auth", None)
|
principal = getattr(request, "auth", None)
|
||||||
if not isinstance(principal, AgentPrincipal):
|
if not isinstance(principal, AgentPrincipal):
|
||||||
raise HttpError(401, "Unauthorized")
|
raise HttpError(401, "Unauthorized")
|
||||||
|
if principal.server_id is None:
|
||||||
|
# Global token compatibility mode: trust caller after auth success.
|
||||||
|
return
|
||||||
if int(principal.server_id) != int(server_id):
|
if int(principal.server_id) != int(server_id):
|
||||||
raise HttpError(403, "Forbidden")
|
raise HttpError(403, "Forbidden")
|
||||||
|
|
||||||
|
|||||||
@@ -90,6 +90,44 @@ class AgentTokenAuth(HttpBearer):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class AgentRuntimeAuth:
|
||||||
|
"""
|
||||||
|
Runtime agent auth: prefer mTLS, with bearer-token compatibility fallback.
|
||||||
|
|
||||||
|
Behavior:
|
||||||
|
- If a mTLS context appears present, attempt mTLS validation first.
|
||||||
|
- If mTLS is absent/invalid, fall back to enrollment-issued bearer token.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._mtls = AgentMTLSAuth()
|
||||||
|
self._token = AgentTokenAuth()
|
||||||
|
|
||||||
|
def __call__(self, request: HttpRequest) -> Optional["AgentPrincipal"]:
|
||||||
|
return self.authenticate(request)
|
||||||
|
|
||||||
|
def authenticate(self, request: HttpRequest) -> Optional["AgentPrincipal"]:
|
||||||
|
bearer = _extract_bearer_token_from_request(request)
|
||||||
|
allow_fallback = bool(
|
||||||
|
getattr(settings, "KEYWARDEN_AGENT_RUNTIME_ALLOW_TOKEN_FALLBACK", True)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not allow_fallback:
|
||||||
|
return self._mtls.authenticate(request)
|
||||||
|
|
||||||
|
# Fast path: no mTLS headers at all but bearer token is present.
|
||||||
|
# This avoids noisy mTLS "missing verify" rejection logs on every poll.
|
||||||
|
if bearer and not _request_has_mtls_context(request):
|
||||||
|
return self._token.authenticate(request, bearer)
|
||||||
|
|
||||||
|
principal = self._mtls.authenticate(request)
|
||||||
|
if principal:
|
||||||
|
return principal
|
||||||
|
if bearer:
|
||||||
|
return self._token.authenticate(request, bearer)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class AgentMTLSAuth:
|
class AgentMTLSAuth:
|
||||||
"""
|
"""
|
||||||
Auth via mTLS client certificate details forwarded by nginx.
|
Auth via mTLS client certificate details forwarded by nginx.
|
||||||
@@ -183,6 +221,26 @@ def _normalize_agent_token(value: object) -> str:
|
|||||||
return token
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_bearer_token_from_request(request: HttpRequest) -> str:
|
||||||
|
authz = str(request.META.get("HTTP_AUTHORIZATION", "")).strip()
|
||||||
|
if not authz:
|
||||||
|
return ""
|
||||||
|
parts = authz.split(None, 1)
|
||||||
|
if len(parts) != 2 or parts[0].lower() != "bearer":
|
||||||
|
return ""
|
||||||
|
return _normalize_agent_token(parts[1])
|
||||||
|
|
||||||
|
|
||||||
|
def _request_has_mtls_context(request: HttpRequest) -> bool:
|
||||||
|
verify = str(request.META.get(AgentMTLSAuth.verify_header, "")).strip().upper()
|
||||||
|
cert_payload = str(request.META.get(AgentMTLSAuth.cert_header, "")).strip()
|
||||||
|
if cert_payload:
|
||||||
|
return True
|
||||||
|
if verify and verify != "NONE":
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def hash_agent_token(value: str) -> str:
|
def hash_agent_token(value: str) -> str:
|
||||||
normalized = _normalize_agent_token(value)
|
normalized = _normalize_agent_token(value)
|
||||||
if not normalized:
|
if not normalized:
|
||||||
|
|||||||
@@ -209,6 +209,10 @@ KEYWARDEN_ACCOUNT_USERNAME_TEMPLATE = os.getenv(
|
|||||||
)
|
)
|
||||||
KEYWARDEN_HEARTBEAT_STALE_SECONDS = int(os.getenv("KEYWARDEN_HEARTBEAT_STALE_SECONDS", "120"))
|
KEYWARDEN_HEARTBEAT_STALE_SECONDS = int(os.getenv("KEYWARDEN_HEARTBEAT_STALE_SECONDS", "120"))
|
||||||
KEYWARDEN_AGENT_API_TOKEN = os.getenv("KEYWARDEN_AGENT_API_TOKEN", "").strip()
|
KEYWARDEN_AGENT_API_TOKEN = os.getenv("KEYWARDEN_AGENT_API_TOKEN", "").strip()
|
||||||
|
KEYWARDEN_AGENT_RUNTIME_ALLOW_TOKEN_FALLBACK = env_bool(
|
||||||
|
"KEYWARDEN_AGENT_RUNTIME_ALLOW_TOKEN_FALLBACK",
|
||||||
|
default=True,
|
||||||
|
)
|
||||||
|
|
||||||
CELERY_BROKER_URL = os.getenv("KEYWARDEN_CELERY_BROKER_URL", REDIS_URL)
|
CELERY_BROKER_URL = os.getenv("KEYWARDEN_CELERY_BROKER_URL", REDIS_URL)
|
||||||
CELERY_RESULT_BACKEND = os.getenv("KEYWARDEN_CELERY_RESULT_BACKEND", REDIS_URL)
|
CELERY_RESULT_BACKEND = os.getenv("KEYWARDEN_CELERY_RESULT_BACKEND", REDIS_URL)
|
||||||
|
|||||||
Reference in new issue
Block a user