boris/keywarden
1
1 Fork 0
Code Issues 17 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

its been a while

Browse Source
This commit is contained in:
boris committed 2026-04-16 17:12:29 +01:00
1 parent ea7470231f
commit 34f13c65e1
47 files changed
+5680 -339
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

agent/internal/client/client.go
+69
View File
@@ -28,6 +28,18 @@ type Client struct {
scheme string
host string
addr string
token string
}
func normalizeAgentToken(raw string) string {
token := strings.TrimSpace(raw)
if token == "" {
return ""
}
if len(token) > 7 && strings.EqualFold(token[:7], "Bearer ") {
token = strings.TrimSpace(token[7:])
}
return token
}
func New(cfg *config.Config) (*Client, error) {
@@ -35,6 +47,10 @@ func New(cfg *config.Config) (*Client, error) {
if baseURL == "" {
return nil, errors.New("server url is required")
}
agentToken := normalizeAgentToken(cfg.AgentAPIToken)
if agentToken == "" {
return nil, errors.New("agent api token is required (set agent_api_token or KEYWARDEN_AGENT_API_TOKEN)")
}
cert, err := tls.LoadX509KeyPair(cfg.ClientCertPath(), cfg.ClientKeyPath())
if err != nil {
return nil, fmt.Errorf("load client cert: %w", err)
@@ -97,9 +113,17 @@ func New(cfg *config.Config) (*Client, error) {
scheme: scheme,
host: host,
addr: addr,
token: agentToken,
}, nil
}
func (c *Client) setAuth(req *http.Request) {
if c.token == "" {
return
}
req.Header.Set("Authorization", "Bearer "+c.token)
}
type EnrollRequest struct {
Token string `json:"token"`
CSRPEM string `json:"csr_pem"`
@@ -113,6 +137,7 @@ type EnrollResponse struct {
ServerID string `json:"server_id"`
ClientCert string `json:"client_cert_pem"`
CACert string `json:"ca_cert_pem"`
AgentToken string `json:"agent_api_token,omitempty"`
SyncProfile string `json:"sync_profile,omitempty"`
DisplayName string `json:"display_name,omitempty"`
}
@@ -149,6 +174,19 @@ type SyncReportRequest struct {
Accounts []AccountSyncEntry `json:"accounts,omitempty"`
}
type LogSourceConfig struct {
SourceID string `json:"source_id"`
Kind string `json:"kind"`
Name string `json:"name"`
ServiceUnit string `json:"service_unit,omitempty"`
FilePath string `json:"file_path,omitempty"`
Parser string `json:"parser,omitempty"`
IncludeMatches map[string][]string `json:"include_matches,omitempty"`
ExcludeMatches map[string][]string `json:"exclude_matches,omitempty"`
Category string `json:"category"`
EventType string `json:"event_type"`
}
func Enroll(ctx context.Context, serverURL string, req EnrollRequest) (*EnrollResponse, error) {
baseURL := strings.TrimRight(serverURL, "/")
if baseURL == "" {
@@ -241,6 +279,7 @@ func (c *Client) FetchAccountAccess(ctx context.Context, serverID string) ([]Acc
if err != nil {
return nil, fmt.Errorf("build account access request: %w", err)
}
c.setAuth(req)
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("fetch account access: %w", err)
@@ -266,6 +305,7 @@ func (c *Client) FetchUserCA(ctx context.Context, serverID string) (*UserCARespo
if err != nil {
return nil, fmt.Errorf("build user ca request: %w", err)
}
c.setAuth(req)
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("fetch user ca: %w", err)
@@ -299,6 +339,7 @@ func (c *Client) SendSyncReport(ctx context.Context, serverID string, report Syn
return fmt.Errorf("build sync report: %w", err)
}
req.Header.Set("Content-Type", "application/json")
c.setAuth(req)
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("send sync report: %w", err)
@@ -310,12 +351,39 @@ func (c *Client) SendSyncReport(ctx context.Context, serverID string, report Syn
return nil
}
func (c *Client) FetchLogConfig(ctx context.Context, serverID string) ([]LogSourceConfig, error) {
req, err := http.NewRequestWithContext(
ctx,
http.MethodGet,
c.baseURL+"/agent/servers/"+serverID+"/log-config",
nil,
)
if err != nil {
return nil, fmt.Errorf("build log config request: %w", err)
}
c.setAuth(req)
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("fetch log config: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
return nil, &HTTPStatusError{StatusCode: resp.StatusCode, Status: resp.Status}
}
var out []LogSourceConfig
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
return nil, fmt.Errorf("decode log config: %w", err)
}
return out, nil
}
func (c *Client) SendLogBatch(ctx context.Context, serverID string, payload []byte) error {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/agent/servers/"+serverID+"/logs", bytes.NewReader(payload))
if err != nil {
return fmt.Errorf("build log request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
c.setAuth(req)
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("send log batch: %w", err)
@@ -344,6 +412,7 @@ func (c *Client) UpdateHost(ctx context.Context, serverID string, reqBody Heartb
return fmt.Errorf("build host update: %w", err)
}
req.Header.Set("Content-Type", "application/json")
c.setAuth(req)
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("send host update: %w", err)
agent/internal/config/config.go
+97 -1
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"strconv"
"strings"
)
@@ -26,12 +27,28 @@ type AccountPolicy struct {
LockOnRevoke bool `json:"lock_on_revoke"`
}
type LogSource struct {
SourceID string `json:"source_id,omitempty"`
Kind string `json:"kind"`
Name string `json:"name,omitempty"`
ServiceUnit string `json:"service_unit,omitempty"`
FilePath string `json:"file_path,omitempty"`
Parser string `json:"parser,omitempty"`
IncludeMatches map[string][]string `json:"include_matches,omitempty"`
ExcludeMatches map[string][]string `json:"exclude_matches,omitempty"`
Category string `json:"category,omitempty"`
EventType string `json:"event_type,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
}
type Config struct {
ServerURL string `json:"server_url"`
ServerID string `json:"server_id,omitempty"`
AgentAPIToken string `json:"agent_api_token,omitempty"`
ServerCAPath string `json:"server_ca_path,omitempty"`
SyncIntervalSeconds int `json:"sync_interval_seconds,omitempty"`
LogBatchSize int `json:"log_batch_size,omitempty"`
LogSources []LogSource `json:"log_sources,omitempty"`
StateDir string `json:"state_dir,omitempty"`
AccountPolicy AccountPolicy `json:"account_policy,omitempty"`
}
@@ -48,7 +65,11 @@ func LoadOrInit(path string, serverURL string) (*Config, error) {
if serverURL == "" {
return nil, errors.New("server url required for first boot")
}
cfg := &Config{ServerURL: serverURL, ServerCAPath: os.Getenv("KEYWARDEN_SERVER_CA_PATH")}
cfg := &Config{
ServerURL: serverURL,
AgentAPIToken: os.Getenv("KEYWARDEN_AGENT_API_TOKEN"),
ServerCAPath: os.Getenv("KEYWARDEN_SERVER_CA_PATH"),
}
applyDefaults(cfg)
if err := validate(cfg, false); err != nil {
return nil, err
@@ -62,9 +83,15 @@ func LoadOrInit(path string, serverURL string) (*Config, error) {
if err := json.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("parse config: %w", err)
}
envAgentToken := strings.TrimSpace(os.Getenv("KEYWARDEN_AGENT_API_TOKEN"))
if cfg.ServerCAPath == "" {
cfg.ServerCAPath = os.Getenv("KEYWARDEN_SERVER_CA_PATH")
}
// Environment should win when set so operators can rotate tokens centrally
// without editing persisted agent.json on every node.
if envAgentToken != "" {
cfg.AgentAPIToken = envAgentToken
}
applyDefaults(cfg)
if err := validate(cfg, false); err != nil {
return nil, err
@@ -108,6 +135,8 @@ func applyDefaults(cfg *Config) {
}
func validate(cfg *Config, requireServerID bool) error {
cfg.ServerID = strings.TrimSpace(cfg.ServerID)
cfg.AgentAPIToken = strings.TrimSpace(cfg.AgentAPIToken)
var missing []string
if cfg.ServerURL == "" {
missing = append(missing, "server_url")
@@ -118,12 +147,57 @@ func validate(cfg *Config, requireServerID bool) error {
if len(missing) > 0 {
return fmt.Errorf("missing required config fields: %v", missing)
}
if looksLikePlaceholder(cfg.ServerID) {
return errors.New("invalid server_id placeholder in config; remove server_id and re-enroll the agent")
}
if cfg.ServerID != "" {
if _, err := strconv.Atoi(cfg.ServerID); err != nil {
return errors.New("invalid server_id in config; expected numeric server id from enrollment")
}
}
if looksLikePlaceholder(cfg.AgentAPIToken) {
return errors.New("invalid agent_api_token placeholder in config; remove agent_api_token and re-enroll the agent")
}
if cfg.SyncIntervalSeconds < 5 {
return errors.New("sync_interval_seconds must be >= 5")
}
for i, source := range cfg.LogSources {
kind := strings.ToLower(strings.TrimSpace(source.Kind))
switch kind {
case "journal":
// journal source is valid with optional include/exclude matches.
case "service":
if strings.TrimSpace(source.ServiceUnit) == "" && strings.TrimSpace(source.Name) == "" {
return fmt.Errorf("log_sources[%d]: service source requires service_unit or name", i)
}
case "file":
if strings.TrimSpace(source.FilePath) == "" && strings.TrimSpace(source.Name) == "" {
return fmt.Errorf("log_sources[%d]: file source requires file_path or name", i)
}
default:
return fmt.Errorf("log_sources[%d]: kind must be 'journal', 'service' or 'file'", i)
}
parser := strings.ToLower(strings.TrimSpace(source.Parser))
if parser == "" {
continue
}
switch parser {
case "none", "syslog", "nginx_access", "json":
default:
return fmt.Errorf("log_sources[%d]: parser must be one of none|syslog|nginx_access|json", i)
}
}
return nil
}
func looksLikePlaceholder(value string) bool {
value = strings.TrimSpace(value)
if value == "" {
return false
}
return strings.Contains(value, "<") || strings.Contains(value, ">")
}
func (c *Config) ClientCertPath() string {
return c.StateDir + "/agent.crt"
}
@@ -144,6 +218,28 @@ func (c *Config) LogSpoolDir() string {
return c.StateDir + "/spool"
}
func (c *Config) LogOffsetsPath() string {
return c.StateDir + "/log.offsets.json"
}
func (c *Config) LogStatePath() string {
return c.StateDir + "/log.state.json"
}
func (c *Config) EnabledLogSources() []LogSource {
if len(c.LogSources) == 0 {
return nil
}
out := make([]LogSource, 0, len(c.LogSources))
for _, source := range c.LogSources {
if source.Enabled != nil && !*source.Enabled {
continue
}
out = append(out, source)
}
return out
}
func dir(path string) string {
if idx := strings.LastIndex(path, string(os.PathSeparator)); idx != -1 {
return path[:idx]
agent/internal/logs/collector.go
+1010 -44
View File
File diff suppressed because it is too large. Load diff
agent/internal/logs/collector_parser_test.go
+60
View File
@@ -0,0 +1,60 @@
package logs
import (
"testing"
"time"
)
func TestApplyNginxAccessParserSetsParsedTimestamp(t *testing.T) {
event := NewEvent(time.Date(2026, time.April, 15, 18, 43, 7, 0, time.UTC))
event.SourceKind = "file"
event.Message = `192.168.2.40 - - [28/Jan/2026:17:43:43 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/8.0"`
if !applyNginxAccessParser(&event) {
t.Fatalf("expected nginx parser to match log line")
}
expected := "2026-01-28T17:43:43Z"
if event.Timestamp != expected {
t.Fatalf("expected parsed timestamp %s, got %s", expected, event.Timestamp)
}
}
func TestParseJSONTimestampSupportsEpochMillis(t *testing.T) {
event := NewEvent(time.Date(2026, time.April, 15, 18, 43, 7, 0, time.UTC))
event.SourceKind = "file"
event.Message = `{"timestamp":1700000000000,"message":"login event"}`
if !applyJSONParser(&event) {
t.Fatalf("expected json parser to parse payload")
}
expected := time.Unix(1700000000, 0).UTC().Format(time.RFC3339Nano)
if event.Timestamp != expected {
t.Fatalf("expected parsed timestamp %s, got %s", expected, event.Timestamp)
}
}
func TestParseSyslogTimestampHandlesYearRollover(t *testing.T) {
now := time.Date(2026, time.January, 1, 0, 0, 30, 0, time.UTC)
parsed, ok := parseSyslogTimestamp("Dec 31 23:59:59", now)
if !ok {
t.Fatalf("expected syslog timestamp to parse")
}
if parsed.Year() != 2025 {
t.Fatalf("expected year rollover to previous year, got %d", parsed.Year())
}
}
func TestMaybeSetParsedTimestampSkipsJournalAndServiceSources(t *testing.T) {
original := "2026-04-15T18:43:07Z"
event := Event{
Timestamp: original,
SourceKind: "service",
}
maybeSetParsedTimestamp(&event, time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC))
if event.Timestamp != original {
t.Fatalf("expected service timestamp to remain unchanged")
}
}
agent/internal/logs/offsets.go
+35
View File
@@ -0,0 +1,35 @@
package logs
import (
"encoding/json"
"os"
)
func ReadOffsets(path string) (map[string]int64, error) {
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return map[string]int64{}, nil
}
return nil, err
}
out := map[string]int64{}
if len(data) == 0 {
return out, nil
}
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return out, nil
}
func WriteOffsets(path string, offsets map[string]int64) error {
if offsets == nil {
offsets = map[string]int64{}
}
data, err := json.Marshal(offsets)
if err != nil {
return err
}
return os.WriteFile(path, data, 0o600)
}
agent/internal/logs/state.go
+35
View File
@@ -0,0 +1,35 @@
package logs
import (
"encoding/json"
"os"
)
func ReadState(path string) (map[string]SourceState, error) {
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return map[string]SourceState{}, nil
}
return nil, err
}
out := map[string]SourceState{}
if len(data) == 0 {
return out, nil
}
if err := json.Unmarshal(data, &out); err != nil {
return nil, err
}
return out, nil
}
func WriteState(path string, state map[string]SourceState) error {
if state == nil {
state = map[string]SourceState{}
}
data, err := json.Marshal(state)
if err != nil {
return err
}
return os.WriteFile(path, data, 0o600)
}
agent/internal/logs/types.go
+34 -13
View File
@@ -3,21 +3,42 @@ package logs
import "time"
type Event struct {
Timestamp string `json:"timestamp"`
Category string `json:"category"`
EventType string `json:"event_type"`
Unit string `json:"unit,omitempty"`
Priority string `json:"priority,omitempty"`
Hostname string `json:"hostname,omitempty"`
Username string `json:"username,omitempty"`
Principal string `json:"principal,omitempty"`
SourceIP string `json:"source_ip,omitempty"`
SessionID string `json:"session_id,omitempty"`
Message string `json:"message,omitempty"`
Raw string `json:"raw,omitempty"`
Fields map[string]string `json:"fields,omitempty"`
Timestamp string `json:"timestamp"`
Category string `json:"category"`
EventType string `json:"event_type"`
SourceKind string `json:"source_kind,omitempty"`
SourceName string `json:"source_name,omitempty"`
Unit string `json:"unit,omitempty"`
Priority string `json:"priority,omitempty"`
Hostname string `json:"hostname,omitempty"`
Username string `json:"username,omitempty"`
Principal string `json:"principal,omitempty"`
SourceIP string `json:"source_ip,omitempty"`
SessionID string `json:"session_id,omitempty"`
Message string `json:"message,omitempty"`
Raw string `json:"raw,omitempty"`
Fields map[string]string `json:"fields,omitempty"`
}
func NewEvent(ts time.Time) Event {
return Event{Timestamp: ts.UTC().Format(time.RFC3339Nano)}
}
type SourceConfig struct {
SourceID string
Kind string
Name string
ServiceUnit string
FilePath string
Parser string
IncludeMatches map[string][]string
ExcludeMatches map[string][]string
Category string
EventType string
}
type SourceState struct {
Cursor string `json:"cursor,omitempty"`
Offset int64 `json:"offset,omitempty"`
Inode uint64 `json:"inode,omitempty"`
}