its been a while

2026-04-16 17:12:29 +01:00
1 parent ea7470231f
commit 34f13c65e1
@@ -4,15 +4,20 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
+	"crypto/sha256"
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"encoding/hex"
 	"encoding/json"
 	"encoding/pem"
+	"errors"
 	"flag"
 	"fmt"
 	"log"
 	"os"
 	"os/signal"
+	"sort"
+	"strings"
 	"syscall"
 	"time"

@@ -23,10 +28,13 @@ import (
 	"keywarden/agent/internal/version"
 )

+var lastNoEventsNotice time.Time
+
 func main() {
 	configPath := flag.String("config", config.DefaultConfigPath, "Path to agent config JSON")
 	serverURL := flag.String("server-url", "", "Keywarden server URL (first boot)")
 	enrollToken := flag.String("enroll-token", "", "Enrollment token (first boot)")
+	forceEnroll := flag.Bool("force-enroll", false, "Force re-enrollment even if already bootstrapped")
 	showVersion := flag.Bool("version", false, "Print version and exit")
 	flag.Parse()

@@ -43,7 +51,7 @@ func main() {
 		log.Fatalf("state dir error: %v", err)
 	}

-	if err := bootstrapIfNeeded(cfg, *configPath, pickEnrollToken(*enrollToken)); err != nil {
+	if err := bootstrapIfNeeded(cfg, *configPath, pickEnrollToken(*enrollToken), *forceEnroll); err != nil {
 		log.Fatalf("bootstrap error: %v", err)
 	}

@@ -80,20 +88,30 @@ func runOnce(ctx context.Context, apiClient *client.Client, cfg *config.Config)
 			log.Printf("host update deferred; will retry: %v", err)
 		} else {
 			log.Printf("host update error: %v", err)
+			logAgentUnauthorizedHint(err)
 		}
 	}
 	if err := apiClient.SyncAccounts(ctx, cfg); err != nil {
 		log.Printf("sync accounts error: %v", err)
+		logAgentUnauthorizedHint(err)
 	}
 	if err := shipLogs(ctx, apiClient, cfg); err != nil {
 		if client.IsRetriable(err) {
 			log.Printf("log shipping deferred; will retry: %v", err)
 		} else {
 			log.Printf("log shipping error: %v", err)
+			logAgentUnauthorizedHint(err)
 		}
 	}
 }

+func logAgentUnauthorizedHint(err error) {
+	var statusErr *client.HTTPStatusError
+	if errors.As(err, &statusErr) && statusErr.StatusCode == 401 {
+		log.Printf("agent auth failed (401): ensure agent_api_token matches the enrollment-issued token for this server (raw token only; no 'Bearer ' prefix)")
+	}
+}
+
 func ensureDirs(cfg *config.Config) error {
 	if err := os.MkdirAll(cfg.StateDir, 0o700); err != nil {
 		return err
@@ -114,16 +132,56 @@ func shipLogs(ctx context.Context, apiClient *client.Client, cfg *config.Config)
 		return err
 	}

-	cursor, err := logs.ReadCursor(cfg.LogCursorPath())
+	state, err := logs.ReadState(cfg.LogStatePath())
+	if err != nil {
+		return err
+	}
+	localSources := toLogSourcesFromConfig(cfg.EnabledLogSources())
+	sourceConfig, err := apiClient.FetchLogConfig(ctx, cfg.ServerID)
+	var sources []logs.SourceConfig
+	if err != nil {
+		if len(localSources) == 0 {
+			return err
+		}
+		log.Printf("log config fetch failed; using local log_sources fallback: %v", err)
+		sources = localSources
+	} else {
+		sources = toLogSources(sourceConfig)
+		if len(sources) > 0 && len(localSources) > 0 {
+			sources = append(sources, localSources...)
+		}
+	}
+	state, err = reconcileLogState(cfg, state, sources)
 	if err != nil {
 		return err
 	}
 	collector := logs.NewCollector()
-	events, nextCursor, err := collector.Collect(ctx, cursor, cfg.LogBatchSize)
+	events, nextState, err := collector.Collect(ctx, cfg.LogBatchSize, sources, state)
 	if err != nil {
-		return err
+		if len(sources) > 0 {
+			return err
+		}
+		fallbackSources := defaultFileFallbackSources()
+		log.Printf("default journald collection failed; falling back to file monitors: %v", err)
+		events, nextState, err = collector.Collect(ctx, cfg.LogBatchSize, fallbackSources, state)
+		if err != nil {
+			return err
+		}
+	}
+	if len(events) == 0 && len(sources) == 0 {
+		fallbackSources := defaultFileFallbackSources()
+		log.Printf("default journald yielded no events; trying file monitors")
+		fileEvents, fileState, fileErr := collector.Collect(ctx, cfg.LogBatchSize, fallbackSources, nextState)
+		if fileErr != nil {
+			return fileErr
+		}
+		events = fileEvents
+		nextState = fileState
 	}
 	if len(events) == 0 {
+		if maybeLogNoEventsNotice(len(sources)) && len(sources) == 0 {
+			log.Printf("log shipping file fallback probe: %s", describeFileSourceAvailability(defaultFileFallbackSources()))
+		}
 		return nil
 	}
 	payload, err := json.Marshal(events)
@@ -136,7 +194,7 @@ func shipLogs(ctx context.Context, apiClient *client.Client, cfg *config.Config)
 		}
 		return err
 	}
-	if err := logs.WriteCursor(cfg.LogCursorPath(), nextCursor); err != nil {
+	if err := logs.WriteState(cfg.LogStatePath(), nextState); err != nil {
 		return err
 	}
 	return nil
@@ -172,13 +230,22 @@ func pickEnrollToken(flagValue string) string {
 	return os.Getenv("KEYWARDEN_ENROLL_TOKEN")
 }

-func bootstrapIfNeeded(cfg *config.Config, configPath string, enrollToken string) error {
-	if cfg.ServerID != "" && fileExists(cfg.ClientCertPath()) && fileExists(cfg.CACertPath()) {
+func bootstrapIfNeeded(cfg *config.Config, configPath string, enrollToken string, forceEnroll bool) error {
+	previousServerID := strings.TrimSpace(cfg.ServerID)
+	if !forceEnroll && cfg.ServerID != "" && fileExists(cfg.ClientCertPath()) && fileExists(cfg.CACertPath()) {
+		if strings.TrimSpace(enrollToken) != "" {
+			log.Printf("enrollment token provided but bootstrap is already complete; skipping enrollment (use -force-enroll to rotate identity)")
+		}
 		return nil
 	}
 	if enrollToken == "" {
 		return fmt.Errorf("missing enrollment token; set KEYWARDEN_ENROLL_TOKEN or -enroll-token")
 	}
+	if forceEnroll {
+		log.Printf("force re-enrollment requested; replacing existing server identity")
+		cfg.ServerID = ""
+		cfg.AgentAPIToken = ""
+	}
 	keyPath := cfg.ClientKeyPath()
 	if !fileExists(keyPath) {
 		if err := generateKey(keyPath); err != nil {
@@ -208,12 +275,156 @@ func bootstrapIfNeeded(cfg *config.Config, configPath string, enrollToken string
 		return err
 	}
 	cfg.ServerID = resp.ServerID
+	if token := strings.TrimSpace(resp.AgentToken); token != "" {
+		cfg.AgentAPIToken = token
+	}
+	if forceEnroll || (previousServerID != "" && previousServerID != cfg.ServerID) {
+		if err := resetLogState(cfg); err != nil {
+			return fmt.Errorf("reset log state: %w", err)
+		}
+	}
 	if err := config.Save(configPath, cfg); err != nil {
 		return err
 	}
 	return nil
 }

+func resetLogState(cfg *config.Config) error {
+	if err := os.Remove(cfg.LogCursorPath()); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	if err := os.Remove(cfg.LogOffsetsPath()); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	if err := os.Remove(cfg.LogStatePath()); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	if err := os.Remove(logSourceStatePath(cfg)); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	if err := os.RemoveAll(cfg.LogSpoolDir()); err != nil {
+		return err
+	}
+	return os.MkdirAll(cfg.LogSpoolDir(), 0o700)
+}
+
+func defaultFileFallbackSources() []logs.SourceConfig {
+	return []logs.SourceConfig{
+		{Kind: "file", Name: "/var/log/auth.log", FilePath: "/var/log/auth.log", Category: "auth", EventType: "file.line", Parser: "syslog"},
+		{Kind: "file", Name: "/var/log/secure", FilePath: "/var/log/secure", Category: "auth", EventType: "file.line", Parser: "syslog"},
+		{Kind: "file", Name: "/var/log/syslog", FilePath: "/var/log/syslog", Category: "system", EventType: "file.line", Parser: "syslog"},
+		{Kind: "file", Name: "/var/log/messages", FilePath: "/var/log/messages", Category: "system", EventType: "file.line", Parser: "syslog"},
+	}
+}
+
+func maybeLogNoEventsNotice(sourceCount int) bool {
+	now := time.Now()
+	if !lastNoEventsNotice.IsZero() && now.Sub(lastNoEventsNotice) < time.Minute {
+		return false
+	}
+	mode := "default journald (current boot)"
+	if sourceCount > 0 {
+		mode = "configured sources"
+	}
+	log.Printf("log shipping: no events collected (%s)", mode)
+	lastNoEventsNotice = now
+	return true
+}
+
+func describeFileSourceAvailability(sources []logs.SourceConfig) string {
+	items := make([]string, 0, len(sources))
+	for _, source := range sources {
+		path := strings.TrimSpace(source.FilePath)
+		if path == "" {
+			path = strings.TrimSpace(source.Name)
+		}
+		if path == "" {
+			continue
+		}
+		item := path + ": "
+		info, err := os.Stat(path)
+		if err != nil {
+			if os.IsNotExist(err) {
+				items = append(items, item+"missing")
+				continue
+			}
+			items = append(items, item+err.Error())
+			continue
+		}
+		if info.IsDir() {
+			items = append(items, item+"is directory")
+			continue
+		}
+		file, openErr := os.Open(path)
+		if openErr != nil {
+			items = append(items, item+openErr.Error())
+			continue
+		}
+		_ = file.Close()
+		items = append(items, fmt.Sprintf("%d bytes readable", info.Size()))
+	}
+	if len(items) == 0 {
+		return "no file sources"
+	}
+	return strings.Join(items, "; ")
+}
+
+func reconcileLogState(
+	cfg *config.Config,
+	state map[string]logs.SourceState,
+	sources []logs.SourceConfig,
+) (map[string]logs.SourceState, error) {
+	statePath := logSourceStatePath(cfg)
+	previous, err := os.ReadFile(statePath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return state, err
+	}
+	previousSig := strings.TrimSpace(string(previous))
+	currentSig := logSourceSignature(sources)
+	needsReset := false
+	if previousSig == "" {
+		if len(state) > 0 {
+			log.Printf("log state signature missing; resetting per-source state for fresh collection")
+			needsReset = true
+		}
+	} else if previousSig != currentSig {
+		log.Printf("log sources changed; resetting per-source state for fresh collection")
+		needsReset = true
+	}
+	if err := os.WriteFile(statePath, []byte(currentSig+"\n"), 0o600); err != nil {
+		return state, err
+	}
+	if needsReset {
+		return map[string]logs.SourceState{}, nil
+	}
+	return state, nil
+}
+
+func logSourceStatePath(cfg *config.Config) string {
+	return cfg.StateDir + "/log.sources.sha256"
+}
+
+func logSourceSignature(sources []logs.SourceConfig) string {
+	parts := make([]string, 0, len(sources))
+	for _, source := range sources {
+		parts = append(parts, strings.Join([]string{
+			strings.TrimSpace(source.SourceID),
+			strings.ToLower(strings.TrimSpace(source.Kind)),
+			strings.TrimSpace(source.Name),
+			strings.TrimSpace(source.ServiceUnit),
+			strings.TrimSpace(source.FilePath),
+			strings.ToLower(strings.TrimSpace(source.Parser)),
+			strings.TrimSpace(source.Category),
+			strings.TrimSpace(source.EventType),
+			encodeMatchMap(source.IncludeMatches),
+			encodeMatchMap(source.ExcludeMatches),
+		}, "\x1f"))
+	}
+	sort.Strings(parts)
+	sum := sha256.Sum256([]byte(strings.Join(parts, "\x1e")))
+	return hex.EncodeToString(sum[:])
+}
+
 func retry(ctx context.Context, delays []time.Duration, fn func() error) error {
 	var lastErr error
 	for attempt := 0; attempt <= len(delays); attempt++ {
@@ -276,3 +487,59 @@ func fileExists(path string) bool {
 	}
 	return !info.IsDir()
 }
+
+func toLogSources(in []client.LogSourceConfig) []logs.SourceConfig {
+	out := make([]logs.SourceConfig, 0, len(in))
+	for _, source := range in {
+		out = append(out, logs.SourceConfig{
+			SourceID:       source.SourceID,
+			Kind:           source.Kind,
+			Name:           source.Name,
+			ServiceUnit:    source.ServiceUnit,
+			FilePath:       source.FilePath,
+			Parser:         source.Parser,
+			IncludeMatches: source.IncludeMatches,
+			ExcludeMatches: source.ExcludeMatches,
+			Category:       source.Category,
+			EventType:      source.EventType,
+		})
+	}
+	return out
+}
+
+func toLogSourcesFromConfig(in []config.LogSource) []logs.SourceConfig {
+	out := make([]logs.SourceConfig, 0, len(in))
+	for _, source := range in {
+		out = append(out, logs.SourceConfig{
+			SourceID:       source.SourceID,
+			Kind:           source.Kind,
+			Name:           source.Name,
+			ServiceUnit:    source.ServiceUnit,
+			FilePath:       source.FilePath,
+			Parser:         source.Parser,
+			IncludeMatches: source.IncludeMatches,
+			ExcludeMatches: source.ExcludeMatches,
+			Category:       source.Category,
+			EventType:      source.EventType,
+		})
+	}
+	return out
+}
+
+func encodeMatchMap(in map[string][]string) string {
+	if len(in) == 0 {
+		return ""
+	}
+	keys := make([]string, 0, len(in))
+	for key := range in {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	parts := make([]string, 0, len(keys))
+	for _, key := range keys {
+		values := append([]string{}, in[key]...)
+		sort.Strings(values)
+		parts = append(parts, key+"="+strings.Join(values, ","))
+	}
+	return strings.Join(parts, ";")
+}