From 57c0d49169c3d3cdc6da22b536fb19a4e1433a16 Mon Sep 17 00:00:00 2001 From: Eugene Blikh Date: Sun, 3 May 2026 20:42:24 +0300 Subject: [PATCH] collector: persist skipped-only parser progress --- docs/tasks/lethe-collector-claude-code.md | 1 + internal/collector/ingest/runner.go | 5 +++ internal/collector/ingest/runner_test.go | 49 +++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/docs/tasks/lethe-collector-claude-code.md b/docs/tasks/lethe-collector-claude-code.md index b67e330f9bce02710b36f00a5003dccc5aec5114..b48ffaee1515c8d989c9c244d7918ccb079b149e 100644 --- a/docs/tasks/lethe-collector-claude-code.md +++ b/docs/tasks/lethe-collector-claude-code.md @@ -322,6 +322,7 @@ Smoke: `go run ./cmd/lethe-collector --config ./tmp/collector-smoke.yaml status` - ureview (final): backfill offset-0 semantics are implemented as `RunBackfillOnce` instead of a mode flag on `RunOnce` — explicit call sites are safer than a boolean parameter that could be misused in daemon loops. - ureview (final): enforced the outbox size cap before replay and normalized trailing slashes in `server_url` — keeps IV5 and IV9 true for preexisting state and valid-looking URLs. - ureview (final): normalized sender `serverURL` and enforced outbox cap before every replay to fix IV5/IV9 violations found in review. +- ureview (final): skipped-only parse results (no events but `newOffset > startOffset`) now persist the new offset so the file is not re-parsed forever and status lag clears. ### Deferred (needs user input) diff --git a/internal/collector/ingest/runner.go b/internal/collector/ingest/runner.go index 3e3e17ea8345b8d50fdd02dda47298000be4e3a1..b2820ebbe05cbf638b16f47166d6166b3a2f77ff 100644 --- a/internal/collector/ingest/runner.go +++ b/internal/collector/ingest/runner.go @@ -186,6 +186,11 @@ func runFileFromOffset(ctx context.Context, cfg config.Config, src config.Source return culpa.Wrap(err, "parse source file") } if len(events) == 0 { + if newOffset > startOffset { + if err := store.SaveOffset(ctx, src.Tool, path, newOffset); err != nil { + return culpa.Wrap(err, "save skipped-only offset") + } + } return nil } stampHost(events, cfg.Host) diff --git a/internal/collector/ingest/runner_test.go b/internal/collector/ingest/runner_test.go index d04fddcb3f21a812d15395ba886d746a31cc4eea..07424741916557fd326c1ca87f0d11b4ba06600e 100644 --- a/internal/collector/ingest/runner_test.go +++ b/internal/collector/ingest/runner_test.go @@ -487,6 +487,55 @@ func TestRunBackfillOnce_InterruptedProgressIsResumable(t *testing.T) { assertOffset(t, ctx, store, "claude-code", file, 300) } +func TestRunOnce_SkippedOnlyParseResultPersistsNewOffsetAndDoesNotPost(t *testing.T) { + ctx := context.Background() + store := openTestStore(t, ctx) + source := testSource(t, "claude-code", 10, 4096) + file := filepath.Join(source.Path, "one.jsonl") + + var posted int + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + posted++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(resultJSON(Result{Accepted: 0}))) + })) + defer ts.Close() + sender := NewSender(ts.URL, ts.Client()) + + p := newFakeParser("claude-code", []parser.SourceFile{{Path: file}}, map[string]parseResult{ + file: {events: []wire.TurnEvent{}, newOffset: 250}, + }) + + err := RunOnce(ctx, testConfig(), source, p, store, sender) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + + assertOffset(t, ctx, store, "claude-code", file, 250) + if posted != 0 { + t.Errorf("POST count = %d, want 0", posted) + } +} + +func TestRunOnce_EmptyParseResultNoProgressDoesNotSaveOffset(t *testing.T) { + ctx := context.Background() + store := openTestStore(t, ctx) + source := testSource(t, "claude-code", 10, 4096) + file := filepath.Join(source.Path, "one.jsonl") + + p := newFakeParser("claude-code", []parser.SourceFile{{Path: file}}, map[string]parseResult{ + file: {events: []wire.TurnEvent{}, newOffset: 0}, + }) + sender := acceptingSender(t, nil) + + err := RunOnce(ctx, testConfig(), source, p, store, sender) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + + assertOffset(t, ctx, store, "claude-code", file, 0) +} + type parseResult struct { events []wire.TurnEvent newOffset int64