From 9716ef4fe0a30bf7518de7dfc2e97f4f12f8a45a Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 14:46:29 +0200 Subject: [PATCH 01/49] Take the account event feed into a durable ledger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The intake half of `basecamp connect`: the feed, the SQLite ledger behind it, the position, and the recovery that makes a crash a delay rather than a loss. Admission and dispatch are the next two cards; nothing here reads a recording or judges one. Intake is the only work on the feed's delivery path — write the pointer, hand over the id — so a slow admission or a busy dispatcher can never stall the socket. Three things in this feed look like "nothing more to read" and are not, and each has a test that fails without the handling: an empty page (a request crosses up to a thousand rows and serves at most a hundred matches), a missing `next` (the walk's frozen head, not the end of history), and a quiet inbox (delivery has write-time brakes that say nothing). The feed's two 410s mean different things and do not share a recovery path. The generated client models both with one error discriminated by a nil epoch, while the connector seam's field is a plain int64, so the obvious conversion presents a closed retention window as "the feed's epoch is event 0". The adapter refuses that conversion instead. An overflow is written to disk before it is accepted, because accepting is owning the incompleteness and a loss that only ever lived in memory is one no restart could find. The repair walk runs on its own cursor, kept on the loss record: it is seeded from a live id far ahead of the poll lane, and a checkpoint taken from one would skip everything the safety delay had not yet served. --- internal/connector/doc.go | 47 ++ internal/connector/feed_adapter.go | 342 ++++++++++ internal/connector/feed_adapter_test.go | 256 +++++++ internal/connector/intake.go | 684 +++++++++++++++++++ internal/connector/intake_feed_test.go | 366 ++++++++++ internal/connector/intake_test.go | 375 ++++++++++ internal/connector/ledger.go | 205 ++++++ internal/connector/ledger_checkpoint.go | 105 +++ internal/connector/ledger_events.go | 253 +++++++ internal/connector/ledger_recovery.go | 372 ++++++++++ internal/connector/ledger_test.go | 193 ++++++ internal/connector/lock.go | 97 +++ internal/connector/lock_test.go | 64 ++ internal/connector/queue.go | 137 ++++ internal/connector/queue_test.go | 105 +++ internal/connector/repair.go | 195 ++++++ internal/connector/repair_test.go | 268 ++++++++ internal/connector/shutdown.go | 40 ++ internal/connector/shutdown_test.go | 17 + internal/mcpserver/model/behavior-model.json | 12 + 20 files changed, 4133 insertions(+) create mode 100644 internal/connector/doc.go create mode 100644 internal/connector/feed_adapter.go create mode 100644 internal/connector/feed_adapter_test.go create mode 100644 internal/connector/intake.go create mode 100644 internal/connector/intake_feed_test.go create mode 100644 internal/connector/intake_test.go create mode 100644 internal/connector/ledger.go create mode 100644 internal/connector/ledger_checkpoint.go create mode 100644 internal/connector/ledger_events.go create mode 100644 internal/connector/ledger_recovery.go create mode 100644 internal/connector/ledger_test.go create mode 100644 internal/connector/lock.go create mode 100644 internal/connector/lock_test.go create mode 100644 internal/connector/queue.go create mode 100644 internal/connector/queue_test.go create mode 100644 internal/connector/repair.go create mode 100644 internal/connector/repair_test.go create mode 100644 internal/connector/shutdown.go create mode 100644 internal/connector/shutdown_test.go diff --git a/internal/connector/doc.go b/internal/connector/doc.go new file mode 100644 index 000000000..356fd95f2 --- /dev/null +++ b/internal/connector/doc.go @@ -0,0 +1,47 @@ +// Package connector is the intake half of `basecamp connect`: the account +// event feed, the durable ledger behind it, and the recovery that makes a +// crash a delay rather than a loss. +// +// # What intake is, and what it deliberately is not +// +// Feed rows are pointers — id, type, bucket, creator, recording — and nothing +// else. No title, no body, no URL, no names. Intake writes that pointer to the +// ledger if the id is new and hands the id to a queue. That is the whole of +// the work on the feed's delivery path, so a slow admission or a busy +// dispatcher can never stall the socket. +// +// Deciding whether an event deserves an agent's attention is admission's job +// and it costs a read per event it cares about. Intake does none of it. It +// also does no filtering of its own: the feed carries no `column_id` or +// `todolist_id`, so narrowing to a column or a list means re-fetching the +// parent listing, and that price belongs where the read already happens. +// +// # The two lanes +// +// The live lane is a WebSocket; the poll lane is an HTTP walk that runs about +// thirty seconds behind it, deliberately, so a page never stops mid-commit. +// The same event therefore arrives twice in the ordinary case, and dedupe is +// by event id across both lanes and across restarts — which is what makes the +// ledger, not a set in memory, the dedupe authority. +// +// Only the poll lane advances the durable position. A live event id is far +// ahead of the poll lane, and a checkpoint taken from one would skip +// everything the safety delay had not yet served. +// +// # Quiet is not caught up +// +// Three separate things in this feed look like "nothing more to read" and are +// not: +// +// - An empty page. A request crosses up to a thousand ledger rows and serves +// at most a hundred matches; rows the filters exclude still advance the +// cursor. Follow `next` until it is absent, never stop at the first empty +// page. +// - A missing `next`. The walk reached its frozen head, which is not the end +// of history: a page cut short by the safety horizon withholds the link on +// purpose. Poll again later. +// - A silent inbox. Delivery has write-time brakes — a per-account rate +// ceiling and an agent-to-agent chain breaker — and a braked event writes +// no addressing and tells the client nothing. A quiet feed is not proof of +// a quiet project, so nothing here ever reports "up to date". +package connector diff --git a/internal/connector/feed_adapter.go b/internal/connector/feed_adapter.go new file mode 100644 index 000000000..efd183c3a --- /dev/null +++ b/internal/connector/feed_adapter.go @@ -0,0 +1,342 @@ +package connector + +import ( + "context" + "errors" + "fmt" + "net/url" + "strconv" + "strings" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// This file is the Layer-1 adapter the eventfeed package leaves open: the +// TicketMinter and PollSource seams, backed by the SDK's generated +// EventFeedService. It is the package's own documented path — "a host that +// wants the live feed supplies its own over the generated operations" — not a +// stand-in for one. +// +// SWAP POINT. basecamp-sdk PR 899 lands these adapters in the SDK, and PR 897 +// an eventfeed.NewLive that wires them. When both are in, NewFeedAdapter's two +// seam methods are replaced by that constructor and everything below the +// translation line goes; the translation of the SDK's error shapes into this +// package's two 410 types stays, because it is what keeps the two recoveries +// apart, and nothing upstream owns that. + +// FeedEpochGoneError is the ACCOUNT feed's 410: the held position fell below +// the feed's epoch. EpochAfterID names where servable history begins, and the +// served resume URL re-enters above that fence. +// +// Recovery: accept, follow the resume exactly as served, record the gap with +// its epoch, and classify the entry by the cursor the server chose. +type FeedEpochGoneError struct { + EpochAfterID int64 + Resume string + Err error +} + +func (e *FeedEpochGoneError) Error() string { + return fmt.Sprintf("event feed position is below the feed epoch %d: %v", e.EpochAfterID, e.Err) +} +func (e *FeedEpochGoneError) Unwrap() error { return e.Err } + +// InboxRetentionGoneError is the INBOX lane's 410: the held position fell out +// of the 30-day retention window. There is no epoch — not a zero one — and the +// resume re-enters at since=0, the earliest retained item. +// +// Recovery: it is a different loss with a different shape, and the account +// lane must never produce one. Intake surfaces it rather than resuming, so +// that a contract change announces itself instead of being absorbed as "the +// epoch is zero". +type InboxRetentionGoneError struct { + Resume string + Err error +} + +func (e *InboxRetentionGoneError) Error() string { + return fmt.Sprintf("inbox position is outside the retention window: %v", e.Err) +} +func (e *InboxRetentionGoneError) Unwrap() error { return e.Err } + +// UndifferentiatedRequestError is a feed 400 the server gave no reason for. +// +// The two reasons need opposite recoveries — an invalid position re-enters +// with since=, an invalid filter is terminal — so a 400 +// that names neither is surfaced rather than guessed. Guessing the position +// turns a bad filter into an endless re-entry loop; guessing the filter kills +// a feed a re-entry would have fixed. +type UndifferentiatedRequestError struct { + Err error +} + +func (e *UndifferentiatedRequestError) Error() string { + return fmt.Sprintf("event feed refused the request without naming a reason: %v", e.Err) +} +func (e *UndifferentiatedRequestError) Unwrap() error { return e.Err } + +// FeedClient is the slice of the SDK's EventFeedService this adapter needs. +// An interface so the adapter's error translation can be tested without a +// network, which is the only part of it that carries judgement. +type FeedClient interface { + PollEvents(ctx context.Context, opts *basecamp.PollEventsOptions) (*basecamp.EventFeedPage, error) + CreateStreamTicket(ctx context.Context) (*basecamp.StreamTicket, error) +} + +// FeedAdapter backs the TicketMinter and PollSource seams. +type FeedAdapter struct { + client FeedClient + origin *url.URL +} + +var ( + _ eventfeed.TicketMinter = (*FeedAdapter)(nil) + _ eventfeed.PollSource = (*FeedAdapter)(nil) +) + +// NewFeedAdapter builds the adapter. origin is the API base the continuation +// and resume URLs are validated against. +func NewFeedAdapter(client FeedClient, origin string) (*FeedAdapter, error) { + if client == nil { + return nil, errors.New("connector: feed adapter needs a client") + } + canonical, err := eventfeed.CanonicalOrigin(origin) + if err != nil { + return nil, fmt.Errorf("connector: feed adapter origin: %w", err) + } + parsed, err := url.Parse(canonical) + if err != nil { + return nil, fmt.Errorf("connector: feed adapter origin: %w", err) + } + return &FeedAdapter{client: client, origin: parsed}, nil +} + +// MintStreamTicket mints one ticket. Neither the ticket nor the URL it rides +// in is ever rendered into an error here: the URL's query string carries the +// bearer. +func (a *FeedAdapter) MintStreamTicket(ctx context.Context) (eventfeed.StreamTicket, error) { + ticket, err := a.client.CreateStreamTicket(ctx) + if err != nil { + return eventfeed.StreamTicket{}, mintError(err) + } + return eventfeed.StreamTicket{ + Ticket: ticket.Ticket, + ExpiresIn: ticket.ExpiresIn, + URL: ticket.URL, + }, nil +} + +// Poll fetches one page at cursor under filters. +func (a *FeedAdapter) Poll(ctx context.Context, cursor eventfeed.Cursor, filters eventfeed.Filters) (eventfeed.PollPage, error) { + opts, err := a.optionsFor(cursor, filters) + if err != nil { + return eventfeed.PollPage{}, err + } + + page, err := a.client.PollEvents(ctx, opts) + if err != nil { + return eventfeed.PollPage{}, pollError(err) + } + + events := make([]eventfeed.Event, 0, len(page.Events)) + for _, e := range page.Events { + events = append(events, eventfeed.Event{ + ID: e.ID, + Kind: e.Kind, + EventType: e.EventType, + Action: e.Action, + CreatedAt: e.CreatedAt, + BucketID: e.BucketID, + CreatorID: e.CreatorID, + PerformedByID: e.PerformedByID, + RecordingID: e.RecordingID, + Details: e.Details, + }) + } + // An empty Events with a Next is ordinary, not an end: a request crosses + // up to a thousand ledger rows and serves at most a hundred matches, and + // the rows the filters excluded still moved the cursor. The run loop + // follows Next; nothing here shortcuts on len(events) == 0. + return eventfeed.PollPage{Events: events, Position: page.Position, Next: page.Next}, nil +} + +// optionsFor turns one cursor into the generated operation's options. Exactly +// one of the cursor's three fields is set; the zero cursor is the bare present +// entry. +func (a *FeedAdapter) optionsFor(cursor eventfeed.Cursor, filters eventfeed.Filters) (*basecamp.PollEventsOptions, error) { + if cursor.PageURL != "" { + // A continuation or a 410 resume. It is server-supplied, so it is + // validated against the configured origin before it is used for + // anything: the SPEC's zero-egress-to-a-foreign-target obligation + // rides on this adapter, not on the package. + if err := a.checkContinuation(cursor.PageURL); err != nil { + return nil, err + } + opts, err := basecamp.PollEventsOptionsFromURL(cursor.PageURL) + if err != nil { + return nil, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: err} + } + // The URL carries the server's own canonical filter set. It is used + // as served: re-imposing the local filters on a resume is how a + // resume stops being the server's. + return opts, nil + } + + opts := &basecamp.PollEventsOptions{ + Since: cursor.Since, + Position: cursor.Position, + Types: filters.Types, + Buckets: filters.Buckets, + Creators: filters.Creators, + Performers: idStrings(filters.Performers), + ExcludePerformers: idStrings(filters.ExcludePerformers), + ActorTypes: filters.ActorTypes, + } + return opts, nil +} + +// checkContinuation enforces same-origin and no-downgrade on a server-supplied +// URL before it is followed. +func (a *FeedAdapter) checkContinuation(raw string) error { + parsed, err := url.Parse(raw) + if err != nil { + return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: fmt.Errorf("connector: unparseable continuation URL")} + } + if !parsed.IsAbs() { + return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: errors.New("connector: continuation URL is not absolute")} + } + if !strings.EqualFold(parsed.Scheme, a.origin.Scheme) || !strings.EqualFold(parsed.Host, a.origin.Host) { + // LocationOrigin is DATA and is deliberately not rendered: a hostile + // target can reflect the bearer into a host label. + return &eventfeed.PollError{ + Kind: eventfeed.PollRedirectRefused, + LocationOrigin: parsed.Scheme + "://" + parsed.Host, + Err: errors.New("connector: continuation URL leaves the configured API origin"), + } + } + return nil +} + +func idStrings(ids []int64) []string { + if len(ids) == 0 { + return nil + } + out := make([]string, 0, len(ids)) + for _, id := range ids { + out = append(out, strconv.FormatInt(id, 10)) + } + return out +} + +// --------------------------------------------------------------------------- +// The translation line. Everything below maps the SDK's error shapes onto the +// seam's taxonomy, and it is the one place the two 410s are told apart. +// --------------------------------------------------------------------------- + +// pollError classifies a failed PollEvents call. +func pollError(err error) error { + if err == nil { + return nil + } + + // The inbox's 410 first, so it can never fall through to the feed's arm. + // + // On sdk main today both lanes answer one *basecamp.FeedPositionGoneError + // with EpochAfterID *int64, nil on the inbox. basecamp-sdk PR 912 splits + // them into two types with the epoch required on the feed's. Either way + // the discrimination happens here and once: a nil epoch flattened into + // eventfeed.PollError's plain int64 EpochAfterID would present a + // retention loss as "the feed's epoch is 0" and send it down the epoch's + // recovery path, which is the failure this arm exists to prevent. + if gone := asFeedGone(err); gone != nil { + if gone.EpochAfterID == nil { + return &eventfeed.PollError{ + Kind: eventfeed.PollUnrecoverable, + Err: &InboxRetentionGoneError{Resume: gone.Resume, Err: err}, + } + } + return &eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: *gone.EpochAfterID, + ResumeURL: gone.Resume, + Err: &FeedEpochGoneError{EpochAfterID: *gone.EpochAfterID, Resume: gone.Resume, Err: err}, + } + } + + var mismatch *basecamp.FeedFilterMismatchError + if errors.As(err, &mismatch) { + return &eventfeed.PollError{ + Kind: eventfeed.PollFilterChanged, + PositionDigest: mismatch.PositionDigest, + FiltersDigest: mismatch.FiltersDigest, + Err: err, + } + } + + var apiErr *basecamp.Error + if !errors.As(err, &apiErr) { + return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: err} + } + + switch { + case apiErr.HTTPStatus == 400: + // SWAP POINT for basecamp-sdk PR 912's *FeedRequestError: when it + // carries reason=invalid_position this becomes PollPositionInvalid, + // and reason=invalid_filter becomes PollFilterInvalid. Until the + // server names the reason, a 400 is undifferentiated and is surfaced + // rather than guessed — see UndifferentiatedRequestError. + return &eventfeed.PollError{ + Kind: eventfeed.PollUnrecoverable, + Msg: apiErr.Message, + Err: &UndifferentiatedRequestError{Err: err}, + } + case apiErr.HTTPStatus == 401 || apiErr.HTTPStatus == 403: + return &eventfeed.PollError{Kind: eventfeed.PollUnauthorized, Err: err} + case apiErr.RetryAfter > 0: + return &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: retryAfter(apiErr), Err: err} + case apiErr.Retryable: + return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: err} + } + return &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Msg: apiErr.Message, Err: err} +} + +// mintError classifies a failed CreateStreamTicket call. +func mintError(err error) error { + if err == nil { + return nil + } + var apiErr *basecamp.Error + if !errors.As(err, &apiErr) { + return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: err} + } + switch { + case apiErr.HTTPStatus == 401 || apiErr.HTTPStatus == 403: + return &eventfeed.MintError{Kind: eventfeed.MintUnauthorized, Err: err} + case apiErr.RetryAfter > 0: + return &eventfeed.MintError{Kind: eventfeed.MintThrottled, RetryAfter: retryAfter(apiErr), Err: err} + case apiErr.Retryable: + return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: err} + } + return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: err} +} + +func retryAfter(apiErr *basecamp.Error) time.Duration { + return time.Duration(apiErr.RetryAfter) * time.Second +} + +// asFeedGone reads the SDK's feed 410 into a shape this package owns, so the +// rest of the file does not move when the SDK's does. +func asFeedGone(err error) *feedGone { + var gone *basecamp.FeedPositionGoneError + if !errors.As(err, &gone) { + return nil + } + return &feedGone{EpochAfterID: gone.EpochAfterID, Resume: gone.Resume} +} + +type feedGone struct { + EpochAfterID *int64 + Resume string +} diff --git a/internal/connector/feed_adapter_test.go b/internal/connector/feed_adapter_test.go new file mode 100644 index 000000000..03f8dae91 --- /dev/null +++ b/internal/connector/feed_adapter_test.go @@ -0,0 +1,256 @@ +package connector + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +type fakeFeedClient struct { + page *basecamp.EventFeedPage + ticket *basecamp.StreamTicket + err error + lastOpt *basecamp.PollEventsOptions + calls int +} + +func (f *fakeFeedClient) PollEvents(_ context.Context, opts *basecamp.PollEventsOptions) (*basecamp.EventFeedPage, error) { + f.calls++ + f.lastOpt = opts + if f.err != nil { + return nil, f.err + } + return f.page, nil +} + +func (f *fakeFeedClient) CreateStreamTicket(context.Context) (*basecamp.StreamTicket, error) { + f.calls++ + if f.err != nil { + return nil, f.err + } + return f.ticket, nil +} + +func newTestAdapter(t *testing.T, client FeedClient) *FeedAdapter { + t.Helper() + adapter, err := NewFeedAdapter(client, "https://3.basecampapi.com") + require.NoError(t, err) + return adapter +} + +// The card's hardest constraint: the feed's two 410s mean different things and +// must not share one recovery path. PR 898 models both with one +// FeedPositionGoneError discriminated by a nil EpochAfterID, while the +// connector seam's PollError.EpochAfterID is a plain int64 — so the naive +// conversion silently turns "the inbox's retention window closed" into "the +// feed's epoch is event 0". +func TestFeedEpoch410BecomesTheGapSignal(t *testing.T) { + epoch := int64(17099838487) + client := &fakeFeedClient{err: &basecamp.FeedPositionGoneError{ + Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 410, Message: "position below epoch"}, + EpochAfterID: &epoch, + Resume: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + }} + + _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollGone, pollErr.Kind) + assert.Equal(t, epoch, pollErr.EpochAfterID) + assert.Equal(t, "https://3.basecampapi.com/2914079/events.json?since=17099838487", pollErr.ResumeURL) + + var epochGone *FeedEpochGoneError + assert.ErrorAs(t, err, &epochGone) +} + +func TestRetention410NeverBecomesTheGapSignal(t *testing.T) { + client := &fakeFeedClient{err: &basecamp.FeedPositionGoneError{ + Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 410, Message: "position outside the retention window"}, + EpochAfterID: nil, // the inbox lane's shape: no epoch, not a zero one + Resume: "https://3.basecampapi.com/2914079/my/inbox.json?since=0", + }} + + _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.NotEqual(t, eventfeed.PollGone, pollErr.Kind, + "a retention 410 dispatched as FeedGap resumes the feed down the epoch's path") + assert.Zero(t, pollErr.EpochAfterID, + "no epoch may be invented for a 410 that carried none") + + var retention *InboxRetentionGoneError + require.ErrorAs(t, err, &retention) + assert.Equal(t, "https://3.basecampapi.com/2914079/my/inbox.json?since=0", retention.Resume) + + var epochGone *FeedEpochGoneError + assert.False(t, errors.As(err, &epochGone), + "the two 410s must not both satisfy the epoch arm") +} + +func TestFilterMismatchCarriesBothDigests(t *testing.T) { + client := &fakeFeedClient{err: &basecamp.FeedFilterMismatchError{ + Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 409, Message: "filters changed"}, + PositionDigest: "9f2ab04e5c11d3a7", + FiltersDigest: "0011223344556677", + }} + + _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollFilterChanged, pollErr.Kind) + assert.Equal(t, "9f2ab04e5c11d3a7", pollErr.PositionDigest) + assert.Equal(t, "0011223344556677", pollErr.FiltersDigest) +} + +// The two reasons behind a feed 400 need opposite recoveries, so a 400 that +// names neither is surfaced rather than guessed. +func TestUnreasonedBadRequestIsSurfacedNotGuessed(t *testing.T) { + client := &fakeFeedClient{err: &basecamp.Error{ + Code: basecamp.CodeValidation, HTTPStatus: 400, Message: "bad request", + }} + + _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.NotEqual(t, eventfeed.PollPositionInvalid, pollErr.Kind, + "guessing the position turns a bad filter into an endless re-entry loop") + assert.NotEqual(t, eventfeed.PollFilterInvalid, pollErr.Kind, + "guessing the filter kills a feed a re-entry would have fixed") + + var undifferentiated *UndifferentiatedRequestError + assert.ErrorAs(t, err, &undifferentiated) +} + +func TestThrottleAndTransientAreClassifiedApart(t *testing.T) { + throttled := &fakeFeedClient{err: &basecamp.Error{ + Code: basecamp.CodeRateLimit, HTTPStatus: 429, Retryable: true, RetryAfter: 7, + }} + _, err := newTestAdapter(t, throttled).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollThrottled, pollErr.Kind) + assert.Equal(t, 7*time.Second, pollErr.RetryAfter) + + transient := &fakeFeedClient{err: &basecamp.Error{ + Code: basecamp.CodeAPI, HTTPStatus: 503, Retryable: true, + }} + _, err = newTestAdapter(t, transient).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollTransient, pollErr.Kind) +} + +func TestForeignContinuationIsRefusedBeforeAnyRequest(t *testing.T) { + client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} + + _, err := newTestAdapter(t, client).Poll(context.Background(), + eventfeed.Cursor{PageURL: "https://evil.example.com/2914079/events.json?position=x"}, + eventfeed.Filters{}) + + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) + assert.Equal(t, "https://evil.example.com", pollErr.LocationOrigin) + assert.NotContains(t, pollErr.Error(), "evil.example.com", + "a hostile target can reflect the bearer into a host label, so the origin is data and never a rendering") + assert.Zero(t, client.calls, "zero egress to a foreign target") +} + +func TestSchemeDowngradeIsRefused(t *testing.T) { + client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} + + _, err := newTestAdapter(t, client).Poll(context.Background(), + eventfeed.Cursor{PageURL: "http://3.basecampapi.com/2914079/events.json?position=x"}, + eventfeed.Filters{}) + + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) + assert.Zero(t, client.calls) +} + +func TestEmptyPageWithANextIsNotAnEnd(t *testing.T) { + client := &fakeFeedClient{page: &basecamp.EventFeedPage{ + Events: nil, + Position: "opaque-position-2", + Next: "https://3.basecampapi.com/2914079/events.json?position=opaque-position-2", + }} + + page, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + require.NoError(t, err) + assert.Empty(t, page.Events) + assert.NotEmpty(t, page.Next, "a page that served no match still advanced the cursor") + assert.Equal(t, "opaque-position-2", page.Position) +} + +func TestPollPassesTheFilterSetAndCursor(t *testing.T) { + client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} + adapter := newTestAdapter(t, client) + + _, err := adapter.Poll(context.Background(), + eventfeed.Cursor{Since: "17099838487"}, + eventfeed.Filters{ + Types: []string{"comment.created"}, + Buckets: []int64{48699913}, + ExcludePerformers: []int64{52007412}, + ActorTypes: []string{"person"}, + }) + require.NoError(t, err) + + require.NotNil(t, client.lastOpt) + assert.Equal(t, "17099838487", client.lastOpt.Since) + assert.Equal(t, []string{"comment.created"}, client.lastOpt.Types) + assert.Equal(t, []int64{48699913}, client.lastOpt.Buckets) + assert.Equal(t, []string{"52007412"}, client.lastOpt.ExcludePerformers, + "the loop guard is the agent's own resolved id") + assert.Equal(t, []string{"person"}, client.lastOpt.ActorTypes) +} + +func TestResumeURLIsUsedAsServed(t *testing.T) { + client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} + adapter := newTestAdapter(t, client) + + _, err := adapter.Poll(context.Background(), + eventfeed.Cursor{PageURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487&types=comment.created"}, + eventfeed.Filters{Types: []string{"card.created"}}) + require.NoError(t, err) + + require.NotNil(t, client.lastOpt) + assert.Equal(t, "17099838487", client.lastOpt.Since) + assert.Equal(t, []string{"comment.created"}, client.lastOpt.Types, + "re-imposing the local filters on a resume is how a resume stops being the server's") +} + +func TestMintClassifiesUnauthorized(t *testing.T) { + client := &fakeFeedClient{err: &basecamp.Error{Code: basecamp.CodeAuth, HTTPStatus: 401}} + + _, err := newTestAdapter(t, client).MintStreamTicket(context.Background()) + + var mintErr *eventfeed.MintError + require.ErrorAs(t, err, &mintErr) + assert.Equal(t, eventfeed.MintUnauthorized, mintErr.Kind) +} + +func TestMintNeverRendersTheTicket(t *testing.T) { + client := &fakeFeedClient{ticket: &basecamp.StreamTicket{ + Ticket: "s3cr3t-bearer", + ExpiresIn: 120, + URL: "wss://cable.basecamp.com/cable?ticket=s3cr3t-bearer", + }} + + ticket, err := newTestAdapter(t, client).MintStreamTicket(context.Background()) + require.NoError(t, err) + assert.Equal(t, "s3cr3t-bearer", ticket.Ticket) + assert.Equal(t, "wss://cable.basecamp.com/cable?ticket=s3cr3t-bearer", ticket.URL, + "the URL is connected to verbatim; the connector never assembles cable topology") +} diff --git a/internal/connector/intake.go b/internal/connector/intake.go new file mode 100644 index 000000000..30f0c707f --- /dev/null +++ b/internal/connector/intake.go @@ -0,0 +1,684 @@ +package connector + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/url" + "strconv" + "sync" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// Defaults for the recovery timers. +const ( + // DefaultRepairInterval is how often a repair walk re-runs while a loss is + // open. It matches the feed's own repair poll, and it is deliberately + // longer than the poll lane's ~30s safety delay: a walk that outran the + // delay would call a still-committing event missing. + DefaultRepairInterval = 60 * time.Second + // DefaultRepairWindow is how long a loss stays open before the ids still + // missing are called unrecovered. Ten minutes is twenty repair polls past + // the safety delay. + DefaultRepairWindow = 10 * time.Minute + // DefaultMembershipInterval is how often the agent's project list is + // re-read. The cable snapshots the agent's buckets when it subscribes, so + // a project granted afterwards is invisible on the live lane until the + // connection is remade. + DefaultMembershipInterval = 10 * time.Minute +) + +// MembershipSource lists the buckets the agent can currently see. +type MembershipSource interface { + Buckets(ctx context.Context) ([]int64, error) +} + +// Options configures intake. +type Options struct { + // Origin is the API base URL. It is part of the checkpoint identity and + // the origin every continuation URL is validated against. + Origin string + // AccountID is the Basecamp account. + AccountID string + // ConsumerNamespace names this connector's checkpoint lineage. Two + // connectors in one account must not share one. + ConsumerNamespace string + // Filters is the feed's filter set. The checkpoint is keyed by its digest, + // so changing it re-enters under a new lineage. + Filters eventfeed.Filters + // SinceEventID, when positive, enters just after that event id whatever + // the ledger holds — the `--since` override. + SinceEventID int64 + + Ledger *Ledger + Queue *Queue + Minter eventfeed.TicketMinter + Polls eventfeed.PollSource + + // Pointers receives one NDJSON line per newly seen event. Writes are + // serialized: an interleaved write tears a line and breaks the watcher + // reading it. + Pointers io.Writer + // Logger receives everything else. Pointer lines are the protocol; + // logging is not. + Logger *slog.Logger + + // Membership, when set, drives the reconnect that makes a newly granted + // project visible on the live lane. + Membership MembershipSource + MembershipInterval time.Duration + + RepairInterval time.Duration + RepairWindow time.Duration + + Clock func() time.Time + Transport eventfeed.CableTransport +} + +// Intake is the feed's delivery path: write the pointer, hand over the id. +// +// Everything else — reading the recording, judging it, dispatching it — is +// downstream of the queue, so a slow admission or a busy dispatcher can never +// stall the socket. +type Intake struct { + opts Options + ledger *Ledger + queue *Queue + log *slog.Logger + now func() time.Time + pointer *pointerWriter + + key eventfeed.CheckpointKey + + mu sync.Mutex + // pollCandidates holds the ids delivered since the last page boundary that + // carry no push-lane transport fields. They become the last poll-served id + // only when a page boundary confirms a poll page actually landed. + pollCandidates []int64 + snapshot map[int64]bool + reconnect chan struct{} + + repairs sync.WaitGroup + // lifetime is Run's context. Repair walks are bound to it rather than to + // a connection, so a reconnect does not abandon a walk and a shutdown does + // not strand Run waiting on one — an unfinished walk simply resumes on the + // next start, which is what OpenLosses is for. + lifetime context.Context + // repairSleep replaces the wait between repair polls. Tests set it so ten + // minutes of repair cadence does not take ten minutes. + repairSleep func(ctx context.Context, d time.Duration) error +} + +// New builds intake. +func New(opts Options) (*Intake, error) { + switch { + case opts.Ledger == nil: + return nil, errors.New("connector: intake needs a ledger") + case opts.Queue == nil: + return nil, errors.New("connector: intake needs a queue") + case opts.Minter == nil || opts.Polls == nil: + return nil, errors.New("connector: intake needs the feed's two seams") + case opts.AccountID == "": + return nil, errors.New("connector: intake needs an account id") + case opts.ConsumerNamespace == "": + return nil, errors.New("connector: intake needs a consumer namespace") + } + + origin, err := eventfeed.CanonicalOrigin(opts.Origin) + if err != nil { + return nil, fmt.Errorf("connector: intake origin: %w", err) + } + if err := opts.Filters.Validate(); err != nil { + return nil, fmt.Errorf("connector: intake filters: %w", err) + } + + if opts.Clock == nil { + opts.Clock = time.Now + } + if opts.Logger == nil { + opts.Logger = slog.New(slog.DiscardHandler) + } + if opts.RepairInterval <= 0 { + opts.RepairInterval = DefaultRepairInterval + } + if opts.RepairWindow <= 0 { + opts.RepairWindow = DefaultRepairWindow + } + if opts.MembershipInterval <= 0 { + opts.MembershipInterval = DefaultMembershipInterval + } + + in := &Intake{ + opts: opts, + ledger: opts.Ledger, + queue: opts.Queue, + log: opts.Logger, + now: opts.Clock, + pointer: newPointerWriter(opts.Pointers), + reconnect: make(chan struct{}, 1), + key: eventfeed.CheckpointKey{ + Origin: origin, + AccountID: opts.AccountID, + ConsumerNamespace: opts.ConsumerNamespace, + FilterKey: opts.Filters.FilterKey(), + }, + } + in.ledger.now = opts.Clock + return in, nil +} + +// CheckpointKey is the identity this intake's position is stored under. +func (in *Intake) CheckpointKey() eventfeed.CheckpointKey { return in.key } + +// Run consumes the feed until ctx is canceled or the feed terminates. +// +// It reconnects on its own only for membership: everything else the feed can +// recover from, it recovers from inside the package. +func (in *Intake) Run(ctx context.Context) error { + in.lifetime = ctx + if err := in.resumeReconciliation(ctx); err != nil { + return err + } + defer in.repairs.Wait() + + for { + err := in.runOnce(ctx) + if ctx.Err() != nil { + return ctx.Err() + } + if !errors.Is(err, errReconnect) { + return err + } + in.log.Info("reconnecting the feed", "reason", "membership changed") + } +} + +// errReconnect asks the supervisor for a fresh connection. It is not a +// failure: the cable's bucket snapshot is taken at subscribe, so a project +// granted afterwards needs a new connection to be heard on the live lane. +var errReconnect = errors.New("connector: reconnect the feed") + +func (in *Intake) runOnce(ctx context.Context) error { + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + + in.takeSnapshot(runCtx) + + position, hadPosition, err := in.ledger.Load(runCtx, in.key) + if err != nil { + return err + } + _ = position + + start := eventfeed.StartResume() + switch { + case in.opts.SinceEventID > 0: + start = eventfeed.StartAfter(in.opts.SinceEventID) + in.log.Info("entering the feed after an explicit event id", "since", in.opts.SinceEventID) + case hadPosition: + in.log.Info("resuming the feed from the stored position", "filter_key", in.key.FilterKey) + default: + // Said out loud because it is a real loss of history, not a neutral + // default: everything committed before this moment is never served. + in.log.Warn("no stored position: entering the feed at the present, so nothing committed before now will be served", + "filter_key", in.key.FilterKey) + } + + options := []eventfeed.Option{ + eventfeed.WithFilters(in.opts.Filters), + eventfeed.WithStart(start), + eventfeed.WithCheckpointStore(in.ledger), + eventfeed.WithConsumerNamespace(in.opts.ConsumerNamespace), + eventfeed.WithSignalHandler(in.handleSignal), + eventfeed.WithObserver(in.observer(runCtx)), + eventfeed.WithRepairInterval(in.opts.RepairInterval), + } + if in.opts.Transport != nil { + options = append(options, eventfeed.WithTransport(in.opts.Transport)) + } + + feed, err := eventfeed.New(in.key.Origin, in.opts.AccountID, in.opts.Minter, in.opts.Polls, options...) + if err != nil { + return fmt.Errorf("connector: build feed: %w", err) + } + defer func() { + _ = feed.Close() + feed.Wait() + }() + + stopMembership := in.watchMembership(runCtx, cancel) + defer stopMembership() + + var feedErr error + for event, err := range feed.Events(runCtx) { + if err != nil { + feedErr = err + break + } + if err := in.ingest(runCtx, event, LaneOf(event)); err != nil { + return err + } + } + + if in.reconnectRequested() { + return errReconnect + } + return in.classifyTerminal(ctx, feedErr) +} + +// ingest is the whole of intake: one pointer written, one id handed over. +func (in *Intake) ingest(ctx context.Context, event eventfeed.Event, lane Lane) error { + fresh, err := in.ledger.RecordSeen(ctx, event, lane) + if err != nil { + // A pointer that could not be written is a pointer the restart will + // not know about. Nothing downstream is allowed to proceed as if it + // had been. + return err + } + if lane == LanePoll { + in.notePollCandidate(event.ID) + } + if !fresh { + // The ordinary case: the poll lane serving what the live lane already + // delivered, or a restart re-walking a page. Dedupe is the point. + return nil + } + + in.noteBucket(event.BucketID) + if err := in.pointer.write(event); err != nil { + return err + } + return in.queue.Offer(ctx, event.ID) +} + +// LaneOf says which lane served an event. +// +// The rows are not labeled, but the two shapes differ: actor_type and +// visible_to_clients are push-lane transport fields that poll rows omit, and +// the SDK keeps both presence-bearing — a string whose empty value is outside +// the vocabulary, and a *bool — rather than defaulting them, precisely so this +// is answerable. +// +// It is used for one thing that matters, the last poll-served id, and it is +// paired there with a page boundary: absence of the push fields nominates an +// id, a delivered poll page confirms it. Erring towards "live" only costs +// duplicates the ledger absorbs; erring towards "poll" would move the re-entry +// past events the poll lane had not served. +func LaneOf(event eventfeed.Event) Lane { + if event.ActorType == "" && event.VisibleToClients == nil { + return LanePoll + } + return LaneLive +} + +func (in *Intake) notePollCandidate(id int64) { + in.mu.Lock() + defer in.mu.Unlock() + in.pollCandidates = append(in.pollCandidates, id) +} + +// confirmPollServed promotes the candidates a delivered poll page confirms. +func (in *Intake) confirmPollServed(ctx context.Context) { + in.mu.Lock() + candidates := in.pollCandidates + in.pollCandidates = nil + in.mu.Unlock() + + var highest int64 + for _, id := range candidates { + if id > highest { + highest = id + } + } + if highest == 0 { + // An empty page. Ordinary — the walk crossed rows the filters exclude + // — and it serves no id, so it advances nothing here. + return + } + if err := in.ledger.NotePollServed(ctx, in.key, highest); err != nil { + in.log.Error("could not record the last poll-served id", "error", err) + } +} + +func (in *Intake) observer(ctx context.Context) eventfeed.Observer { + return eventfeed.Observer{ + Connected: func() { in.log.Info("feed socket connected") }, + Confirmed: func() { in.log.Info("feed subscription confirmed") }, + Disconnected: func(reason string, err error) { + in.log.Warn("feed socket disconnected", "reason", reason, "error", err) + }, + CatchUpStarted: func(eventfeed.Cursor) { in.log.Info("feed catch-up walk started") }, + PageDelivered: func(int, string) { + // Detached deliberately: the position the page just moved must be + // recorded even if the run's context is on its way down, or a + // shutdown mid-page loses the id the next re-entry needs. + in.confirmPollServed(context.WithoutCancel(ctx)) //nolint:contextcheck // detached on purpose, see above + }, + CaughtUp: func() { + // "Caught up with the walk", not "caught up with the account": + // delivery has write-time brakes that write no addressing and say + // nothing, so a quiet feed is never proof of a quiet project. + in.log.Info("feed walk reached its head and the buffer drained") + }, + CheckpointSaveFailed: func(err error) { + in.log.Error("could not save the feed position", "error", err) + }, + Gap: func(epochAfterID int64, resumeOrigin string) { + in.log.Warn("feed served a 410", "epoch_after_id", epochAfterID, "resume_origin", resumeOrigin) + }, + PositionRejected: func(kind eventfeed.PollErrorKind) { + in.log.Warn("feed rejected the held position", "kind", kind.String()) + }, + FilterConflict: func(positionDigest, filtersDigest string) { + in.log.Warn("feed position was minted for a different filter set", + "position_digest", positionDigest, "filters_digest", filtersDigest) + }, + StaleConnection: func(d time.Duration) { + in.log.Warn("feed socket went stale", "since_last_frame", d) + }, + BufferOverflow: func(dropped int) { + in.log.Warn("live buffer overflowed", "dropped", dropped) + }, + } +} + +// classifyTerminal turns the feed's terminal error into the connector's own +// verdict. The one case that needs saying is the inbox's 410 arriving on the +// account lane: it is a different loss with a different resume, and it is +// surfaced rather than absorbed. +func (in *Intake) classifyTerminal(ctx context.Context, err error) error { + if err == nil { + return nil + } + var retention *InboxRetentionGoneError + if errors.As(err, &retention) { + if _, recordErr := in.ledger.RecordGap(ctx, Gap{ + DetectedAt: in.now(), + Class: GapRetention, + EntryClass: EntryUnknown, + Note: "a retention 410 was served on the account lane, which has an epoch instead; not resumed", + }); recordErr != nil { + in.log.Error("could not record the retention gap", "error", recordErr) + } + in.log.Error("the account feed answered the inbox lane's 410: its resume re-enters at the earliest retained item, not above an epoch, so it is not followed here") + } + return err +} + +// handleSignal decides what a semantic signal means for this connector. It +// runs synchronously on the delivery path, so it does only what must happen +// before the disposition takes effect and starts the rest elsewhere. +func (in *Intake) handleSignal(signal eventfeed.Signal) eventfeed.Disposition { + ctx := context.WithoutCancel(context.Background()) + + switch s := signal.(type) { + case eventfeed.FeedGap: + // The ACCOUNT lane's 410, and the only one that reaches here: the + // adapter never maps a retention 410 onto this signal. The resume URL + // is followed exactly as served — the entry class is the server's + // decision, read out of its cursor, never substituted. + epoch := s.EpochAfterID + if _, err := in.ledger.RecordGap(ctx, Gap{ + DetectedAt: in.now(), + Class: GapEpoch, + EpochAfterID: &epoch, + EntryClass: entryClassOf(s.ResumeURL), + Note: "the feed's served history before the epoch is gone", + }); err != nil { + // A gap we cannot write down is a gap nothing will ever report. + in.log.Error("could not record the feed gap; refusing to continue past it", "error", err) + return eventfeed.Terminate + } + return eventfeed.Accept + + case eventfeed.BufferOverflow: + loss, err := in.ledger.RecordLoss(ctx, s.DroppedIDs, in.now(), in.opts.RepairWindow) + if err != nil { + // Accept means owning the incompleteness. Owning it begins with + // it being on disk: accepting after a failed write would leave a + // loss that no restart could ever find, which is the one outcome + // worse than terminating. + in.log.Error("could not record the buffer overflow; refusing to accept it", "error", err) + return eventfeed.Terminate + } + in.log.Warn("live buffer overflowed; reconciling", + "dropped", s.DroppedCount, "loss_id", loss.ID, "repair_since", loss.RepairSince) + in.startRepair(in.repairContext(), loss) + return eventfeed.Accept + } + return eventfeed.Terminate +} + +// entryClassOf reads the entry class out of the cursor the server served. +func entryClassOf(resumeURL string) EntryClass { + parsed, err := url.Parse(resumeURL) + if err != nil { + return EntryUnknown + } + switch since := parsed.Query().Get("since"); since { + case "now": + return EntryPresent + case "": + return EntryUnknown + default: + if _, err := strconv.ParseInt(since, 10, 64); err != nil { + return EntryUnknown + } + return EntryReplay + } +} + +// resumeReconciliation restarts every open loss's repair walk on start. A +// crash between the overflow and its repair is a delay, not a loss of the +// record. +func (in *Intake) resumeReconciliation(ctx context.Context) error { + losses, err := in.ledger.OpenLosses(ctx) + if err != nil { + return err + } + for _, loss := range losses { + in.log.Info("resuming reconciliation of an open loss", "loss_id", loss.ID, "repair_since", loss.RepairSince) + in.startRepair(ctx, loss) + } + return nil +} + +// repairContext is the lifetime a repair walk runs under. handleSignal is +// invoked by the feed with no context of its own, so the walk takes Run's. +func (in *Intake) repairContext() context.Context { + if in.lifetime != nil { + return in.lifetime + } + return context.Background() +} + +func (in *Intake) startRepair(ctx context.Context, loss Loss) { + if loss.ResolvedAt != nil { + return + } + in.repairs.Add(1) + go func() { + defer in.repairs.Done() + // Off the delivery path: nothing about live intake waits for this. + walker := &repairWalker{ + ledger: in.ledger, + polls: in.opts.Polls, + filters: in.opts.Filters, + ingest: in.ingest, + now: in.now, + interval: in.opts.RepairInterval, + log: in.log, + sleep: in.repairSleep, + } + switch err := walker.reconcile(ctx, loss); { + case err == nil: + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + // A shutdown mid-walk is a delay: the loss is still open on disk + // and the next start picks it up where this one left off. + in.log.Info("reconciliation paused by shutdown; it resumes on the next start", "loss_id", loss.ID) + default: + in.log.Error("reconciliation of a loss ended early", "loss_id", loss.ID, "error", err) + } + }() +} + +// takeSnapshot records the buckets the agent can see at the moment this +// connection subscribes — the same set the cable snapshots. +func (in *Intake) takeSnapshot(ctx context.Context) { + if in.opts.Membership == nil { + return + } + buckets, err := in.opts.Membership.Buckets(ctx) + if err != nil { + in.log.Warn("could not read the agent's projects; keeping the previous snapshot", "error", err) + return + } + in.mu.Lock() + defer in.mu.Unlock() + in.snapshot = make(map[int64]bool, len(buckets)) + for _, id := range buckets { + in.snapshot[id] = true + } +} + +// noteBucket asks for a reconnect when an event arrives from a bucket the live +// snapshot did not hold. The poll lane authorizes at read time, so it covers +// the new project immediately; the live lane cannot until it re-subscribes. +func (in *Intake) noteBucket(bucketID int64) { + in.mu.Lock() + known := in.snapshot == nil || in.snapshot[bucketID] + in.mu.Unlock() + if known { + return + } + in.log.Info("an event arrived from a project the live subscription does not hold", "bucket_id", bucketID) + in.requestReconnect() +} + +// watchMembership re-reads the agent's projects on a timer and asks for a +// reconnect when the set changes. It returns a stop function. +func (in *Intake) watchMembership(ctx context.Context, cancel context.CancelFunc) func() { + if in.opts.Membership == nil { + return func() {} + } + done := make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(in.opts.MembershipInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + buckets, err := in.opts.Membership.Buckets(ctx) + if err != nil { + // A failed read leaves the snapshot alone. Treating it as + // a change would reconnect the feed every time the API + // hiccupped. + in.log.Warn("could not refresh the agent's projects", "error", err) + continue + } + if in.membershipChanged(buckets) { + in.requestReconnect() + cancel() + return + } + } + } + }() + return func() { <-done } +} + +func (in *Intake) membershipChanged(buckets []int64) bool { + in.mu.Lock() + defer in.mu.Unlock() + if in.snapshot == nil { + return false + } + if len(buckets) != len(in.snapshot) { + return true + } + for _, id := range buckets { + if !in.snapshot[id] { + return true + } + } + return false +} + +func (in *Intake) requestReconnect() { + select { + case in.reconnect <- struct{}{}: + default: + } +} + +func (in *Intake) reconnectRequested() bool { + select { + case <-in.reconnect: + return true + default: + return false + } +} + +// pointerWriter serializes the NDJSON pointer lines. One line is one write: +// two goroutines interleaving inside a line tear it, and the reader on the +// other end has no way to recover a torn line. +type pointerWriter struct { + mu sync.Mutex + w io.Writer +} + +func newPointerWriter(w io.Writer) *pointerWriter { return &pointerWriter{w: w} } + +// Pointer is the line intake writes to stdout for each newly seen event. It +// carries what the feed carried and nothing more: no title, no body, no URL, +// no names. Whoever wants those pays for a read. +type Pointer struct { + EventID int64 `json:"event_id"` + EventType string `json:"event_type"` + Kind string `json:"kind"` + Action string `json:"action"` + BucketID int64 `json:"bucket_id"` + CreatorID int64 `json:"creator_id"` + PerformedByID *int64 `json:"performed_by_id"` + RecordingID int64 `json:"recording_id"` + CreatedAt string `json:"created_at"` + Lane Lane `json:"lane"` + State string `json:"state"` +} + +func (p *pointerWriter) write(event eventfeed.Event) error { + if p.w == nil { + return nil + } + line, err := json.Marshal(Pointer{ + EventID: event.ID, + EventType: event.EventType, + Kind: event.Kind, + Action: event.Action, + BucketID: event.BucketID, + CreatorID: event.CreatorID, + PerformedByID: event.PerformedByID, + RecordingID: event.RecordingID, + CreatedAt: event.CreatedAt.UTC().Format(time.RFC3339), + Lane: LaneOf(event), + State: string(StateSeen), + }) + if err != nil { + return fmt.Errorf("connector: encode pointer line: %w", err) + } + p.mu.Lock() + defer p.mu.Unlock() + if _, err := p.w.Write(append(line, '\n')); err != nil { + return fmt.Errorf("connector: write pointer line: %w", err) + } + return nil +} diff --git a/internal/connector/intake_feed_test.go b/internal/connector/intake_feed_test.go new file mode 100644 index 000000000..8b4ef6287 --- /dev/null +++ b/internal/connector/intake_feed_test.go @@ -0,0 +1,366 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +// End-to-end through the real eventfeed run loop, on the package's own +// deterministic fakes: subscribe, catch up, drain, stream — and then a restart +// over the same ledger. + +type safeBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *safeBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *safeBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func pushFrame(t *testing.T, identifier string, event eventfeed.Event) []byte { + t.Helper() + payload := map[string]any{ + "id": event.ID, + "kind": event.Kind, + "event_type": event.EventType, + "action": event.Action, + "created_at": event.CreatedAt.UTC().Format(time.RFC3339), + "bucket_id": event.BucketID, + "creator_id": event.CreatorID, + "performed_by_id": nil, + "actor_type": "person", + "recording_id": event.RecordingID, + "visible_to_clients": true, + } + raw, err := json.Marshal(payload) + require.NoError(t, err) + frame, err := json.Marshal(map[string]json.RawMessage{ + "identifier": mustJSON(t, identifier), + "message": raw, + }) + require.NoError(t, err) + return frame +} + +func confirmFrame(t *testing.T, identifier string) []byte { + t.Helper() + frame, err := json.Marshal(map[string]any{ + "type": "confirm_subscription", + "identifier": identifier, + }) + require.NoError(t, err) + return frame +} + +func mustJSON(t *testing.T, v any) json.RawMessage { + t.Helper() + raw, err := json.Marshal(v) + require.NoError(t, err) + return raw +} + +// subscribedConn waits for the connector's subscribe command and answers it, +// returning the connection and the identifier the connector chose. +func subscribedConn(t *testing.T, transport *feedtest.Transport) (*feedtest.Conn, string) { + t.Helper() + var ( + conn *feedtest.Conn + identifier string + ) + require.Eventually(t, func() bool { + conn = transport.LastConn() + return conn != nil + }, 5*time.Second, 5*time.Millisecond, "the connector should dial the cable URL the mint served") + + // Action Cable greets first; the subscribe follows the welcome. + conn.Serve([]byte(`{"type":"welcome"}`)) + + require.Eventually(t, func() bool { + writes := conn.Writes() + if len(writes) == 0 { + return false + } + var command struct { + Command string `json:"command"` + Identifier string `json:"identifier"` + } + if err := json.Unmarshal(writes[0], &command); err != nil { + return false + } + identifier = command.Identifier + return command.Command == "subscribe" && identifier != "" + }, 5*time.Second, 5*time.Millisecond, "the connector should subscribe before it takes a position") + + conn.Serve(confirmFrame(t, identifier)) + return conn, identifier +} + +func TestIntakeRunsTheFeedThroughCatchUpAndStreaming(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{testEvent(17099838500)}, + Position: "caught-up-position", + }) + + var pointers safeBuffer + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + Pointers: &pointers, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + + conn, identifier := subscribedConn(t, transport) + + // The catch-up page lands first; then a live event on the socket. + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838500) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond, "the catch-up walk's events reach the ledger") + + conn.Serve(pushFrame(t, identifier, testEvent(17099838600))) + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond, "a live event reaches the ledger") + + // Only the poll lane advances the durable position. + position, ok, err := ledger.Load(ctx, intake.CheckpointKey()) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "caught-up-position", position) + + live, _, err := ledger.Get(ctx, 17099838600) + require.NoError(t, err) + assert.Equal(t, LaneLive, live.Lane) + polled, _, err := ledger.Get(ctx, 17099838500) + require.NoError(t, err) + assert.Equal(t, LanePoll, polled.Lane) + + assert.Equal(t, 2, countLines(pointers.String())) + assert.Equal(t, 2, queue.Depth()) + + cancel() + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("Run should return when its context is canceled") + } +} + +// Kill and restart: no duplicate, and the walk resumes from the stored +// position rather than the present. +func TestIntakeSurvivesARestartWithoutDuplicating(t *testing.T) { + dir := t.TempDir() + path := dir + "/connector.db" + + firstLedger, err := OpenLedger(path) + require.NoError(t, err) + _, err = firstLedger.RecordSeen(context.Background(), testEvent(17099838500), LanePoll) + require.NoError(t, err) + require.NoError(t, firstLedger.Save(context.Background(), eventfeed.CheckpointKey{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + FilterKey: eventfeed.Filters{}.FilterKey(), + }, "position-from-the-previous-run")) + require.NoError(t, firstLedger.Close()) + + ledger, err := OpenLedger(path) + require.NoError(t, err) + t.Cleanup(func() { _ = ledger.Close() }) + + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + // The same event the previous run already saw, plus one it did not. + polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{testEvent(17099838500), testEvent(17099838501)}, + Position: "position-after-the-restart", + }) + + var pointers safeBuffer + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + Pointers: &pointers, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + + subscribedConn(t, transport) + + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838501) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond) + + calls := polls.Calls() + require.NotEmpty(t, calls) + assert.Equal(t, "position-from-the-previous-run", calls[0].Cursor.Position, + "a restart resumes from the stored position, never at the head") + + assert.Equal(t, 1, countLines(pointers.String()), + "the event the previous run already saw is not a second unit of work") + assert.Equal(t, 1, queue.Depth()) + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run should return when its context is canceled") + } +} + +func TestIntakeRecordsTheLastPollServedIDFromTheWalk(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + polls.ScriptPage(eventfeed.PollPage{ + Events: []eventfeed.Event{testEvent(17099838500), testEvent(17099838501)}, + Position: "p1", + }) + + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = intake.Run(ctx) }() + + conn, identifier := subscribedConn(t, transport) + + require.Eventually(t, func() bool { + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + return err == nil && served == 17099838501 + }, 5*time.Second, 5*time.Millisecond, "the poll lane's highest served id is durable") + + // A live event far ahead of the poll lane must not move it. + conn.Serve(pushFrame(t, identifier, testEvent(17099999999))) + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099999999) + return err == nil && ok + }, 5*time.Second, 5*time.Millisecond) + + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Equal(t, int64(17099838501), served, + fmt.Sprintf("a live id is not a poll-served id; re-entering at %d would skip the safety delay", 17099999999)) +} + +// A repair walk must not strand the process on the way out. It is bound to +// Run's lifetime, and an unfinished walk is a delay: the loss is still open on +// disk and the next start resumes it. +func TestShutdownDoesNotWaitOutAnOpenRepairWalk(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + _, err = ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Hour) + require.NoError(t, err) + + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + minter.ScriptTicket(eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"}) + polls := feedtest.NewPolls() + + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: minter, + Polls: polls, + Transport: transport, + // A repair cadence far longer than the test: the walk is asleep when + // the shutdown arrives, which is the case that used to hang. + RepairInterval: time.Hour, + }) + require.NoError(t, err) + + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + + subscribedConn(t, transport) + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Run should return on shutdown rather than wait out the repair cadence") + } + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + assert.Len(t, open, 1, "the loss stays on disk for the next start to resume") +} diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go new file mode 100644 index 000000000..5f8d9a376 --- /dev/null +++ b/internal/connector/intake_test.go @@ -0,0 +1,375 @@ +package connector + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "io" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +type stubMinter struct{} + +func (stubMinter) MintStreamTicket(context.Context) (eventfeed.StreamTicket, error) { + return eventfeed.StreamTicket{}, nil +} + +type scriptedPolls struct { + pages []eventfeed.PollPage + errs []error + cursors []eventfeed.Cursor + calls int +} + +func (s *scriptedPolls) Poll(_ context.Context, cursor eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { + s.cursors = append(s.cursors, cursor) + i := s.calls + s.calls++ + if i < len(s.errs) && s.errs[i] != nil { + return eventfeed.PollPage{}, s.errs[i] + } + if i < len(s.pages) { + return s.pages[i], nil + } + return eventfeed.PollPage{Position: "drained"}, nil +} + +type fixedClock struct{ at time.Time } + +func (c *fixedClock) now() time.Time { return c.at } + +func newTestIntake(t *testing.T, polls eventfeed.PollSource, pointers io.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() + ledger := newTestLedger(t) + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + if polls == nil { + polls = &scriptedPolls{} + } + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: polls, + Pointers: pointers, + Clock: (&fixedClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)}).now, + }) + require.NoError(t, err) + return intake, ledger, queue +} + +func TestLaneOfReadsThePushLaneTransportFields(t *testing.T) { + poll := testEvent(1) + assert.Equal(t, LanePoll, LaneOf(poll), "poll rows omit actor_type and visible_to_clients") + + visible := true + push := testEvent(2) + push.ActorType = "person" + push.VisibleToClients = &visible + assert.Equal(t, LaneLive, LaneOf(push)) + + // Presence, not value: visible_to_clients false is still present. + hidden := false + pushHidden := testEvent(3) + pushHidden.ActorType = "agent" + pushHidden.VisibleToClients = &hidden + assert.Equal(t, LaneLive, LaneOf(pushHidden)) +} + +func TestIngestWritesOnePointerAndQueuesOnce(t *testing.T) { + var pointers bytes.Buffer + intake, _, queue := newTestIntake(t, nil, &pointers) + ctx := context.Background() + + require.NoError(t, intake.ingest(ctx, testEvent(17099838500), LaneLive)) + require.NoError(t, intake.ingest(ctx, testEvent(17099838500), LanePoll)) + + assert.Equal(t, 1, queue.Depth(), "the poll lane repeating the live lane is not a second unit of work") + assert.Equal(t, 1, countLines(pointers.String())) + + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &pointer)) + assert.Equal(t, int64(17099838500), pointer.EventID) + assert.Equal(t, "comment.created", pointer.EventType) + assert.Equal(t, int64(10304028972), pointer.RecordingID) + assert.Equal(t, string(StateSeen), pointer.State) +} + +// The pointer carries what the feed carried and nothing more. Anything with a +// title or a body in it would mean intake had read the recording, which is the +// one thing the delivery path must not do. +func TestPointerLineCarriesNoContent(t *testing.T) { + var pointers bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &pointers) + require.NoError(t, intake.ingest(context.Background(), testEvent(1), LanePoll)) + + var raw map[string]any + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &raw)) + for _, forbidden := range []string{"title", "content", "body", "url", "app_url", "creator_name", "excerpt"} { + assert.NotContains(t, raw, forbidden) + } +} + +func TestPollServedIDAdvancesOnlyOnAPageBoundary(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + + live := testEvent(17099999999) + live.ActorType = "person" + visible := true + live.VisibleToClients = &visible + require.NoError(t, intake.ingest(ctx, live, LaneOf(live))) + + intake.confirmPollServed(ctx) + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Zero(t, served, + "a live id is far ahead of the poll lane; re-entering at one skips everything inside the safety delay") + + polled := testEvent(17099838500) + require.NoError(t, intake.ingest(ctx, polled, LaneOf(polled))) + intake.confirmPollServed(ctx) + + served, err = ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Equal(t, int64(17099838500), served) +} + +func TestOverflowIsOnDiskBeforeItIsAccepted(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + + // One of the dropped ids is already in the ledger: it was dropped from the + // buffer, not from the connector. + _, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneLive) + require.NoError(t, err) + + disposition := intake.handleSignal(eventfeed.BufferOverflow{ + DroppedIDs: []int64{17099838500, 17099838501, 17099838502}, + DroppedCount: 3, + }) + assert.Equal(t, eventfeed.Accept, disposition) + + losses, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, losses, 1, "a loss that only ever lived in memory is one no restart could find") + assert.Equal(t, 3, losses[0].DroppedCount) + assert.Equal(t, int64(17099838500), losses[0].RepairSince, + "the walk enters one below the lowest MISSING id, and the feed's since is exclusive") + + missing, err := ledger.MissingIDs(ctx, losses[0].ID, LossMissing) + require.NoError(t, err) + assert.Equal(t, []int64{17099838501, 17099838502}, missing) + + neverLost, err := ledger.MissingIDs(ctx, losses[0].ID, LossNeverLost) + require.NoError(t, err) + assert.Equal(t, []int64{17099838500}, neverLost) +} + +func TestOverflowIsRefusedWhenItCannotBeWrittenDown(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + require.NoError(t, ledger.Close()) + + disposition := intake.handleSignal(eventfeed.BufferOverflow{ + DroppedIDs: []int64{17099838501}, + DroppedCount: 1, + }) + assert.Equal(t, eventfeed.Terminate, disposition, + "accepting an incompleteness that was never recorded is worse than ending the feed") +} + +func TestOverflowOfIdsAlreadyHeldResolvesImmediately(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneLive) + require.NoError(t, err) + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.BufferOverflow{ + DroppedIDs: []int64{17099838500}, + DroppedCount: 1, + })) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open, "nothing was lost, so there is nothing to walk for") +} + +func TestFeedGapRecordsTheEpochAndTheServedEntryClass(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + })) + + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Equal(t, GapEpoch, gaps[0].Class) + require.NotNil(t, gaps[0].EpochAfterID) + assert.Equal(t, int64(17099838487), *gaps[0].EpochAfterID) + assert.Equal(t, EntryReplay, gaps[0].EntryClass) +} + +func TestFeedGapClassifiesAPresentEntryFromTheServedCursor(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=now", + })) + + gaps, err := ledger.Gaps(context.Background()) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Equal(t, EntryPresent, gaps[0].EntryClass) +} + +func TestFeedGapIsRefusedWhenItCannotBeRecorded(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + require.NoError(t, ledger.Close()) + + assert.Equal(t, eventfeed.Terminate, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + }), "a gap nothing wrote down is a gap nothing will ever report") +} + +// A retention 410 reaching the account lane means the server said something +// this connector does not model. It is surfaced and recorded as its own class, +// never resumed as if it had been the epoch's. +func TestRetentionGoneOnTheAccountLaneIsRecordedAsItsOwnClass(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + ctx := context.Background() + + err := intake.classifyTerminal(ctx, &eventfeed.PollError{ + Kind: eventfeed.PollUnrecoverable, + Err: &InboxRetentionGoneError{Resume: "https://3.basecampapi.com/2914079/my/inbox.json?since=0"}, + }) + require.Error(t, err) + + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Equal(t, GapRetention, gaps[0].Class) + assert.Nil(t, gaps[0].EpochAfterID, "there is no epoch here, and zero is not one") +} + +func TestLedgerRefusesAGapWhoseClassAndEpochDisagree(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + _, err := ledger.RecordGap(ctx, Gap{DetectedAt: time.Now(), Class: GapEpoch}) + assert.Error(t, err, "an epoch gap without an epoch is the flattening this schema exists to refuse") + + epoch := int64(1) + _, err = ledger.RecordGap(ctx, Gap{DetectedAt: time.Now(), Class: GapRetention, EpochAfterID: &epoch}) + assert.Error(t, err, "a retention gap has no epoch to carry") +} + +func TestEntryClassOf(t *testing.T) { + assert.Equal(t, EntryPresent, entryClassOf("https://3.basecampapi.com/1/events.json?since=now")) + assert.Equal(t, EntryReplay, entryClassOf("https://3.basecampapi.com/1/events.json?since=17099838487")) + assert.Equal(t, EntryReplay, entryClassOf("https://3.basecampapi.com/1/events.json?since=0")) + assert.Equal(t, EntryUnknown, entryClassOf("https://3.basecampapi.com/1/events.json")) + assert.Equal(t, EntryUnknown, entryClassOf("https://3.basecampapi.com/1/events.json?since=soon")) + assert.Equal(t, EntryUnknown, entryClassOf("://not-a-url")) +} + +func TestReconciliationResumesOnStart(t *testing.T) { + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838501)}, + Position: "walk-1", + }}} + intake, ledger, _ := newTestIntake(t, polls, nil) + ctx := context.Background() + + _, err := ledger.RecordLoss(ctx, []int64{17099838501}, time.Now(), time.Minute) + require.NoError(t, err) + + require.NoError(t, intake.resumeReconciliation(ctx)) + intake.repairs.Wait() + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open, "a crash between the overflow and its repair is a delay, not a lost record") + + _, ok, err := ledger.Get(ctx, 17099838501) + require.NoError(t, err) + assert.True(t, ok) +} + +func TestNewRefusesAnIncompleteConfiguration(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 2) + require.NoError(t, err) + + base := Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + } + _, err = New(base) + require.NoError(t, err) + + noNamespace := base + noNamespace.ConsumerNamespace = "" + _, err = New(noNamespace) + assert.Error(t, err, "two connectors in one account must not share a checkpoint lineage") + + badFilters := base + badFilters.Filters = eventfeed.Filters{Types: []string{"comment.created,card.created"}} + _, err = New(badFilters) + assert.Error(t, err, "a filter set is validated before any wire attempt") +} + +func countLines(s string) int { + scanner := bufio.NewScanner(strings.NewReader(s)) + n := 0 + for scanner.Scan() { + if strings.TrimSpace(scanner.Text()) != "" { + n++ + } + } + return n +} + +// The card's acceptance number: a burst of a thousand events is seen within a +// minute. The bound here is far tighter than a minute on purpose — the point +// is to catch a regression that makes intake per-event expensive (a fsync per +// row, a read per pointer), not to measure the machine. +func TestABurstOfAThousandEventsIsAbsorbedQuickly(t *testing.T) { + var pointers bytes.Buffer + intake, ledger, queue := newTestIntake(t, nil, &pointers) + ctx := context.Background() + + start := time.Now() + for id := int64(17099838500); id < 17099838500+1000; id++ { + require.NoError(t, intake.ingest(ctx, testEvent(id), LanePoll)) + } + elapsed := time.Since(start) + + assert.Equal(t, 1000, queue.Depth()) + assert.Equal(t, 1000, countLines(pointers.String())) + seen, err := ledger.CountInState(ctx, StateSeen) + require.NoError(t, err) + assert.Equal(t, 1000, seen) + assert.Less(t, elapsed, 30*time.Second, "intake is the only work on the feed's delivery path") + t.Logf("1,000 events through intake in %s", elapsed) +} diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go new file mode 100644 index 000000000..579dbc210 --- /dev/null +++ b/internal/connector/ledger.go @@ -0,0 +1,205 @@ +package connector + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + _ "modernc.org/sqlite" // database/sql driver "sqlite", pure Go: no cgo on any of the five release targets. +) + +// RecordState is where an event sits in the ledger's lifecycle. +// +// Intake only ever writes StateSeen. The rest of the vocabulary is declared +// here because the states are one lifecycle, and a store that cannot name the +// state a later card writes cannot recover it on start either. +type RecordState string + +const ( + // StateSeen is a pointer intake wrote and nothing has judged yet. Every + // seen record re-runs the gate on start. + StateSeen RecordState = "seen" + // StateAdmitted passed the gate and awaits dispatch. + StateAdmitted RecordState = "admitted" + // StateQueued waits behind another event on its conversation. + StateQueued RecordState = "queued" + // StateBlocked is retained and retried: a reason, never a transport + // failure dressed up as a verdict. + StateBlocked RecordState = "blocked" + // StateDispatched was handed to a worker. + StateDispatched RecordState = "dispatched" + // StateCompleted is terminal with an outcome. + StateCompleted RecordState = "completed" + // StateDiscarded is terminal with a verified verdict. + StateDiscarded RecordState = "discarded" +) + +// Lane names which lane first served an event. It is diagnostic: dedupe is by +// id, and the same event ordinarily arrives on both. +type Lane string + +const ( + // LaneLive is the WebSocket. + LaneLive Lane = "live" + // LanePoll is the catch-up or streaming poll walk. + LanePoll Lane = "poll" + // LaneRepair is a repair walk after an overflow — its own cursor, never + // the feed's. + LaneRepair Lane = "repair" +) + +// Ledger is the connector's durable memory: the dedupe authority, the feed +// position, and the record of what was lost. +// +// It is a SQLite file rather than a set in memory because every promise the +// connector makes about a crash rests on the answer to "have I seen this id +// before?" surviving the crash. +type Ledger struct { + db *sql.DB + now func() time.Time +} + +// OpenLedger opens (creating if absent) the ledger at path and brings its +// schema up to date. +func OpenLedger(path string) (*Ledger, error) { + if path == "" { + return nil, errors.New("connector: ledger path is required") + } + if dir := filepath.Dir(path); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("connector: create ledger directory: %w", err) + } + } + + // _txlock=immediate takes the write lock when a transaction opens rather + // than on its first write. Without it two connectors racing on one file + // can both start, both read, and one is refused at COMMIT with the work + // already done. + dsn := "file:" + path + "?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("connector: open ledger: %w", err) + } + // One writer. SQLite serializes writers anyway, and a pool merely turns + // that serialization into SQLITE_BUSY under load. + db.SetMaxOpenConns(1) + + l := &Ledger{db: db, now: time.Now} + if err := l.migrate(context.Background()); err != nil { + _ = db.Close() + return nil, err + } + return l, nil +} + +// Close releases the ledger's handle. +func (l *Ledger) Close() error { return l.db.Close() } + +// migrations are applied in order, each exactly once. A migration is never +// edited after it ships: the ledger outlives the binary that created it. +var migrations = []string{ + ` +CREATE TABLE events ( + id INTEGER PRIMARY KEY, + state TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + lane TEXT NOT NULL, + event_type TEXT NOT NULL, + kind TEXT NOT NULL, + action TEXT NOT NULL, + bucket_id INTEGER NOT NULL, + creator_id INTEGER NOT NULL, + performed_by_id INTEGER, + recording_id INTEGER NOT NULL, + details BLOB, + actor_type TEXT NOT NULL DEFAULT '', + visible_to_clients INTEGER, + created_at TEXT NOT NULL, + seen_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + content_dropped INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX events_state_id ON events (state, id); + +CREATE TABLE checkpoints ( + flat_key TEXT PRIMARY KEY, + position TEXT NOT NULL, + last_poll_served_id INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL +); + +CREATE TABLE losses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + detected_at TEXT NOT NULL, + dropped_count INTEGER NOT NULL, + repair_since INTEGER NOT NULL, + repair_cursor TEXT NOT NULL DEFAULT '', + deadline_at TEXT NOT NULL, + resolved_at TEXT +); + +CREATE TABLE loss_ids ( + loss_id INTEGER NOT NULL REFERENCES losses (id) ON DELETE CASCADE, + event_id INTEGER NOT NULL, + state TEXT NOT NULL, + PRIMARY KEY (loss_id, event_id) +); +CREATE INDEX loss_ids_event ON loss_ids (event_id, state); + +CREATE TABLE gaps ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + detected_at TEXT NOT NULL, + class TEXT NOT NULL, + epoch_after_id INTEGER, + entry_class TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '' +); +`, +} + +func (l *Ledger) migrate(ctx context.Context) error { + if _, err := l.db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL +)`); err != nil { + return fmt.Errorf("connector: create migration table: %w", err) + } + + var applied int + if err := l.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&applied); err != nil { + return fmt.Errorf("connector: read schema version: %w", err) + } + + for i := applied; i < len(migrations); i++ { + version := i + 1 + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("connector: begin migration %d: %w", version, err) + } + if _, err := tx.ExecContext(ctx, migrations[i]); err != nil { + _ = tx.Rollback() + return fmt.Errorf("connector: apply migration %d: %w", version, err) + } + if _, err := tx.ExecContext(ctx, `INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)`, version, l.timestamp()); err != nil { + _ = tx.Rollback() + return fmt.Errorf("connector: record migration %d: %w", version, err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("connector: commit migration %d: %w", version, err) + } + } + return nil +} + +// SchemaVersion reports the highest applied migration. +func (l *Ledger) SchemaVersion(ctx context.Context) (int, error) { + var version int + err := l.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&version) + return version, err +} + +func (l *Ledger) timestamp() string { return l.now().UTC().Format(time.RFC3339Nano) } diff --git a/internal/connector/ledger_checkpoint.go b/internal/connector/ledger_checkpoint.go new file mode 100644 index 000000000..25340ca97 --- /dev/null +++ b/internal/connector/ledger_checkpoint.go @@ -0,0 +1,105 @@ +package connector + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// Ledger implements the feed's checkpoint seam. +var _ eventfeed.CheckpointStore = (*Ledger)(nil) + +// Load returns the stored position for key. It runs once, before the first +// mint, and a store failure is a failure to start rather than a silent entry +// at the present: re-entering at the head after a crash drops everything the +// feed committed while we were down. +func (l *Ledger) Load(ctx context.Context, key eventfeed.CheckpointKey) (string, bool, error) { + var position string + err := l.db.QueryRowContext(ctx, + `SELECT position FROM checkpoints WHERE flat_key = ? AND position <> ''`, key.FlatKey()).Scan(&position) + switch { + case errors.Is(err, sql.ErrNoRows): + // No row, or a row whose position was dropped by a 409 while its + // last poll-served id was kept. Both mean "no position to resume"; + // an empty string reported as present would be sent to the server + // as a position and refused as malformed. + return "", false, nil + case err != nil: + return "", false, fmt.Errorf("connector: load checkpoint: %w", err) + } + return position, true, nil +} + +// Save durably records position under key. +// +// The feed calls this only for a poll page, and only once that page's events +// have been accepted by the consumer — which, here, means they are already +// rows in the events table. A live event id never reaches this method, and +// must not: a live id runs ahead of the poll lane's safety delay, so a +// position taken from one would skip everything inside that window. +func (l *Ledger) Save(ctx context.Context, key eventfeed.CheckpointKey, position string) error { + _, err := l.db.ExecContext(ctx, ` +INSERT INTO checkpoints (flat_key, position, updated_at) VALUES (?, ?, ?) +ON CONFLICT (flat_key) DO UPDATE SET position = excluded.position, updated_at = excluded.updated_at`, + key.FlatKey(), position, l.timestamp()) + if err != nil { + return fmt.Errorf("connector: save checkpoint: %w", err) + } + return nil +} + +// NotePollServed advances the last poll-served event id for key. +// +// This is tracked apart from the position, and apart from the last id +// delivered, because all three answer different questions. The position is an +// opaque signed token, so nothing can be compared to it or derived from it. The +// last delivered id is usually a live id, thirty-odd seconds ahead of the poll +// lane. Only the last poll-served id is a safe place to re-enter the feed +// after a 409 or a rejected position — anything ahead of it skips events the +// poll lane had not served yet. +// +// It only ever moves forward. +func (l *Ledger) NotePollServed(ctx context.Context, key eventfeed.CheckpointKey, eventID int64) error { + _, err := l.db.ExecContext(ctx, ` +INSERT INTO checkpoints (flat_key, position, last_poll_served_id, updated_at) VALUES (?, '', ?, ?) +ON CONFLICT (flat_key) DO UPDATE SET + last_poll_served_id = MAX(checkpoints.last_poll_served_id, excluded.last_poll_served_id), + updated_at = excluded.updated_at`, + key.FlatKey(), eventID, l.timestamp()) + if err != nil { + return fmt.Errorf("connector: note poll-served id: %w", err) + } + return nil +} + +// LastPollServedID returns the last id the poll lane served under key, or zero +// if it has served none. +func (l *Ledger) LastPollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) { + var id int64 + err := l.db.QueryRowContext(ctx, + `SELECT last_poll_served_id FROM checkpoints WHERE flat_key = ?`, key.FlatKey()).Scan(&id) + switch { + case errors.Is(err, sql.ErrNoRows): + return 0, nil + case err != nil: + return 0, fmt.Errorf("connector: read last poll-served id: %w", err) + } + return id, nil +} + +// ForgetPosition drops the held position for key while keeping the last +// poll-served id — the 409 path. The server refused the position because it +// was minted for a different filter set; the id the poll lane had reached is +// still true, and is where the new digest re-enters. +func (l *Ledger) ForgetPosition(ctx context.Context, key eventfeed.CheckpointKey) error { + _, err := l.db.ExecContext(ctx, + `UPDATE checkpoints SET position = '', updated_at = ? WHERE flat_key = ?`, + l.timestamp(), key.FlatKey()) + if err != nil { + return fmt.Errorf("connector: forget position: %w", err) + } + return nil +} diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go new file mode 100644 index 000000000..448e0d5ae --- /dev/null +++ b/internal/connector/ledger_events.go @@ -0,0 +1,253 @@ +package connector + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// Record is one row of the events table. +type Record struct { + ID int64 + State RecordState + Reason string + Lane Lane + EventType string + Kind string + Action string + BucketID int64 + CreatorID int64 + PerformedByID *int64 + RecordingID int64 + Details json.RawMessage + ActorType string + VisibleToClients *bool + CreatedAt time.Time + SeenAt time.Time + UpdatedAt time.Time + ContentDropped bool +} + +// EffectivePerformer is the id the performers/exclude_performers filters +// match: the delegating agent when the action was delegated, else the creator. +func (r Record) EffectivePerformer() int64 { + if r.PerformedByID != nil { + return *r.PerformedByID + } + return r.CreatorID +} + +// RecordSeen writes the pointer as seen if the id is new, and reports whether +// it was. +// +// This is the dedupe, and it is durable on purpose. The poll lane runs about +// thirty seconds behind the live lane, so the ordinary case is the same event +// arriving twice; a restart in between would let an in-memory set answer "new" +// to the second copy and dispatch it a second time. +// +// A known id is never rewritten. That is what makes a terminal record a +// tombstone: the ledger keeps (id, state, timestamps) indefinitely even after +// its pointer payload is dropped, so an explicit replay of old history can +// never turn a finished event back into a new task. +// +// It also resolves the id against any open loss: an event the repair walk — or +// the ordinary poll lane, later — serves is one the overflow did not cost us. +func (l *Ledger) RecordSeen(ctx context.Context, ev eventfeed.Event, lane Lane) (bool, error) { + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return false, fmt.Errorf("connector: begin record seen: %w", err) + } + defer func() { _ = tx.Rollback() }() + + now := l.timestamp() + res, err := tx.ExecContext(ctx, ` +INSERT INTO events ( + id, state, lane, event_type, kind, action, bucket_id, creator_id, + performed_by_id, recording_id, details, actor_type, visible_to_clients, + created_at, seen_at, updated_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT (id) DO NOTHING`, + ev.ID, string(StateSeen), string(lane), ev.EventType, ev.Kind, ev.Action, + ev.BucketID, ev.CreatorID, ev.PerformedByID, ev.RecordingID, + detailsArg(ev.Details), ev.ActorType, ev.VisibleToClients, + ev.CreatedAt.UTC().Format(time.RFC3339Nano), now, now) + if err != nil { + return false, fmt.Errorf("connector: record seen %d: %w", ev.ID, err) + } + affected, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("connector: record seen %d: %w", ev.ID, err) + } + + if _, err := tx.ExecContext(ctx, + `UPDATE loss_ids SET state = ? WHERE event_id = ? AND state = ?`, + string(LossRecovered), ev.ID, string(LossMissing)); err != nil { + return false, fmt.Errorf("connector: resolve loss for %d: %w", ev.ID, err) + } + + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("connector: commit record seen %d: %w", ev.ID, err) + } + return affected == 1, nil +} + +// detailsArg keeps a details object out of the row when the event publishes +// none. Only boost.created and card.moved do; storing an empty blob for every +// other type would make "has details" unanswerable in SQL. +func detailsArg(details json.RawMessage) any { + if len(details) == 0 { + return nil + } + return []byte(details) +} + +// Get returns one record by event id. +func (l *Ledger) Get(ctx context.Context, id int64) (Record, bool, error) { + rows, err := l.db.QueryContext(ctx, selectRecords+` WHERE id = ?`, id) + if err != nil { + return Record{}, false, fmt.Errorf("connector: get event %d: %w", id, err) + } + records, err := scanRecords(rows) + if err != nil || len(records) == 0 { + return Record{}, false, err + } + return records[0], true, nil +} + +// RecordsInState returns up to limit records in state, oldest event first. +// Every non-terminal state is re-run on start, and this is how they are found. +func (l *Ledger) RecordsInState(ctx context.Context, state RecordState, limit int) ([]Record, error) { + rows, err := l.db.QueryContext(ctx, selectRecords+` WHERE state = ? ORDER BY id LIMIT ?`, string(state), limit) + if err != nil { + return nil, fmt.Errorf("connector: list %s records: %w", state, err) + } + return scanRecords(rows) +} + +// CountInState counts the records in state. +func (l *Ledger) CountInState(ctx context.Context, state RecordState) (int, error) { + var n int + err := l.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM events WHERE state = ?`, string(state)).Scan(&n) + if err != nil { + return 0, fmt.Errorf("connector: count %s records: %w", state, err) + } + return n, nil +} + +// SetState moves a record to state with a reason, which must be empty for +// every state but blocked and discarded — those two are the only ones a reason +// explains. +func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reason string) error { + res, err := l.db.ExecContext(ctx, + `UPDATE events SET state = ?, reason = ?, updated_at = ? WHERE id = ?`, + string(state), reason, l.timestamp(), id) + if err != nil { + return fmt.Errorf("connector: set state of %d: %w", id, err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("connector: set state of %d: %w", id, err) + } + if affected == 0 { + return fmt.Errorf("connector: set state of %d: %w", id, ErrNoSuchRecord) + } + return nil +} + +// ErrNoSuchRecord reports a state change addressed at an id the ledger does +// not hold. +var ErrNoSuchRecord = errors.New("no such event record") + +// DropContent drops the pointer payload of terminal records past their +// retention window and leaves the tombstone: id, state, outcome timestamps. +// +// The tombstone is what makes an explicit replay safe forever, so it is never +// deleted — only the payload goes. Non-terminal records are never touched: +// their payload is the only copy of what intake was told. +func (l *Ledger) DropContent(ctx context.Context, discardedBefore, completedBefore time.Time) (int, error) { + res, err := l.db.ExecContext(ctx, ` +UPDATE events +SET details = NULL, event_type = '', kind = '', action = '', bucket_id = 0, + creator_id = 0, performed_by_id = NULL, recording_id = 0, actor_type = '', + visible_to_clients = NULL, content_dropped = 1, updated_at = updated_at +WHERE content_dropped = 0 + AND ((state = ? AND updated_at < ?) OR (state = ? AND updated_at < ?))`, + string(StateDiscarded), discardedBefore.UTC().Format(time.RFC3339Nano), + string(StateCompleted), completedBefore.UTC().Format(time.RFC3339Nano)) + if err != nil { + return 0, fmt.Errorf("connector: drop content: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("connector: drop content: %w", err) + } + return int(affected), nil +} + +const selectRecords = ` +SELECT id, state, reason, lane, event_type, kind, action, bucket_id, creator_id, + performed_by_id, recording_id, details, actor_type, visible_to_clients, + created_at, seen_at, updated_at, content_dropped +FROM events` + +func scanRecords(rows *sql.Rows) ([]Record, error) { + defer func() { _ = rows.Close() }() + + var records []Record + for rows.Next() { + var ( + r Record + state, lane string + details []byte + createdAt, seenAt, updatedAt string + contentDropped int + performedBy sql.NullInt64 + visibleToClients sql.NullBool + ) + if err := rows.Scan(&r.ID, &state, &r.Reason, &lane, &r.EventType, &r.Kind, + &r.Action, &r.BucketID, &r.CreatorID, &performedBy, &r.RecordingID, + &details, &r.ActorType, &visibleToClients, &createdAt, &seenAt, + &updatedAt, &contentDropped); err != nil { + return nil, fmt.Errorf("connector: scan event record: %w", err) + } + r.State = RecordState(state) + r.Lane = Lane(lane) + if performedBy.Valid { + id := performedBy.Int64 + r.PerformedByID = &id + } + if visibleToClients.Valid { + v := visibleToClients.Bool + r.VisibleToClients = &v + } + if len(details) > 0 { + r.Details = json.RawMessage(details) + } + var err error + if r.CreatedAt, err = parseStamp(createdAt); err != nil { + return nil, err + } + if r.SeenAt, err = parseStamp(seenAt); err != nil { + return nil, err + } + if r.UpdatedAt, err = parseStamp(updatedAt); err != nil { + return nil, err + } + r.ContentDropped = contentDropped != 0 + records = append(records, r) + } + return records, rows.Err() +} + +func parseStamp(s string) (time.Time, error) { + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{}, fmt.Errorf("connector: parse ledger timestamp %q: %w", s, err) + } + return t, nil +} diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go new file mode 100644 index 000000000..9d7802119 --- /dev/null +++ b/internal/connector/ledger_recovery.go @@ -0,0 +1,372 @@ +package connector + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" +) + +// LossState is what became of one id the live buffer dropped. +type LossState string + +const ( + // LossMissing is an id the overflow dropped and nothing has served since. + LossMissing LossState = "missing" + // LossNeverLost is an id the ledger already held when the overflow was + // signaled. It was dropped from the buffer, not from the connector. + LossNeverLost LossState = "never_lost" + // LossRecovered is an id a later poll — the repair walk or the ordinary + // lane — served. + LossRecovered LossState = "recovered" + // LossUnrecovered is an id still missing when the repair window closed: a + // late straggler, a recording deleted before it became poll-visible, or + // history behind the epoch. It is reported, never hidden. + LossUnrecovered LossState = "unrecovered" +) + +// Loss is one accepted buffer overflow and the state of its reconciliation. +type Loss struct { + ID int64 + DetectedAt time.Time + DroppedCount int + // RepairSince is the walk's entry: one below the lowest missing id. + RepairSince int64 + // RepairCursor is the walk's own position, kept here and never written to + // the feed's checkpoint. + RepairCursor string + DeadlineAt time.Time + ResolvedAt *time.Time +} + +// GapClass distinguishes the feed's two 410s. They mean different things and +// must never share a recovery path. +type GapClass string + +const ( + // GapEpoch is the account feed's 410: the held position fell below the + // feed's epoch, and epoch_after_id names where servable history begins. + // The served resume URL is followed as given. + GapEpoch GapClass = "epoch" + // GapRetention is the inbox lane's 410: the held position fell out of the + // 30-day retention window, and there is no epoch — epoch_after_id is + // absent, not zero. The account lane must never produce one, and a + // connector that sees one has been told something it does not model. + GapRetention GapClass = "retention" +) + +// EntryClass is how the feed re-entered after a 410, decided by the cursor the +// server served — never by what the connector would have chosen. +type EntryClass string + +const ( + // EntryPresent is a resume at since=now: the entry takes the head and + // history below it is gone. + EntryPresent EntryClass = "present" + // EntryReplay is a resume at since=: the entry is a replay that + // checkpoints from its first poll page. + EntryReplay EntryClass = "replay" + // EntryUnknown is a resume whose cursor could not be read. + EntryUnknown EntryClass = "unknown" +) + +// Gap is one recorded 410. +type Gap struct { + ID int64 + DetectedAt time.Time + Class GapClass + // EpochAfterID is set exactly when Class is GapEpoch. It is a pointer + // because the distinction between "absent" and "zero" is the whole of the + // difference between the two 410s. + EpochAfterID *int64 + EntryClass EntryClass + Note string +} + +// RecordLoss writes an accepted buffer overflow, in one transaction, before +// the handler returns Accept. +// +// Accept means the consumer owns the incompleteness. Owning it starts with it +// being on disk: a crash a millisecond later must still find the loss and +// resume its reconciliation, and a loss that only ever lived in memory is one +// the next start would never know to repair. The corollary is that a failed +// write is a reason to refuse the signal, not to accept it anyway. +// +// Dropped ids the ledger already holds were lost from the buffer, not from the +// connector; they are recorded as never_lost so the repair walk does not go +// looking for events it already has. +func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Time, window time.Duration) (Loss, error) { + if len(droppedIDs) == 0 { + return Loss{}, errors.New("connector: buffer overflow with no dropped ids") + } + + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return Loss{}, fmt.Errorf("connector: begin record loss: %w", err) + } + defer func() { _ = tx.Rollback() }() + + states := make(map[int64]LossState, len(droppedIDs)) + lowestMissing := int64(0) + for _, id := range droppedIDs { + if _, seen := states[id]; seen { + continue + } + var one int + err := tx.QueryRowContext(ctx, `SELECT 1 FROM events WHERE id = ?`, id).Scan(&one) + switch { + case errors.Is(err, sql.ErrNoRows): + states[id] = LossMissing + if lowestMissing == 0 || id < lowestMissing { + lowestMissing = id + } + case err != nil: + return Loss{}, fmt.Errorf("connector: classify dropped id %d: %w", id, err) + default: + states[id] = LossNeverLost + } + } + + loss := Loss{ + DetectedAt: now.UTC(), + DroppedCount: len(droppedIDs), + // One below the lowest missing id, so the walk's first page can serve + // it: the feed's since is exclusive. + RepairSince: lowestMissing - 1, + DeadlineAt: now.UTC().Add(window), + } + if lowestMissing == 0 { + // Every dropped id was already in the ledger. There is nothing to + // walk for, but the overflow still happened and is still recorded. + loss.RepairSince = 0 + resolved := now.UTC() + loss.ResolvedAt = &resolved + } + + res, err := tx.ExecContext(ctx, + `INSERT INTO losses (detected_at, dropped_count, repair_since, deadline_at, resolved_at) VALUES (?, ?, ?, ?, ?)`, + stamp(loss.DetectedAt), loss.DroppedCount, loss.RepairSince, stamp(loss.DeadlineAt), nullableStamp(loss.ResolvedAt)) + if err != nil { + return Loss{}, fmt.Errorf("connector: insert loss: %w", err) + } + if loss.ID, err = res.LastInsertId(); err != nil { + return Loss{}, fmt.Errorf("connector: insert loss: %w", err) + } + + for id, state := range states { + if _, err := tx.ExecContext(ctx, + `INSERT INTO loss_ids (loss_id, event_id, state) VALUES (?, ?, ?)`, + loss.ID, id, string(state)); err != nil { + return Loss{}, fmt.Errorf("connector: insert dropped id %d: %w", id, err) + } + } + + if err := tx.Commit(); err != nil { + return Loss{}, fmt.Errorf("connector: commit loss: %w", err) + } + return loss, nil +} + +// OpenLosses returns the losses whose reconciliation has not finished, oldest +// first. Reconciliation resumes from this on every start. +func (l *Ledger) OpenLosses(ctx context.Context) ([]Loss, error) { + rows, err := l.db.QueryContext(ctx, ` +SELECT id, detected_at, dropped_count, repair_since, repair_cursor, deadline_at, resolved_at +FROM losses WHERE resolved_at IS NULL ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("connector: list open losses: %w", err) + } + defer func() { _ = rows.Close() }() + + var losses []Loss + for rows.Next() { + var ( + loss Loss + detected, deadline string + resolved sql.NullString + ) + if err := rows.Scan(&loss.ID, &detected, &loss.DroppedCount, &loss.RepairSince, + &loss.RepairCursor, &deadline, &resolved); err != nil { + return nil, fmt.Errorf("connector: scan loss: %w", err) + } + var err error + if loss.DetectedAt, err = parseStamp(detected); err != nil { + return nil, err + } + if loss.DeadlineAt, err = parseStamp(deadline); err != nil { + return nil, err + } + if resolved.Valid { + t, err := parseStamp(resolved.String) + if err != nil { + return nil, err + } + loss.ResolvedAt = &t + } + losses = append(losses, loss) + } + return losses, rows.Err() +} + +// MissingIDs returns the ids of a loss still in state, ascending. +func (l *Ledger) MissingIDs(ctx context.Context, lossID int64, state LossState) ([]int64, error) { + rows, err := l.db.QueryContext(ctx, + `SELECT event_id FROM loss_ids WHERE loss_id = ? AND state = ? ORDER BY event_id`, + lossID, string(state)) + if err != nil { + return nil, fmt.Errorf("connector: list loss ids: %w", err) + } + defer func() { _ = rows.Close() }() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("connector: scan loss id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// SaveRepairCursor records where the repair walk has reached. +// +// This is the walk's own cursor and it is stored on the loss, never on the +// feed's checkpoint. The walk is seeded from a live id that ran far ahead of +// the poll lane; a checkpoint taken from it would skip every event still +// inside the safety delay behind it. +func (l *Ledger) SaveRepairCursor(ctx context.Context, lossID int64, cursor string) error { + _, err := l.db.ExecContext(ctx, + `UPDATE losses SET repair_cursor = ? WHERE id = ?`, cursor, lossID) + if err != nil { + return fmt.Errorf("connector: save repair cursor: %w", err) + } + return nil +} + +// CloseLoss ends a loss's reconciliation: everything still missing becomes +// unrecovered, and the loss is resolved. It reports how many ids were left +// unrecovered. +func (l *Ledger) CloseLoss(ctx context.Context, lossID int64, now time.Time) (int, error) { + tx, err := l.db.BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("connector: begin close loss: %w", err) + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.ExecContext(ctx, + `UPDATE loss_ids SET state = ? WHERE loss_id = ? AND state = ?`, + string(LossUnrecovered), lossID, string(LossMissing)) + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered: %w", err) + } + unrecovered, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered: %w", err) + } + + if _, err := tx.ExecContext(ctx, + `UPDATE losses SET resolved_at = ? WHERE id = ? AND resolved_at IS NULL`, + stamp(now.UTC()), lossID); err != nil { + return 0, fmt.Errorf("connector: resolve loss: %w", err) + } + + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("connector: commit close loss: %w", err) + } + return int(unrecovered), nil +} + +// UnrecoveredIDs returns every id the connector has given up on, across all +// losses. `basecamp connect status` and `doctor` show these: an unrecovered id +// is documented, not hidden, and the poll lane may still serve it later. +func (l *Ledger) UnrecoveredIDs(ctx context.Context) ([]int64, error) { + rows, err := l.db.QueryContext(ctx, + `SELECT DISTINCT event_id FROM loss_ids WHERE state = ? ORDER BY event_id`, + string(LossUnrecovered)) + if err != nil { + return nil, fmt.Errorf("connector: list unrecovered ids: %w", err) + } + defer func() { _ = rows.Close() }() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("connector: scan unrecovered id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// RecordGap writes one 410. Nothing is posted to Basecamp for a gap; it is a +// fact about the feed, shown by status and doctor. +func (l *Ledger) RecordGap(ctx context.Context, gap Gap) (int64, error) { + switch gap.Class { + case GapEpoch: + if gap.EpochAfterID == nil { + return 0, errors.New("connector: an epoch gap must carry epoch_after_id") + } + case GapRetention: + if gap.EpochAfterID != nil { + return 0, errors.New("connector: a retention gap has no epoch_after_id") + } + default: + return 0, fmt.Errorf("connector: unknown gap class %q", gap.Class) + } + + res, err := l.db.ExecContext(ctx, + `INSERT INTO gaps (detected_at, class, epoch_after_id, entry_class, note) VALUES (?, ?, ?, ?, ?)`, + stamp(gap.DetectedAt.UTC()), string(gap.Class), gap.EpochAfterID, string(gap.EntryClass), gap.Note) + if err != nil { + return 0, fmt.Errorf("connector: record gap: %w", err) + } + return res.LastInsertId() +} + +// Gaps returns every recorded 410, oldest first. +func (l *Ledger) Gaps(ctx context.Context) ([]Gap, error) { + rows, err := l.db.QueryContext(ctx, + `SELECT id, detected_at, class, epoch_after_id, entry_class, note FROM gaps ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("connector: list gaps: %w", err) + } + defer func() { _ = rows.Close() }() + + var gaps []Gap + for rows.Next() { + var ( + gap Gap + detected string + class string + entryClass string + epoch sql.NullInt64 + ) + if err := rows.Scan(&gap.ID, &detected, &class, &epoch, &entryClass, &gap.Note); err != nil { + return nil, fmt.Errorf("connector: scan gap: %w", err) + } + var err error + if gap.DetectedAt, err = parseStamp(detected); err != nil { + return nil, err + } + gap.Class = GapClass(class) + gap.EntryClass = EntryClass(entryClass) + if epoch.Valid { + id := epoch.Int64 + gap.EpochAfterID = &id + } + gaps = append(gaps, gap) + } + return gaps, rows.Err() +} + +func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339Nano) } + +func nullableStamp(t *time.Time) any { + if t == nil { + return nil + } + return stamp(*t) +} diff --git a/internal/connector/ledger_test.go b/internal/connector/ledger_test.go new file mode 100644 index 000000000..47c0fe16a --- /dev/null +++ b/internal/connector/ledger_test.go @@ -0,0 +1,193 @@ +package connector + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +func newTestLedger(t *testing.T) *Ledger { + t.Helper() + ledger, err := OpenLedger(filepath.Join(t.TempDir(), "connector.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = ledger.Close() }) + return ledger +} + +func testEvent(id int64) eventfeed.Event { + return eventfeed.Event{ + ID: id, + Kind: "comment_created", + EventType: "comment.created", + Action: "created", + CreatedAt: time.Date(2026, 9, 16, 10, 0, 0, 0, time.UTC), + BucketID: 48699913, + CreatorID: 26909558, + RecordingID: 10304028972, + } +} + +func testKey() eventfeed.CheckpointKey { + return eventfeed.CheckpointKey{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + FilterKey: "srv2-9f2ab04e5c11d3a7", + } +} + +func TestRecordSeenDedupesAcrossLanes(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + fresh, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneLive) + require.NoError(t, err) + assert.True(t, fresh) + + // The poll lane serving what the live lane already delivered is the + // ordinary case, not an anomaly. + fresh, err = ledger.RecordSeen(ctx, testEvent(17099838500), LanePoll) + require.NoError(t, err) + assert.False(t, fresh) + + record, ok, err := ledger.Get(ctx, 17099838500) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, LaneLive, record.Lane, "the first lane to serve it stands") + assert.Equal(t, StateSeen, record.State) +} + +func TestRecordSeenKeepsDetailsVerbatimAndNilWhenAbsent(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + withDetails := testEvent(1) + withDetails.EventType = "card.moved" + withDetails.Details = json.RawMessage(`{"column_id":10287321848,"previous_column_id":null}`) + _, err := ledger.RecordSeen(ctx, withDetails, LanePoll) + require.NoError(t, err) + + _, err = ledger.RecordSeen(ctx, testEvent(2), LanePoll) + require.NoError(t, err) + + moved, _, err := ledger.Get(ctx, 1) + require.NoError(t, err) + assert.JSONEq(t, `{"column_id":10287321848,"previous_column_id":null}`, string(moved.Details)) + + plain, _, err := ledger.Get(ctx, 2) + require.NoError(t, err) + assert.Nil(t, plain.Details, "an event that publishes no details must not be stored with an empty one") +} + +func TestDroppedContentLeavesATombstoneThatStillDedupes(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + _, err := ledger.RecordSeen(ctx, testEvent(42), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 42, StateDiscarded, "untrusted_author")) + + dropped, err := ledger.DropContent(ctx, time.Now().Add(time.Hour), time.Now().Add(time.Hour)) + require.NoError(t, err) + assert.Equal(t, 1, dropped) + + record, ok, err := ledger.Get(ctx, 42) + require.NoError(t, err) + require.True(t, ok) + assert.True(t, record.ContentDropped) + assert.Empty(t, record.EventType, "the payload goes") + assert.Equal(t, StateDiscarded, record.State, "the outcome stays") + + // The whole point of keeping the tombstone: an explicit replay of old + // history can never turn a finished event back into a new task. + fresh, err := ledger.RecordSeen(ctx, testEvent(42), LanePoll) + require.NoError(t, err) + assert.False(t, fresh) +} + +func TestDropContentLeavesNonTerminalRecordsAlone(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + _, err := ledger.RecordSeen(ctx, testEvent(7), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 7, StateBlocked, "read_failed")) + + dropped, err := ledger.DropContent(ctx, time.Now().Add(time.Hour), time.Now().Add(time.Hour)) + require.NoError(t, err) + assert.Zero(t, dropped) + + record, _, err := ledger.Get(ctx, 7) + require.NoError(t, err) + assert.Equal(t, "comment.created", record.EventType, + "a blocked record's pointer is the only copy of what intake was told") +} + +func TestCheckpointRoundTrip(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + key := testKey() + + _, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + assert.False(t, ok) + + require.NoError(t, ledger.Save(ctx, key, "opaque-position-1")) + position, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, "opaque-position-1", position) +} + +func TestForgetPositionKeepsTheLastPollServedID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + key := testKey() + + require.NoError(t, ledger.Save(ctx, key, "opaque-position-1")) + require.NoError(t, ledger.NotePollServed(ctx, key, 17099838600)) + require.NoError(t, ledger.ForgetPosition(ctx, key)) + + _, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + assert.False(t, ok, "a dropped position must not read back as a present empty one") + + served, err := ledger.LastPollServedID(ctx, key) + require.NoError(t, err) + assert.Equal(t, int64(17099838600), served, + "the 409 invalidates the position, not the id the poll lane had reached") +} + +func TestNotePollServedOnlyMovesForward(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + key := testKey() + + require.NoError(t, ledger.NotePollServed(ctx, key, 500)) + require.NoError(t, ledger.NotePollServed(ctx, key, 400)) + + served, err := ledger.LastPollServedID(ctx, key) + require.NoError(t, err) + assert.Equal(t, int64(500), served) +} + +func TestCheckpointsAreKeyedByFilterDigest(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + narrow := testKey() + wide := testKey() + wide.FilterKey = "srv2-0000000000000000" + + require.NoError(t, ledger.Save(ctx, narrow, "narrow-position")) + _, ok, err := ledger.Load(ctx, wide) + require.NoError(t, err) + assert.False(t, ok, "a filter change re-enters under its own lineage") +} diff --git a/internal/connector/lock.go b/internal/connector/lock.go new file mode 100644 index 000000000..1ca43897b --- /dev/null +++ b/internal/connector/lock.go @@ -0,0 +1,97 @@ +package connector + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/gofrs/flock" +) + +// ErrAlreadyRunning reports a second connector for the same agent. +var ErrAlreadyRunning = errors.New("connector: another connector already holds this account and agent") + +// InstanceLock is the refusal of a second connector on one agent identity. +// +// It is keyed on the ACCOUNT and the agent's Person id, not on the profile +// name. Two profiles can hold credentials for the same agent, and the failure +// this prevents is not "the same configuration twice" — it is one agent's +// mentions being dispatched twice, which is a property of the identity, not of +// the file that names it. +// +// The kernel drops an flock when the holding descriptor closes, process death +// included, so a crashed connector cannot wedge the lock and there is no +// stale-lock reaping to get wrong. The metadata written beside it is +// diagnostic only: the lock is the lock. +type InstanceLock struct { + flock *flock.Flock + path string +} + +// instanceHolder is what a running connector writes beside its lock so the +// refusal can say who is holding it. +type instanceHolder struct { + PID int `json:"pid"` + StartedAt string `json:"started_at"` + AccountID string `json:"account_id"` + AgentID int64 `json:"agent_person_id"` +} + +// AcquireInstanceLock takes the lock for one account and agent, or refuses. +func AcquireInstanceLock(dir, accountID string, agentPersonID int64, now time.Time) (*InstanceLock, error) { + if accountID == "" || agentPersonID <= 0 { + return nil, errors.New("connector: the instance lock needs an account id and an agent person id") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("connector: create state directory: %w", err) + } + + path := filepath.Join(dir, "instance-"+accountID+"-"+strconv.FormatInt(agentPersonID, 10)+".lock") + lock := flock.New(path) + held, err := lock.TryLock() + if err != nil { + return nil, fmt.Errorf("connector: take the instance lock: %w", err) + } + if !held { + return nil, fmt.Errorf("%w: %s", ErrAlreadyRunning, describeHolder(path)) + } + + holder, err := json.Marshal(instanceHolder{ + PID: os.Getpid(), + StartedAt: now.UTC().Format(time.RFC3339), + AccountID: accountID, + AgentID: agentPersonID, + }) + if err == nil { + // Best effort, and deliberately after the lock is held: a connector + // that cannot describe itself still must not be a second connector. + _ = os.WriteFile(path+".json", append(holder, '\n'), 0o600) + } + + return &InstanceLock{flock: lock, path: path}, nil +} + +// Release drops the lock. +func (l *InstanceLock) Release() error { + _ = os.Remove(l.path + ".json") + return l.flock.Unlock() +} + +// Path is the lock file, for diagnostics. +func (l *InstanceLock) Path() string { return l.path } + +func describeHolder(path string) string { + raw, err := os.ReadFile(path + ".json") + if err != nil { + return "held by another process" + } + var holder instanceHolder + if err := json.Unmarshal(raw, &holder); err != nil { + return "held by another process" + } + return fmt.Sprintf("held by pid %d since %s", holder.PID, holder.StartedAt) +} diff --git a/internal/connector/lock_test.go b/internal/connector/lock_test.go new file mode 100644 index 000000000..a9de38cfd --- /dev/null +++ b/internal/connector/lock_test.go @@ -0,0 +1,64 @@ +package connector + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Two connectors on one agent identity dispatch every mention twice. The Ruby +// connector's own notes record that happening. +func TestSecondConnectorOnTheSameAgentIsRefused(t *testing.T) { + dir := t.TempDir() + now := time.Now() + + first, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + t.Cleanup(func() { _ = first.Release() }) + + _, err = AcquireInstanceLock(dir, "2914079", 52007412, now) + require.ErrorIs(t, err, ErrAlreadyRunning) + assert.Contains(t, err.Error(), "pid", "the refusal says who is holding it") +} + +// The key is the identity, not the profile that names it: two profiles can +// hold credentials for one agent, and it is the agent that gets double-served. +func TestTheLockIsKeyedOnAccountAndAgent(t *testing.T) { + dir := t.TempDir() + now := time.Now() + + first, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + t.Cleanup(func() { _ = first.Release() }) + + otherAgent, err := AcquireInstanceLock(dir, "2914079", 26909558, now) + require.NoError(t, err, "a different agent in the same account is a different connector") + t.Cleanup(func() { _ = otherAgent.Release() }) + + otherAccount, err := AcquireInstanceLock(dir, "5951425", 52007412, now) + require.NoError(t, err, "the same agent id in another account is a different connector") + t.Cleanup(func() { _ = otherAccount.Release() }) +} + +func TestReleasedLockCanBeRetaken(t *testing.T) { + dir := t.TempDir() + now := time.Now() + + first, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + require.NoError(t, first.Release()) + + second, err := AcquireInstanceLock(dir, "2914079", 52007412, now) + require.NoError(t, err) + require.NoError(t, second.Release()) +} + +func TestLockRefusesAnIncompleteIdentity(t *testing.T) { + dir := t.TempDir() + _, err := AcquireInstanceLock(dir, "", 52007412, time.Now()) + assert.Error(t, err) + _, err = AcquireInstanceLock(dir, "2914079", 0, time.Now()) + assert.Error(t, err) +} diff --git a/internal/connector/queue.go b/internal/connector/queue.go new file mode 100644 index 000000000..6e5577fb8 --- /dev/null +++ b/internal/connector/queue.go @@ -0,0 +1,137 @@ +package connector + +import ( + "context" + "errors" + "sync" + "sync/atomic" +) + +// Backlog thresholds. Intake is the only work on the feed's delivery path, so +// the queue is where a slow admission shows up. +const ( + // DefaultBacklogWarn is the depth at which the backlog is worth saying out + // loud. Nothing changes; the operator is told. + DefaultBacklogWarn = 1_000 + // DefaultBacklogPause is the depth at which intake stops consuming the + // feed. The checkpoint does not move while it is paused, because the + // package saves a position only after its page's events were accepted and + // intake has stopped accepting them — so a crash while paused resumes from + // before the backlog rather than after it. + DefaultBacklogPause = 10_000 +) + +// ErrQueueClosed reports work offered to, or taken from, a closed queue. +var ErrQueueClosed = errors.New("connector: intake queue is closed") + +// Queue is the seam between intake and admission: intake writes a pointer and +// hands over an id, admission reads it when it gets there. Two queues with +// visible depth rather than one pipeline, so a busy dispatcher can never stall +// the socket — and so the place where work is piling up is the place the depth +// is showing. +type Queue struct { + ids chan int64 + warnAt int + closeOne sync.Once + + warned atomic.Bool + paused atomic.Bool + + // OnWarn fires when the depth first crosses the warning threshold, and + // OnRecover when it falls back below. Both are optional. + OnWarn func(depth int) + OnRecover func(depth int) + // OnPause fires when an offer begins waiting for room, and OnResume when + // it stops. The feed is not being consumed in between. + OnPause func(depth int) + OnResume func(depth int) +} + +// NewQueue builds a queue that warns at warnAt and pauses the feed at pauseAt. +func NewQueue(warnAt, pauseAt int) (*Queue, error) { + if pauseAt <= 0 { + return nil, errors.New("connector: queue pause threshold must be positive") + } + if warnAt <= 0 || warnAt > pauseAt { + return nil, errors.New("connector: queue warning threshold must be positive and no higher than the pause threshold") + } + // Capacity IS the pause threshold: a full channel is a blocked offer is a + // feed that has stopped being read. There is no second mechanism to keep + // in agreement with this one. + return &Queue{ids: make(chan int64, pauseAt), warnAt: warnAt}, nil +} + +// Offer hands an event id to admission, waiting for room when the backlog is +// at the pause threshold. Waiting here is the pause: the caller is the feed's +// delivery path, and it is not reading the feed while it waits. +func (q *Queue) Offer(ctx context.Context, id int64) error { + select { + case q.ids <- id: + q.noteDepth() + return nil + default: + } + + q.paused.Store(true) + if q.OnPause != nil { + q.OnPause(q.Depth()) + } + defer func() { + q.paused.Store(false) + if q.OnResume != nil { + q.OnResume(q.Depth()) + } + }() + + select { + case q.ids <- id: + q.noteDepth() + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Take returns the next id, waiting for one. It reports ErrQueueClosed once +// the queue is closed and drained. +func (q *Queue) Take(ctx context.Context) (int64, error) { + select { + case id, ok := <-q.ids: + if !ok { + return 0, ErrQueueClosed + } + q.noteDepth() + return id, nil + case <-ctx.Done(): + return 0, ctx.Err() + } +} + +// Depth is the number of ids waiting. +func (q *Queue) Depth() int { return len(q.ids) } + +// Paused reports whether an offer is currently waiting for room — which is to +// say whether the feed is being consumed. +func (q *Queue) Paused() bool { return q.paused.Load() } + +// Close stops the queue. Takers drain what is already queued and then see +// ErrQueueClosed. +func (q *Queue) Close() { q.closeOne.Do(func() { close(q.ids) }) } + +// noteDepth fires the warning edges. It is edge-triggered, not level: a +// backlog that sits above the threshold for an hour is one warning, and the +// recovery is the other half of the pair, so a warning is never left standing +// after the thing it warned about went away. +func (q *Queue) noteDepth() { + depth := q.Depth() + switch { + case depth >= q.warnAt && q.warned.CompareAndSwap(false, true): + if q.OnWarn != nil { + q.OnWarn(depth) + } + case depth < q.warnAt && q.warned.CompareAndSwap(true, false): + if q.OnRecover != nil { + q.OnRecover(depth) + } + } +} diff --git a/internal/connector/queue_test.go b/internal/connector/queue_test.go new file mode 100644 index 000000000..2010d74a0 --- /dev/null +++ b/internal/connector/queue_test.go @@ -0,0 +1,105 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestQueueWarnsOnceOnTheWayUpAndRecoversOnTheWayDown(t *testing.T) { + queue, err := NewQueue(2, 4) + require.NoError(t, err) + + var warnings, recoveries int + queue.OnWarn = func(int) { warnings++ } + queue.OnRecover = func(int) { recoveries++ } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + assert.Zero(t, warnings) + + require.NoError(t, queue.Offer(ctx, 2)) + require.NoError(t, queue.Offer(ctx, 3)) + assert.Equal(t, 1, warnings, "a backlog that sits above the threshold is one warning, not one per event") + + _, err = queue.Take(ctx) + require.NoError(t, err) + _, err = queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, 1, recoveries, "a warning must not be left standing after the backlog drained") +} + +// At the pause threshold intake stops consuming the feed. Because the feed +// saves a position only after its page's events were accepted, and intake has +// stopped accepting them, the checkpoint stops moving too — which is what makes +// the pause safe rather than a way to lose the backlog. +func TestQueuePausesTheCallerAtTheThreshold(t *testing.T) { + queue, err := NewQueue(1, 2) + require.NoError(t, err) + + paused := make(chan int, 1) + queue.OnPause = func(depth int) { paused <- depth } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + require.NoError(t, queue.Offer(ctx, 2)) + assert.Equal(t, 2, queue.Depth()) + + done := make(chan error, 1) + go func() { done <- queue.Offer(ctx, 3) }() + + select { + case depth := <-paused: + assert.Equal(t, 2, depth) + case <-time.After(2 * time.Second): + t.Fatal("the third offer should have waited for room") + } + assert.True(t, queue.Paused()) + + select { + case err := <-done: + t.Fatalf("the third offer returned while the queue was full: %v", err) + case <-time.After(50 * time.Millisecond): + } + + _, err = queue.Take(ctx) + require.NoError(t, err) + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("the offer should have resumed once there was room") + } +} + +func TestQueueOfferHonoursCancellation(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + require.NoError(t, queue.Offer(context.Background(), 1)) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- queue.Offer(ctx, 2) }() + time.Sleep(20 * time.Millisecond) + cancel() + + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("a paused offer must unblock on shutdown") + } +} + +func TestQueueRefusesNonsenseThresholds(t *testing.T) { + _, err := NewQueue(10, 5) + assert.Error(t, err) + _, err = NewQueue(0, 5) + assert.Error(t, err) + _, err = NewQueue(5, 0) + assert.Error(t, err) +} diff --git a/internal/connector/repair.go b/internal/connector/repair.go new file mode 100644 index 000000000..4c07236c3 --- /dev/null +++ b/internal/connector/repair.go @@ -0,0 +1,195 @@ +package connector + +import ( + "context" + "errors" + "log/slog" + "strconv" + "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// repairWalker reconciles one accepted buffer overflow. +// +// It repairs with EXPLICIT IDS and never with the feed's position. The +// position is an opaque signed token, so nothing can be compared to it; and +// the overflow's ids are live ids, far ahead of the poll lane's safety delay, +// so a checkpoint taken from one would skip everything behind it that the +// delay had not yet served. The walk therefore runs on its own cursor, kept on +// the loss record, and the feed's checkpoint is never written from it. +// +// It runs off the delivery path. Nothing about live intake waits for it. +type repairWalker struct { + ledger *Ledger + polls eventfeed.PollSource + filters eventfeed.Filters + ingest func(ctx context.Context, event eventfeed.Event, lane Lane) error + now func() time.Time + interval time.Duration + log *slog.Logger + + // sleep is the wait between repair polls; nil means time.After. Injected + // so a test can run ten minutes of repair polls without taking ten + // minutes. + sleep func(ctx context.Context, d time.Duration) error +} + +func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { + for { + done, err := w.settled(ctx, loss) + if err != nil || done { + return err + } + + cursor, err := w.walk(ctx, &loss) + if errors.Is(err, errReconciliationEnded) { + return nil + } + if err != nil { + return err + } + loss.RepairCursor = cursor + + // Re-read before waiting: a walk that served everything has nothing + // left to wait for. + done, err = w.settled(ctx, loss) + if err != nil || done { + return err + } + + if !w.now().Before(loss.DeadlineAt) { + // The window is closed. What is still missing is a late + // straggler, a recording deleted before it became poll-visible, + // or history behind the epoch. It is recorded as unrecovered and + // shown by status — the poll lane may still serve it later, and + // intake resolves it then like any other event. + unrecovered, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()) + if err != nil { + return err + } + if unrecovered > 0 { + w.log.Error("a buffer overflow left events unrecovered", "loss_id", loss.ID, "unrecovered", unrecovered) + } + return nil + } + + // A missing id the walk did not serve is NOT yet a gap: `next` can end + // while an event is still inside the poll lane's safety delay. The + // walk repeats on the repair cadence until the window closes. + if err := w.wait(ctx, w.interval); err != nil { + return err + } + } +} + +// settled closes the loss and reports true when nothing is missing any more. +func (w *repairWalker) settled(ctx context.Context, loss Loss) (bool, error) { + missing, err := w.ledger.MissingIDs(ctx, loss.ID, LossMissing) + if err != nil { + return false, err + } + if len(missing) > 0 { + return false, nil + } + if _, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()); err != nil { + return false, err + } + w.log.Info("a buffer overflow was fully reconciled", "loss_id", loss.ID) + return true, nil +} + +// walk runs one pass from the loss's own cursor to the frozen head, following +// `next` until it is absent. It returns the cursor to resume the next pass at. +func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { + cursor := eventfeed.Cursor{Since: strconv.FormatInt(loss.RepairSince, 10)} + if loss.RepairCursor != "" { + cursor = eventfeed.Cursor{Position: loss.RepairCursor} + } + + last := loss.RepairCursor + for { + page, err := w.polls.Poll(ctx, cursor, w.filters) + if err != nil { + return last, w.pollFailure(ctx, loss, err) + } + + for _, event := range page.Events { + // Through intake like any other event: the ledger's dedupe is + // what marks the missing id recovered, and it is one code path + // rather than two that must agree. + if err := w.ingest(ctx, event, LaneRepair); err != nil { + return last, err + } + } + + if page.Position != "" { + last = page.Position + if err := w.ledger.SaveRepairCursor(ctx, loss.ID, page.Position); err != nil { + return last, err + } + } + + if page.Next == "" { + // The walk reached its frozen head. That is NOT "caught up": a + // page cut short by the safety horizon withholds the link on + // purpose, so the caller polls again rather than concluding + // anything. + return last, nil + } + // An empty page with a `next` is ordinary — the walk crossed rows the + // filters excluded — so the loop never stops on len(Events) == 0. + cursor = eventfeed.Cursor{PageURL: page.Next} + } +} + +// errReconciliationEnded reports that the walk itself closed the loss, so the +// caller stops rather than looping back to find nothing missing and calling +// that a full recovery. +var errReconciliationEnded = errors.New("connector: reconciliation ended inside the repair walk") + +// pollFailure decides what one failed repair poll means. Only a 410 ends the +// reconciliation: the ids below the epoch are gone, and no number of repeats +// will serve them. +func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, err error) error { + var pollErr *eventfeed.PollError + if errors.As(err, &pollErr) && pollErr.Kind == eventfeed.PollGone { + epoch := pollErr.EpochAfterID + if _, recordErr := w.ledger.RecordGap(ctx, Gap{ + DetectedAt: w.now(), + Class: GapEpoch, + EpochAfterID: &epoch, + EntryClass: entryClassOf(pollErr.ResumeURL), + Note: "the repair walk was seeded below the feed's epoch", + }); recordErr != nil { + return recordErr + } + unrecovered, closeErr := w.ledger.CloseLoss(ctx, loss.ID, w.now()) + if closeErr != nil { + return closeErr + } + w.log.Error("the repair walk fell below the feed's epoch; the dropped events are behind it", + "loss_id", loss.ID, "epoch_after_id", epoch, "unrecovered", unrecovered) + return errReconciliationEnded + } + + // Anything else — a transient, a throttle, an unauthorized — is a reason + // to try again on the next repair poll, not a reason to call the ids + // unrecovered. A slow or throttled walk delays nothing else. + w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "error", err) + return nil +} + +func (w *repairWalker) wait(ctx context.Context, d time.Duration) error { + if w.sleep != nil { + return w.sleep(ctx, d) + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/connector/repair_test.go b/internal/connector/repair_test.go new file mode 100644 index 000000000..533b34265 --- /dev/null +++ b/internal/connector/repair_test.go @@ -0,0 +1,268 @@ +package connector + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +type walkClock struct{ at time.Time } + +func (c *walkClock) now() time.Time { return c.at } +func (c *walkClock) advance(d time.Duration) func(context.Context, time.Duration) error { + return func(context.Context, time.Duration) error { + c.at = c.at.Add(d) + return nil + } +} + +func newTestWalker(t *testing.T, ledger *Ledger, polls eventfeed.PollSource, clock *walkClock) (*repairWalker, *[]int64) { + t.Helper() + var ingested []int64 + walker := &repairWalker{ + ledger: ledger, + polls: polls, + now: clock.now, + log: slog.New(slog.DiscardHandler), + sleep: clock.advance(time.Minute), + ingest: func(ctx context.Context, event eventfeed.Event, lane Lane) error { + ingested = append(ingested, event.ID) + _, err := ledger.RecordSeen(ctx, event, lane) + return err + }, + interval: time.Minute, + } + return walker, &ingested +} + +func TestRepairWalkEntersOneBelowTheLowestMissingID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838505, 17099838502}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838502), testEvent(17099838505)}, + Position: "walk-1", + }}} + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + require.NotEmpty(t, polls.cursors) + assert.Equal(t, "17099838501", polls.cursors[0].Since, + "the feed's since is exclusive, so the walk enters one below the lowest missing id") + assert.Equal(t, []int64{17099838502, 17099838505}, *ingested) + + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838502, 17099838505}, recovered) +} + +// A request crosses up to a thousand ledger rows and serves at most a hundred +// matches; the rows the filters excluded still advanced the cursor. Stopping at +// the first empty page abandons the repair one page short. +func TestRepairWalkFollowsNextThroughAnEmptyPage(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: nil, Position: "walk-1", Next: "https://3.basecampapi.com/2914079/events.json?position=walk-1"}, + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "walk-2"}, + }} + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 2, polls.calls) + assert.Equal(t, []int64{17099838509}, *ingested) + assert.Equal(t, "https://3.basecampapi.com/2914079/events.json?position=walk-1", polls.cursors[1].PageURL) +} + +// A missing `next` means the walk reached its frozen head, not that history +// ended: a page cut short by the safety horizon withholds the link on purpose. +func TestRepairWalkRepeatsAfterAMissingNext(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: nil, Position: "walk-1"}, + {Events: nil, Position: "walk-2"}, + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "walk-3"}, + }} + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 3, polls.calls, "the walk repeats on the repair cadence rather than concluding") + assert.Equal(t, []int64{17099838509}, *ingested) + assert.Equal(t, "walk-2", polls.cursors[2].Position, + "each repeat resumes from the walk's own cursor") +} + +// The walk is seeded from a live id that ran far ahead of the poll lane. Its +// cursor belongs to the loss record and must never reach the feed's checkpoint. +func TestRepairCursorNeverReachesTheFeedCheckpoint(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + key := testKey() + require.NoError(t, ledger.Save(ctx, key, "feed-position-before-the-overflow")) + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838509)}, + Position: "repair-walk-position", + }}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + position, ok, err := ledger.Load(ctx, key) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "feed-position-before-the-overflow", position, + "a checkpoint taken from the repair walk would skip everything inside the safety delay behind it") +} + +func TestRepairWalkSavesItsOwnCursorOnTheLoss(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{{Events: nil, Position: "repair-walk-position"}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + // One pass only: stop the clock past the deadline so the walk gives up. + walker.sleep = func(context.Context, time.Duration) error { + clock.at = clock.at.Add(11 * time.Minute) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered) +} + +func TestIdsStillMissingWhenTheWindowClosesAreUnrecovered(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509, 17099838510}, clock.at, 10*time.Minute) + require.NoError(t, err) + + // The straggler arrives on the fourth repair poll; the other never does. + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Position: "w1"}, + {Position: "w2"}, + {Position: "w3"}, + {Events: []eventfeed.Event{testEvent(17099838510)}, Position: "w4"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.GreaterOrEqual(t, polls.calls, 10, "ten minutes at a sixty-second cadence") + + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838510}, recovered) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered, + "a late straggler is reported, never hidden") +} + +// An unrecovered id the poll lane serves later is resolved by intake like any +// other event — the ledger is one dedupe, not two. +func TestAnUnrecoveredIDIsStillIngestedNormallyLater(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute) + require.NoError(t, err) + _, err = ledger.CloseLoss(ctx, loss.ID, time.Now()) + require.NoError(t, err) + + fresh, err := ledger.RecordSeen(ctx, testEvent(17099838509), LanePoll) + require.NoError(t, err) + assert.True(t, fresh) + + record, ok, err := ledger.Get(ctx, 17099838509) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateSeen, record.State) +} + +func TestRepairWalkBelowTheEpochEndsWithAGap(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{100, 101}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + }}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 1, polls.calls, "no number of repeats will serve history below the epoch") + + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Equal(t, GapEpoch, gaps[0].Class) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{100, 101}, unrecovered) +} + +func TestATransientRepairPollIsRetriedNotGivenUpOn(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: time.Second}, nil}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "w2"}}, + } + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, []int64{17099838509}, *ingested, + "a throttled walk delays the repair; it does not condemn the ids") + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered) +} diff --git a/internal/connector/shutdown.go b/internal/connector/shutdown.go new file mode 100644 index 000000000..1e9299256 --- /dev/null +++ b/internal/connector/shutdown.go @@ -0,0 +1,40 @@ +package connector + +import ( + "os" + "os/signal" + "syscall" +) + +// Exit codes for a signaled shutdown. 128+n is the shell's convention, and a +// supervisor reading the connector's exit status needs to tell an interrupt +// from a termination from a crash. +const ( + // ExitInterrupted is 128 + SIGINT. + ExitInterrupted = 130 + // ExitTerminated is 128 + SIGTERM. + ExitTerminated = 143 +) + +// ExitCodeForSignal maps a shutdown signal to the exit code the connector +// leaves behind. An unknown signal reports 1: the process ended, and pretending +// it ended cleanly would be a lie to whatever restarts it. +func ExitCodeForSignal(sig os.Signal) int { + switch sig { + case os.Interrupt, syscall.SIGINT: + return ExitInterrupted + case syscall.SIGTERM: + return ExitTerminated + default: + return 1 + } +} + +// NotifyShutdown returns a channel carrying the first shutdown signal, and a +// stop function. Separated from the exit-code mapping so the mapping can be +// tested without sending real signals to the test binary. +func NotifyShutdown() (<-chan os.Signal, func()) { + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + return ch, func() { signal.Stop(ch) } +} diff --git a/internal/connector/shutdown_test.go b/internal/connector/shutdown_test.go new file mode 100644 index 000000000..90ab7662c --- /dev/null +++ b/internal/connector/shutdown_test.go @@ -0,0 +1,17 @@ +package connector + +import ( + "os" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExitCodeForSignal(t *testing.T) { + assert.Equal(t, ExitInterrupted, ExitCodeForSignal(os.Interrupt)) + assert.Equal(t, ExitInterrupted, ExitCodeForSignal(syscall.SIGINT)) + assert.Equal(t, ExitTerminated, ExitCodeForSignal(syscall.SIGTERM)) + assert.Equal(t, 1, ExitCodeForSignal(syscall.SIGHUP), + "an unrecognized end is not a clean one") +} diff --git a/internal/mcpserver/model/behavior-model.json b/internal/mcpserver/model/behavior-model.json index 8913ae7e7..775b63ad8 100644 --- a/internal/mcpserver/model/behavior-model.json +++ b/internal/mcpserver/model/behavior-model.json @@ -294,6 +294,18 @@ ] } }, + "CreateStreamTicket": { + "idempotent": true, + "retry": { + "max": 3, + "base_delay_ms": 1000, + "backoff": "exponential", + "retry_on": [ + 429, + 503 + ] + } + }, "CreateTemplate": { "retry": { "max": 2, From e4852c2edf788b608c4d83cf5c2f73d0d0586559 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 15:04:08 +0200 Subject: [PATCH 02/49] Keep intake's recovery paths from swallowing, stalling or skipping Review of the first head found places where a recovery path did the wrong quiet thing. Each fix has a test that failed on that head. A refused stored position right after a restart re-entered at the present: the SDK's reset cursor lives in memory and starts at zero. The connection is now ended before that re-entry polls and remade after the last poll-served id the ledger holds. A filter change with no position of its own takes the same id, tracked per consumer lineage rather than per digest. --since is an entry, applied to the first connection only. A terminal feed waited out the membership ticker and any open repair walk before returning its error; both now end with the connection or the run. A reconnect request cancels the connection at once, after the event that raised it is queued, instead of latching until the next membership tick and turning the next terminal error into a silent reconnect. The repair walk treated every failed poll as transient. A retention 410 is now recorded as its own class and not retried; an epoch 410 condemns only the ids behind the epoch and follows the resume for the rest; a refused walk cursor re-enters from its explicit id; a failure no repeat fixes ends this start's walk and leaves the loss open for the next. Also: a late arrival clears an unrecovered id, the pointer line carries the lane intake was given, a failed first membership read no longer disables change detection, the queue has no Close to race an offer, and the migration reads its version inside its own transaction. --- internal/connector/intake.go | 163 +++++++++-- internal/connector/ledger.go | 28 +- internal/connector/ledger_checkpoint.go | 32 ++- internal/connector/ledger_events.go | 8 +- internal/connector/ledger_recovery.go | 18 +- internal/connector/lock.go | 6 +- internal/connector/queue.go | 25 +- internal/connector/repair.go | 120 ++++++-- internal/connector/review_fixes_test.go | 362 ++++++++++++++++++++++++ 9 files changed, 681 insertions(+), 81 deletions(-) create mode 100644 internal/connector/review_fixes_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 30f0c707f..02a249474 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -102,6 +102,15 @@ type Intake struct { pollCandidates []int64 snapshot map[int64]bool reconnect chan struct{} + // cancelRun ends the current connection; nil between connections. + cancelRun context.CancelFunc + // promotedThisRun is whether this connection has recorded a poll-served + // id. Until it has, the package's own reset cursor is zero, and a refused + // position would send it to the present. + promotedThisRun bool + // reentryAfter is the safe re-entry the next connection takes after a + // refused stored position. + reentryAfter int64 repairs sync.WaitGroup // lifetime is Run's context. Repair walks are bound to it rather than to @@ -177,51 +186,89 @@ func (in *Intake) CheckpointKey() eventfeed.CheckpointKey { return in.key } // Run consumes the feed until ctx is canceled or the feed terminates. // -// It reconnects on its own only for membership: everything else the feed can -// recover from, it recovers from inside the package. +// It reconnects on its own for two reasons only — membership, and a stored +// position refused before this run had a safe re-entry of its own. Everything +// else the feed can recover from, it recovers from inside the package. func (in *Intake) Run(ctx context.Context) error { - in.lifetime = ctx - if err := in.resumeReconciliation(ctx); err != nil { + // Repairs get a child lifetime that ends when Run does, whatever the + // reason. A repair is off the delivery path: a terminal feed must not wait + // out its sixty-second cadence, and an unfinished repair resumes on the + // next start from the loss record. + repairCtx, stopRepairs := context.WithCancel(ctx) + defer in.repairs.Wait() + defer stopRepairs() + in.lifetime = repairCtx + + if err := in.resumeReconciliation(repairCtx); err != nil { return err } - defer in.repairs.Wait() + since := in.opts.SinceEventID for { - err := in.runOnce(ctx) + err := in.runOnce(ctx, since) + // --since is an entry, not a standing instruction: a reconnect + // resumes from what this run stored. + since = 0 if ctx.Err() != nil { return ctx.Err() } - if !errors.Is(err, errReconnect) { + switch { + case errors.Is(err, errReconnect): + in.log.Info("reconnecting the feed") + default: return err } - in.log.Info("reconnecting the feed", "reason", "membership changed") } } // errReconnect asks the supervisor for a fresh connection. It is not a -// failure: the cable's bucket snapshot is taken at subscribe, so a project -// granted afterwards needs a new connection to be heard on the live lane. +// failure. var errReconnect = errors.New("connector: reconnect the feed") -func (in *Intake) runOnce(ctx context.Context) error { +func (in *Intake) runOnce(ctx context.Context, since int64) error { runCtx, cancel := context.WithCancel(ctx) defer cancel() + in.mu.Lock() + in.cancelRun = cancel + in.promotedThisRun = false + reentry := in.reentryAfter + in.reentryAfter = 0 + in.mu.Unlock() + defer func() { + in.mu.Lock() + in.cancelRun = nil + in.mu.Unlock() + }() + in.takeSnapshot(runCtx) - position, hadPosition, err := in.ledger.Load(runCtx, in.key) + _, hadPosition, err := in.ledger.Load(runCtx, in.key) + if err != nil { + return err + } + lineageServed, err := in.ledger.LineagePollServedID(runCtx, in.key) if err != nil { return err } - _ = position start := eventfeed.StartResume() switch { - case in.opts.SinceEventID > 0: - start = eventfeed.StartAfter(in.opts.SinceEventID) - in.log.Info("entering the feed after an explicit event id", "since", in.opts.SinceEventID) + case since > 0: + start = eventfeed.StartAfter(since) + in.log.Info("entering the feed after an explicit event id", "since", since) + case reentry > 0: + start = eventfeed.StartAfter(reentry) + in.log.Warn("the stored position was refused; re-entering after the last poll-served id", "since", reentry) case hadPosition: in.log.Info("resuming the feed from the stored position", "filter_key", in.key.FilterKey) + case lineageServed > 0: + // A filter change: this digest has no position, but the consumer's + // poll lane had reached this id under another. Entering at the + // present would skip everything since. + start = eventfeed.StartAfter(lineageServed) + in.log.Warn("no position for this filter set; re-entering after the last poll-served id under the previous one", + "since", lineageServed, "filter_key", in.key.FilterKey) default: // Said out loud because it is a real loss of history, not a neutral // default: everything committed before this moment is never served. @@ -246,13 +293,16 @@ func (in *Intake) runOnce(ctx context.Context) error { if err != nil { return fmt.Errorf("connector: build feed: %w", err) } + // Deferred in this order so they run in the reverse: the connection is + // canceled first, which is what lets the membership watcher and the feed + // return, and only then are they waited on. defer func() { _ = feed.Close() feed.Wait() }() - - stopMembership := in.watchMembership(runCtx, cancel) + stopMembership := in.watchMembership(runCtx) defer stopMembership() + defer cancel() var feedErr error for event, err := range feed.Events(runCtx) { @@ -261,6 +311,9 @@ func (in *Intake) runOnce(ctx context.Context) error { break } if err := in.ingest(runCtx, event, LaneOf(event)); err != nil { + if in.reconnectRequested() { + return errReconnect + } return err } } @@ -289,11 +342,17 @@ func (in *Intake) ingest(ctx context.Context, event eventfeed.Event, lane Lane) return nil } - in.noteBucket(event.BucketID) - if err := in.pointer.write(event); err != nil { + if err := in.pointer.write(event, lane); err != nil { + return err + } + if err := in.queue.Offer(ctx, event.ID); err != nil { return err } - return in.queue.Offer(ctx, event.ID) + // Only once the id is handed over: the reconnect cancels the connection + // this event arrived on, and the event must not be stranded between the + // ledger and the queue by it. + in.noteBucket(event.BucketID) + return nil } // LaneOf says which lane served an event. @@ -342,7 +401,11 @@ func (in *Intake) confirmPollServed(ctx context.Context) { } if err := in.ledger.NotePollServed(ctx, in.key, highest); err != nil { in.log.Error("could not record the last poll-served id", "error", err) + return } + in.mu.Lock() + in.promotedThisRun = true + in.mu.Unlock() } func (in *Intake) observer(ctx context.Context) eventfeed.Observer { @@ -373,6 +436,7 @@ func (in *Intake) observer(ctx context.Context) eventfeed.Observer { }, PositionRejected: func(kind eventfeed.PollErrorKind) { in.log.Warn("feed rejected the held position", "kind", kind.String()) + in.onPositionRejected(ctx) }, FilterConflict: func(positionDigest, filtersDigest string) { in.log.Warn("feed position was minted for a different filter set", @@ -560,12 +624,40 @@ func (in *Intake) noteBucket(bucketID int64) { in.requestReconnect() } +// onPositionRejected handles a refused position on a connection that has not +// yet recorded a poll-served id of its own. +// +// The package re-enters from an in-memory id that starts at zero each run, so +// right after a restart it would take the present and skip everything since +// the refused position. The ledger holds the id the poll lane had reached, so +// the connection is ended before that re-entry polls and remade after it. +func (in *Intake) onPositionRejected(ctx context.Context) { + in.mu.Lock() + promoted := in.promotedThisRun + in.mu.Unlock() + if promoted { + // The package's own reset cursor is this run's poll-served id, which + // is at least what the ledger holds. + return + } + served, err := in.ledger.LineagePollServedID(ctx, in.key) + if err != nil || served == 0 { + return + } + in.mu.Lock() + in.reentryAfter = served + in.mu.Unlock() + in.requestReconnect() +} + // watchMembership re-reads the agent's projects on a timer and asks for a -// reconnect when the set changes. It returns a stop function. -func (in *Intake) watchMembership(ctx context.Context, cancel context.CancelFunc) func() { +// reconnect when the set changes. The returned stop function ends the watcher +// and waits for it, and is safe to call whatever state ctx is in. +func (in *Intake) watchMembership(ctx context.Context) func() { if in.opts.Membership == nil { return func() {} } + ctx, stop := context.WithCancel(ctx) done := make(chan struct{}) go func() { defer close(done) @@ -586,19 +678,28 @@ func (in *Intake) watchMembership(ctx context.Context, cancel context.CancelFunc } if in.membershipChanged(buckets) { in.requestReconnect() - cancel() return } } } }() - return func() { <-done } + return func() { + stop() + <-done + } } +// membershipChanged compares a fresh read with the snapshot. With no snapshot +// — the read at subscribe failed — the fresh read becomes the baseline; +// otherwise a failed first read would disable change detection for good. func (in *Intake) membershipChanged(buckets []int64) bool { in.mu.Lock() defer in.mu.Unlock() if in.snapshot == nil { + in.snapshot = make(map[int64]bool, len(buckets)) + for _, id := range buckets { + in.snapshot[id] = true + } return false } if len(buckets) != len(in.snapshot) { @@ -612,11 +713,19 @@ func (in *Intake) membershipChanged(buckets []int64) bool { return false } +// requestReconnect marks a reconnect due and ends the current connection now, +// rather than whenever the feed next yields. func (in *Intake) requestReconnect() { select { case in.reconnect <- struct{}{}: default: } + in.mu.Lock() + cancel := in.cancelRun + in.mu.Unlock() + if cancel != nil { + cancel() + } } func (in *Intake) reconnectRequested() bool { @@ -655,7 +764,7 @@ type Pointer struct { State string `json:"state"` } -func (p *pointerWriter) write(event eventfeed.Event) error { +func (p *pointerWriter) write(event eventfeed.Event, lane Lane) error { if p.w == nil { return nil } @@ -669,7 +778,7 @@ func (p *pointerWriter) write(event eventfeed.Event) error { PerformedByID: event.PerformedByID, RecordingID: event.RecordingID, CreatedAt: event.CreatedAt.UTC().Format(time.RFC3339), - Lane: LaneOf(event), + Lane: lane, State: string(StateSeen), }) if err != nil { diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index 579dbc210..fc3dcadbc 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" _ "modernc.org/sqlite" // database/sql driver "sqlite", pure Go: no cgo on any of the five release targets. @@ -69,6 +70,11 @@ func OpenLedger(path string) (*Ledger, error) { if path == "" { return nil, errors.New("connector: ledger path is required") } + if strings.ContainsAny(path, "?#%") { + // The driver reads the path as a URI; these would be taken as its + // query, fragment or an escape, and open some other file. + return nil, fmt.Errorf("connector: ledger path %q contains a character the SQLite URI cannot carry (?, # or %%)", path) + } if dir := filepath.Dir(path); dir != "" && dir != "." { if err := os.MkdirAll(dir, 0o700); err != nil { return nil, fmt.Errorf("connector: create ledger directory: %w", err) @@ -127,10 +133,12 @@ CREATE INDEX events_state_id ON events (state, id); CREATE TABLE checkpoints ( flat_key TEXT PRIMARY KEY, + lineage TEXT NOT NULL, position TEXT NOT NULL, last_poll_served_id INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL ); +CREATE INDEX checkpoints_lineage ON checkpoints (lineage); CREATE TABLE losses ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -169,17 +177,25 @@ func (l *Ledger) migrate(ctx context.Context) error { return fmt.Errorf("connector: create migration table: %w", err) } - var applied int - if err := l.db.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&applied); err != nil { - return fmt.Errorf("connector: read schema version: %w", err) - } - - for i := applied; i < len(migrations); i++ { + for i := range migrations { version := i + 1 + // The version is read inside the migration's own transaction, which + // takes the write lock as it opens: two processes opening a fresh + // ledger at once — `status` beside a starting connector — must not + // both decide migration 1 is theirs to apply. tx, err := l.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("connector: begin migration %d: %w", version, err) } + var applied int + if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(&applied); err != nil { + _ = tx.Rollback() + return fmt.Errorf("connector: read schema version: %w", err) + } + if applied >= version { + _ = tx.Rollback() + continue + } if _, err := tx.ExecContext(ctx, migrations[i]); err != nil { _ = tx.Rollback() return fmt.Errorf("connector: apply migration %d: %w", version, err) diff --git a/internal/connector/ledger_checkpoint.go b/internal/connector/ledger_checkpoint.go index 25340ca97..7650b3040 100644 --- a/internal/connector/ledger_checkpoint.go +++ b/internal/connector/ledger_checkpoint.go @@ -42,9 +42,9 @@ func (l *Ledger) Load(ctx context.Context, key eventfeed.CheckpointKey) (string, // position taken from one would skip everything inside that window. func (l *Ledger) Save(ctx context.Context, key eventfeed.CheckpointKey, position string) error { _, err := l.db.ExecContext(ctx, ` -INSERT INTO checkpoints (flat_key, position, updated_at) VALUES (?, ?, ?) +INSERT INTO checkpoints (flat_key, lineage, position, updated_at) VALUES (?, ?, ?, ?) ON CONFLICT (flat_key) DO UPDATE SET position = excluded.position, updated_at = excluded.updated_at`, - key.FlatKey(), position, l.timestamp()) + key.FlatKey(), lineageOf(key), position, l.timestamp()) if err != nil { return fmt.Errorf("connector: save checkpoint: %w", err) } @@ -64,11 +64,11 @@ ON CONFLICT (flat_key) DO UPDATE SET position = excluded.position, updated_at = // It only ever moves forward. func (l *Ledger) NotePollServed(ctx context.Context, key eventfeed.CheckpointKey, eventID int64) error { _, err := l.db.ExecContext(ctx, ` -INSERT INTO checkpoints (flat_key, position, last_poll_served_id, updated_at) VALUES (?, '', ?, ?) +INSERT INTO checkpoints (flat_key, lineage, position, last_poll_served_id, updated_at) VALUES (?, ?, '', ?, ?) ON CONFLICT (flat_key) DO UPDATE SET last_poll_served_id = MAX(checkpoints.last_poll_served_id, excluded.last_poll_served_id), updated_at = excluded.updated_at`, - key.FlatKey(), eventID, l.timestamp()) + key.FlatKey(), lineageOf(key), eventID, l.timestamp()) if err != nil { return fmt.Errorf("connector: note poll-served id: %w", err) } @@ -90,6 +90,30 @@ func (l *Ledger) LastPollServedID(ctx context.Context, key eventfeed.CheckpointK return id, nil } +// LineagePollServedID returns the highest id the poll lane has served to this +// consumer under ANY filter set: same origin, account and namespace. +// +// It is the re-entry for a filter change. The new digest holds no position of +// its own, and entering it at the present would skip everything committed +// since the old digest's walk stopped. Events a narrower old filter excluded +// before this id are not recovered by widening — the spec says so, and so does +// this comment — but nothing after it is skipped. +func (l *Ledger) LineagePollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) { + var id int64 + err := l.db.QueryRowContext(ctx, + `SELECT COALESCE(MAX(last_poll_served_id), 0) FROM checkpoints WHERE lineage = ?`, lineageOf(key)).Scan(&id) + if err != nil { + return 0, fmt.Errorf("connector: read lineage poll-served id: %w", err) + } + return id, nil +} + +// lineageOf is the checkpoint identity without its filter digest. +func lineageOf(key eventfeed.CheckpointKey) string { + key.FilterKey = "" + return key.FlatKey() +} + // ForgetPosition drops the held position for key while keeping the last // poll-served id — the 409 path. The server refused the position because it // was minted for a different filter set; the id the poll lane had reached is diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go index 448e0d5ae..72d61d01d 100644 --- a/internal/connector/ledger_events.go +++ b/internal/connector/ledger_events.go @@ -55,7 +55,7 @@ func (r Record) EffectivePerformer() int64 { // its pointer payload is dropped, so an explicit replay of old history can // never turn a finished event back into a new task. // -// It also resolves the id against any open loss: an event the repair walk — or +// It also resolves the id against any loss, open or closed: an event the repair walk — or // the ordinary poll lane, later — serves is one the overflow did not cost us. func (l *Ledger) RecordSeen(ctx context.Context, ev eventfeed.Event, lane Lane) (bool, error) { tx, err := l.db.BeginTx(ctx, nil) @@ -84,9 +84,11 @@ ON CONFLICT (id) DO NOTHING`, return false, fmt.Errorf("connector: record seen %d: %w", ev.ID, err) } + // Missing or already given up on: an id intake has received is recovered, + // however late it came, and status must stop reporting it. if _, err := tx.ExecContext(ctx, - `UPDATE loss_ids SET state = ? WHERE event_id = ? AND state = ?`, - string(LossRecovered), ev.ID, string(LossMissing)); err != nil { + `UPDATE loss_ids SET state = ? WHERE event_id = ? AND state IN (?, ?)`, + string(LossRecovered), ev.ID, string(LossMissing), string(LossUnrecovered)); err != nil { return false, fmt.Errorf("connector: resolve loss for %d: %w", ev.ID, err) } diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go index 9d7802119..b96653753 100644 --- a/internal/connector/ledger_recovery.go +++ b/internal/connector/ledger_recovery.go @@ -130,7 +130,7 @@ func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Ti loss := Loss{ DetectedAt: now.UTC(), - DroppedCount: len(droppedIDs), + DroppedCount: len(states), // One below the lowest missing id, so the walk's first page can serve // it: the feed's since is exclusive. RepairSince: lowestMissing - 1, @@ -278,6 +278,22 @@ func (l *Ledger) CloseLoss(ctx context.Context, lossID int64, now time.Time) (in return int(unrecovered), nil } +// MarkUnrecoveredThrough gives up on a loss's missing ids at or below id — the +// ones a 410's epoch has fenced off — and leaves the rest missing. +func (l *Ledger) MarkUnrecoveredThrough(ctx context.Context, lossID, id int64) (int, error) { + res, err := l.db.ExecContext(ctx, + `UPDATE loss_ids SET state = ? WHERE loss_id = ? AND state = ? AND event_id <= ?`, + string(LossUnrecovered), lossID, string(LossMissing), id) + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered below the epoch: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("connector: mark unrecovered below the epoch: %w", err) + } + return int(n), nil +} + // UnrecoveredIDs returns every id the connector has given up on, across all // losses. `basecamp connect status` and `doctor` show these: an unrecovered id // is documented, not hidden, and the poll lane may still serve it later. diff --git a/internal/connector/lock.go b/internal/connector/lock.go index 1ca43897b..c12ba2262 100644 --- a/internal/connector/lock.go +++ b/internal/connector/lock.go @@ -43,8 +43,10 @@ type instanceHolder struct { // AcquireInstanceLock takes the lock for one account and agent, or refuses. func AcquireInstanceLock(dir, accountID string, agentPersonID int64, now time.Time) (*InstanceLock, error) { - if accountID == "" || agentPersonID <= 0 { - return nil, errors.New("connector: the instance lock needs an account id and an agent person id") + if _, err := strconv.ParseUint(accountID, 10, 64); err != nil || agentPersonID <= 0 { + // The account id becomes part of a file name, so it is held to what an + // account id is: digits. + return nil, errors.New("connector: the instance lock needs a numeric account id and an agent person id") } if err := os.MkdirAll(dir, 0o700); err != nil { return nil, fmt.Errorf("connector: create state directory: %w", err) diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 6e5577fb8..5afc08a1d 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -3,7 +3,6 @@ package connector import ( "context" "errors" - "sync" "sync/atomic" ) @@ -21,18 +20,14 @@ const ( DefaultBacklogPause = 10_000 ) -// ErrQueueClosed reports work offered to, or taken from, a closed queue. -var ErrQueueClosed = errors.New("connector: intake queue is closed") - // Queue is the seam between intake and admission: intake writes a pointer and // hands over an id, admission reads it when it gets there. Two queues with // visible depth rather than one pipeline, so a busy dispatcher can never stall // the socket — and so the place where work is piling up is the place the depth // is showing. type Queue struct { - ids chan int64 - warnAt int - closeOne sync.Once + ids chan int64 + warnAt int warned atomic.Bool paused atomic.Bool @@ -92,14 +87,14 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { } } -// Take returns the next id, waiting for one. It reports ErrQueueClosed once -// the queue is closed and drained. +// Take returns the next id, waiting for one until ctx ends. +// +// There is deliberately no Close. Shutdown is the context: a closed channel +// would turn every offer racing the close into a panic, and nothing here needs +// a drained-and-done signal that cancellation does not already give. func (q *Queue) Take(ctx context.Context) (int64, error) { select { - case id, ok := <-q.ids: - if !ok { - return 0, ErrQueueClosed - } + case id := <-q.ids: q.noteDepth() return id, nil case <-ctx.Done(): @@ -114,10 +109,6 @@ func (q *Queue) Depth() int { return len(q.ids) } // say whether the feed is being consumed. func (q *Queue) Paused() bool { return q.paused.Load() } -// Close stops the queue. Takers drain what is already queued and then see -// ErrQueueClosed. -func (q *Queue) Close() { q.closeOne.Do(func() { close(q.ids) }) } - // noteDepth fires the warning edges. It is edge-triggered, not level: a // backlog that sits above the threshold for an hour is one warning, and the // recovery is the other half of the pair, so a warning is never left standing diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 4c07236c3..a3767fe49 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -108,10 +108,19 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { } last := loss.RepairCursor + reentered := false for { page, err := w.polls.Poll(ctx, cursor, w.filters) if err != nil { - return last, w.pollFailure(ctx, loss, err) + next, err := w.pollFailure(ctx, loss, cursor, err, &reentered) + if err != nil || next == nil { + return last, err + } + cursor = *next + if cursor.PageURL == "" && cursor.Position == "" { + last = "" + } + continue } for _, event := range page.Events { @@ -143,17 +152,49 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { } } -// errReconciliationEnded reports that the walk itself closed the loss, so the -// caller stops rather than looping back to find nothing missing and calling -// that a full recovery. +// errReconciliationEnded reports that the walk itself settled the loss's fate +// for this start — closed it, or left it open for the next — so the caller +// stops rather than looping back to find nothing missing and calling that a +// full recovery. var errReconciliationEnded = errors.New("connector: reconciliation ended inside the repair walk") -// pollFailure decides what one failed repair poll means. Only a 410 ends the -// reconciliation: the ids below the epoch are gone, and no number of repeats -// will serve them. -func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, err error) error { +// pollFailure decides what one failed repair poll means. It returns the cursor +// to continue this pass at, nil with no error to end the pass and wait for the +// next repair poll, or an error. +func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor eventfeed.Cursor, err error, reentered *bool) (*eventfeed.Cursor, error) { + var retention *InboxRetentionGoneError + if errors.As(err, &retention) { + // The inbox lane's 410 on the account lane: the same foreign loss the + // feed surfaces, recorded as its own class and never resumed as if it + // were the epoch's. Nothing on this lane will serve the ids. + if _, recordErr := w.ledger.RecordGap(ctx, Gap{ + DetectedAt: w.now(), + Class: GapRetention, + EntryClass: EntryUnknown, + Note: "a retention 410 was served to the repair walk on the account lane; not resumed", + }); recordErr != nil { + return nil, recordErr + } + unrecovered, closeErr := w.ledger.CloseLoss(ctx, loss.ID, w.now()) + if closeErr != nil { + return nil, closeErr + } + w.log.Error("the repair walk was answered with the inbox lane's 410; not resumed", + "loss_id", loss.ID, "unrecovered", unrecovered) + return nil, errReconciliationEnded + } + var pollErr *eventfeed.PollError - if errors.As(err, &pollErr) && pollErr.Kind == eventfeed.PollGone { + if !errors.As(err, &pollErr) { + w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "error", err) + return nil, nil + } + + switch pollErr.Kind { + case eventfeed.PollGone: + // History up to the epoch is gone, and the ids there with it. Ids + // above the epoch are still servable, so the resume is followed as + // served rather than condemning them too. epoch := pollErr.EpochAfterID if _, recordErr := w.ledger.RecordGap(ctx, Gap{ DetectedAt: w.now(), @@ -162,22 +203,59 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, err error) e EntryClass: entryClassOf(pollErr.ResumeURL), Note: "the repair walk was seeded below the feed's epoch", }); recordErr != nil { - return recordErr + return nil, recordErr } - unrecovered, closeErr := w.ledger.CloseLoss(ctx, loss.ID, w.now()) - if closeErr != nil { - return closeErr + behind, markErr := w.ledger.MarkUnrecoveredThrough(ctx, loss.ID, epoch) + if markErr != nil { + return nil, markErr + } + w.log.Error("the repair walk fell below the feed's epoch; the dropped events behind it are gone", + "loss_id", loss.ID, "epoch_after_id", epoch, "unrecovered", behind) + stillMissing, missErr := w.ledger.MissingIDs(ctx, loss.ID, LossMissing) + if missErr != nil { + return nil, missErr + } + if len(stillMissing) == 0 || pollErr.ResumeURL == "" { + // Every id was behind the fence, or there is no resume to follow: + // no repeat will serve anything more. + if _, closeErr := w.ledger.CloseLoss(ctx, loss.ID, w.now()); closeErr != nil { + return nil, closeErr + } + return nil, errReconciliationEnded } - w.log.Error("the repair walk fell below the feed's epoch; the dropped events are behind it", - "loss_id", loss.ID, "epoch_after_id", epoch, "unrecovered", unrecovered) - return errReconciliationEnded + return &eventfeed.Cursor{PageURL: pollErr.ResumeURL}, nil + + case eventfeed.PollFilterChanged, eventfeed.PollPositionInvalid: + // The walk's own cursor was refused — minted under another filter + // set, or no longer honored. Its explicit id is still good. Once per + // pass: a refusal of the explicit id too is left to the cadence. + if *reentered || (cursor.Position == "" && cursor.PageURL == "") { + w.log.Warn("a repair poll was refused; retrying on the repair cadence", "loss_id", loss.ID, "error", err) + return nil, nil + } + *reentered = true + loss.RepairCursor = "" + if saveErr := w.ledger.SaveRepairCursor(ctx, loss.ID, ""); saveErr != nil { + return nil, saveErr + } + return &eventfeed.Cursor{Since: strconv.FormatInt(loss.RepairSince, 10)}, nil + + case eventfeed.PollTransient, eventfeed.PollThrottled, eventfeed.PollUnauthorized: + // A reason to try again on the next repair poll, not a reason to call + // the ids unrecovered. A slow or throttled walk delays nothing else. + w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "error", err) + return nil, nil + + case eventfeed.PollFilterInvalid, eventfeed.PollRedirectRefused, eventfeed.PollUnrecoverable: } - // Anything else — a transient, a throttle, an unauthorized — is a reason - // to try again on the next repair poll, not a reason to call the ids - // unrecovered. A slow or throttled walk delays nothing else. - w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "error", err) - return nil + // Refused redirects, invalid filters, undifferentiated 400s, anything + // unrecoverable: no repeat inside this window will change the answer. + // The loss stays open, so the next start — perhaps after the cause is + // fixed — tries again instead of finding the ids already condemned. + w.log.Error("a repair poll failed in a way retrying will not fix; the loss stays open for the next start", + "loss_id", loss.ID, "error", err) + return nil, errReconciliationEnded } func (w *repairWalker) wait(ctx context.Context, d time.Duration) error { diff --git a/internal/connector/review_fixes_test.go b/internal/connector/review_fixes_test.go new file mode 100644 index 000000000..a21bda413 --- /dev/null +++ b/internal/connector/review_fixes_test.go @@ -0,0 +1,362 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" +) + +func TestALateArrivalClearsAnUnrecoveredID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute) + require.NoError(t, err) + _, err = ledger.CloseLoss(ctx, loss.ID, time.Now()) + require.NoError(t, err) + + _, err = ledger.RecordSeen(ctx, testEvent(17099838509), LanePoll) + require.NoError(t, err) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered, "an id intake received is not unrecovered, however late it came") +} + +func TestPointerLineReportsTheLaneIntakeWasGiven(t *testing.T) { + var pointers bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &pointers) + require.NoError(t, intake.ingest(context.Background(), testEvent(1), LaneRepair)) + + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &pointer)) + assert.Equal(t, LaneRepair, pointer.Lane, "the pointer line and the ledger must agree on the lane") +} + +func TestMembershipBaselineIsTakenFromTheFirstSuccessfulRead(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + + // The snapshot at subscribe failed, so there is none. + assert.False(t, intake.membershipChanged([]int64{1, 2}), "the first successful read is a baseline, not a change") + assert.True(t, intake.membershipChanged([]int64{1, 2, 3}), "and a later grant is then seen") +} + +type flakyMembership struct { + mu sync.Mutex + buckets []int64 + err error +} + +func (m *flakyMembership) Buckets(context.Context) ([]int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.buckets, m.err +} + +func newFeedIntake(t *testing.T, ledger *Ledger, opts Options) (*Intake, *feedtest.Transport, *feedtest.Minter, *feedtest.Polls) { + t.Helper() + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + transport := feedtest.NewTransport() + minter := feedtest.NewMinter() + polls := feedtest.NewPolls() + opts.Origin = "https://3.basecampapi.com" + opts.AccountID = "2914079" + opts.ConsumerNamespace = "connector-test" + opts.Ledger = ledger + opts.Queue = queue + opts.Minter = minter + opts.Polls = polls + opts.Transport = transport + intake, err := New(opts) + require.NoError(t, err) + return intake, transport, minter, polls +} + +func ticket() eventfeed.StreamTicket { + return eventfeed.StreamTicket{Ticket: "t", ExpiresIn: 120, URL: "wss://cable.basecamp.com/cable?ticket=t"} +} + +func runInBackground(ctx context.Context, t *testing.T, intake *Intake) chan error { + t.Helper() + done := make(chan error, 1) + go func() { done <- intake.Run(ctx) }() + return done +} + +func awaitReturn(t *testing.T, done chan error, why string) error { + t.Helper() + select { + case err := <-done: + return err + case <-time.After(5 * time.Second): + t.Fatal(why) + return nil + } +} + +func TestATerminalFeedIsNotHeldOpenByTheMembershipWatcher(t *testing.T) { + ledger := newTestLedger(t) + intake, _, minter, _ := newFeedIntake(t, ledger, Options{ + Membership: &flakyMembership{buckets: []int64{48699913}}, + MembershipInterval: time.Hour, + }) + minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: errors.New("gone")}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := awaitReturn(t, runInBackground(ctx, t, intake), + "a feed that terminated must return its error, not wait out the membership ticker") + assert.Error(t, err) +} + +func TestATerminalFeedIsNotHeldOpenByARepairWalk(t *testing.T) { + ledger := newTestLedger(t) + _, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, time.Now(), time.Hour) + require.NoError(t, err) + + intake, _, minter, _ := newFeedIntake(t, ledger, Options{RepairInterval: time.Hour}) + minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: errors.New("gone")}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err = awaitReturn(t, runInBackground(ctx, t, intake), + "a repair walk is off the delivery path and must not delay the supervisor by its window") + assert.Error(t, err) + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + assert.Len(t, open, 1, "the loss stays open for the next start") +} + +func TestAnEventFromAnUnsnapshottedProjectReconnectsPromptly(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: &flakyMembership{buckets: []int64{1}}, + MembershipInterval: time.Hour, + }) + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838500)}, Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return len(transport.Dials()) >= 2 }, 5*time.Second, 10*time.Millisecond, + "the poll lane served project 48699913, which the live subscription does not hold; the reconnect is due now, not at the next membership tick") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// After a restart the SDK's own reset cursor starts at zero, so a rejected +// stored position would re-enter at the present. The ledger holds the safe +// re-entry; intake must use it. +func TestARejectedStoredPositionReentersAtThePersistedPollServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + key := intake.CheckpointKey() + require.NoError(t, ledger.Save(ctx, key, "position-the-server-will-refuse")) + require.NoError(t, ledger.NotePollServed(ctx, key, 17099838500)) + + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "p2"}) + polls.ScriptPage(eventfeed.PollPage{Position: "p3"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + // Whatever the connector does next — re-enter on this connection or make + // a new one — the second poll is the re-entry. A new connection is + // answered as it appears. + answered := 1 + var reentry eventfeed.Cursor + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + calls := polls.Calls() + if len(calls) < 2 { + return false + } + reentry = calls[1].Cursor + return true + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "17099838500", reentry.Since, + "re-entering at the present skips everything between the refused position and now") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A filter change re-enters under the new digest at the last poll-served id, +// which is tracked apart from any one digest's position. +func TestAFilterChangeReentersAtTheLastPollServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Filters: eventfeed.Filters{Types: []string{"comment.created"}}, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + old := intake.CheckpointKey() + old.FilterKey = eventfeed.Filters{}.FilterKey() + require.NoError(t, ledger.Save(ctx, old, "old-digest-position")) + require.NoError(t, ledger.NotePollServed(ctx, old, 17099838500)) + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "17099838500", polls.Calls()[0].Cursor.Since, + "a new filter set with no position of its own must not enter at the present") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// answerSubscription greets a connection and confirms its subscription without +// failing the test, so it can run inside a polling condition. +func answerSubscription(conn *feedtest.Conn) { + conn.Serve([]byte(`{"type":"welcome"}`)) + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if writes := conn.Writes(); len(writes) > 0 { + var command struct { + Identifier string `json:"identifier"` + } + if json.Unmarshal(writes[0], &command) == nil && command.Identifier != "" { + frame, _ := json.Marshal(map[string]string{"type": "confirm_subscription", "identifier": command.Identifier}) + conn.Serve(frame) + return + } + } + time.Sleep(5 * time.Millisecond) + } +} + +// A retention 410 reaching the repair walk is the same foreign loss it is on +// the feed: recorded as its own class and not retried as if it were transient. +func TestARetention410InTheRepairWalkIsRecordedNotRetried(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + adapter := newTestAdapter(t, &fakeFeedClient{err: retentionGone()}) + walker, _ := newTestWalker(t, ledger, adapter, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + require.Len(t, gaps, 1, "the retention 410 is a fact about the feed, recorded once") + assert.Equal(t, GapRetention, gaps[0].Class) + assert.Nil(t, gaps[0].EpochAfterID) +} + +// A 410 in the walk fences off the ids below the epoch. Ids above it are still +// servable from the resume, and giving up on them too would lose them for +// nothing. +func TestAnEpoch410InTheRepairWalkOnlyCondemnsTheIDsBehindIt(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150", + }}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(200)}, Position: "after-epoch"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + require.GreaterOrEqual(t, len(polls.cursors), 2) + assert.Equal(t, "https://3.basecampapi.com/2914079/events.json?since=150", polls.cursors[1].PageURL, + "the resume is followed as served") + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{100}, unrecovered) + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{200}, recovered) +} + +// The walk's own cursor can be refused — minted under another filter set, or +// by a rotated secret. The walk re-enters from its explicit id rather than +// retrying the refused cursor for the whole window. +func TestARefusedRepairCursorReentersFromTheExplicitID(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + require.NoError(t, ledger.SaveRepairCursor(ctx, loss.ID, "cursor-under-old-filters")) + loss.RepairCursor = "cursor-under-old-filters" + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollFilterChanged}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "fresh"}}, + } + walker, ingested := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + require.GreaterOrEqual(t, len(polls.cursors), 2) + assert.Equal(t, "17099838508", polls.cursors[1].Since) + assert.Equal(t, []int64{17099838509}, *ingested) +} + +// A failure no repeat will fix ends this start's walk but leaves the loss open, +// so the next start — perhaps after the cause is fixed — tries again rather +// than finding the ids already condemned. +func TestAnUnrecoverableRepairPollLeavesTheLossOpen(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollRedirectRefused}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.Equal(t, 1, polls.calls, "a refused redirect is not worth ten minutes of retries") + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Len(t, open, 1) +} + +func retentionGone() error { + return &basecamp.FeedPositionGoneError{ + Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 410, Message: "outside retention"}, + Resume: "https://3.basecampapi.com/2914079/my/inbox.json?since=0", + } +} From 076b24ed144c5121e6fa5f6284db6bf267403c9d Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 15:07:10 +0200 Subject: [PATCH 03/49] Stop a repair walk from walking into the same 410 on every pass Once a 410 has fenced off the ids behind the epoch, later passes start at the fence, and a resume that answers 410 again waits for the repair cadence instead of looping and writing a gap per turn. --- internal/connector/ledger_recovery.go | 11 +++++++++++ internal/connector/repair.go | 14 ++++++++++++++ internal/connector/review_fixes_test.go | 21 +++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go index b96653753..e393f5984 100644 --- a/internal/connector/ledger_recovery.go +++ b/internal/connector/ledger_recovery.go @@ -278,6 +278,17 @@ func (l *Ledger) CloseLoss(ctx context.Context, lossID int64, now time.Time) (in return int(unrecovered), nil } +// SetRepairEntry moves a loss's walk entry to since and drops its cursor — +// after a 410 has fenced off everything at or below since. +func (l *Ledger) SetRepairEntry(ctx context.Context, lossID, since int64) error { + _, err := l.db.ExecContext(ctx, + `UPDATE losses SET repair_since = ?, repair_cursor = '' WHERE id = ?`, since, lossID) + if err != nil { + return fmt.Errorf("connector: set repair entry: %w", err) + } + return nil +} + // MarkUnrecoveredThrough gives up on a loss's missing ids at or below id — the // ones a 410's epoch has fenced off — and leaves the rest missing. func (l *Ledger) MarkUnrecoveredThrough(ctx context.Context, lossID, id int64) (int, error) { diff --git a/internal/connector/repair.go b/internal/connector/repair.go index a3767fe49..e6de09543 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -192,6 +192,13 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event switch pollErr.Kind { case eventfeed.PollGone: + if (cursor.PageURL != "" && cursor.PageURL == pollErr.ResumeURL) || pollErr.EpochAfterID == loss.RepairSince { + // The resume itself answered with the same 410. The gap is + // already recorded; following it again would loop. Leave it to the + // repair cadence. + w.log.Warn("the repair walk's resume answered 410 again; retrying on the repair cadence", "loss_id", loss.ID) + return nil, nil + } // History up to the epoch is gone, and the ids there with it. Ids // above the epoch are still servable, so the resume is followed as // served rather than condemning them too. @@ -215,6 +222,13 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event if missErr != nil { return nil, missErr } + // Everything at or below the epoch is settled, so later passes start + // at the fence rather than walking into the same 410 again. + loss.RepairSince = epoch + loss.RepairCursor = "" + if setErr := w.ledger.SetRepairEntry(ctx, loss.ID, epoch); setErr != nil { + return nil, setErr + } if len(stillMissing) == 0 || pollErr.ResumeURL == "" { // Every id was behind the fence, or there is no resume to follow: // no repeat will serve anything more. diff --git a/internal/connector/review_fixes_test.go b/internal/connector/review_fixes_test.go index a21bda413..98a62df0f 100644 --- a/internal/connector/review_fixes_test.go +++ b/internal/connector/review_fixes_test.go @@ -360,3 +360,24 @@ func retentionGone() error { Resume: "https://3.basecampapi.com/2914079/my/inbox.json?since=0", } } + +// A resume that answers with the same 410 again must not become an endless +// loop that writes a gap per turn. +func TestARepeated410OnTheResumeEndsTheWalk(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + require.NoError(t, err) + + gone := &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150"} + polls := &scriptedPolls{errs: []error{gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone, gone}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.LessOrEqual(t, polls.calls, 12, "a 410 on the resume is retried on the cadence, not in a tight loop") + gaps, err := ledger.Gaps(ctx) + require.NoError(t, err) + assert.Len(t, gaps, 1, "one 410, one gap") +} From a8e461b95b0308c276c9de6149eea7459bda50fc Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 15:21:33 +0200 Subject: [PATCH 04/49] Keep the ledger private, refuse feed redirects, and hand off through a reconnect Second review round. Each fix has a test that failed on the previous head, or failed when its fix was reverted where the defect was already gone. The ledger holds feed positions, which are signed tokens, and every event's metadata. It is now created 0600 in a 0700 directory, and an existing directory or file with looser permissions is refused rather than silently tightened. The SDK's default client follows a 3xx, to a foreign host too, before any continuation check here can run. The live adapter now builds its own client on a transport that refuses every redirect hop, so nothing reaches the target, and reports the refusal as the seam's redirect_refused. A reconnect canceling the connection while intake waited for queue room left the event seen and never queued. The hand-off now runs on Run's context and the reconnect waits for it. A project the membership list never names is learned for the process instead of costing a reconnect per event. A refused position re-enters at this filter set's own poll-served id before any other set's, which may be past events this one never served. Caller cancellation passes through the adapter unchanged rather than as a transient failure; the queue counts blocked offers so one resuming does not report the feed unpaused while another still waits; a reconnect latch left between connections is cleared by the connection that answers it. --- internal/connector/feed_adapter.go | 75 +++++- internal/connector/intake.go | 63 ++++- internal/connector/intake_feed_test.go | 3 +- internal/connector/ledger.go | 70 ++++- internal/connector/ledger_test.go | 2 +- internal/connector/live_adapter_test.go | 43 ++++ internal/connector/queue.go | 13 +- internal/connector/review2_test.go | 327 ++++++++++++++++++++++++ 8 files changed, 566 insertions(+), 30 deletions(-) create mode 100644 internal/connector/live_adapter_test.go create mode 100644 internal/connector/review2_test.go diff --git a/internal/connector/feed_adapter.go b/internal/connector/feed_adapter.go index efd183c3a..1cb32a507 100644 --- a/internal/connector/feed_adapter.go +++ b/internal/connector/feed_adapter.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "net/url" "strconv" "strings" @@ -85,6 +86,48 @@ type FeedClient interface { CreateStreamTicket(ctx context.Context) (*basecamp.StreamTicket, error) } +// NewLiveFeedAdapter builds the adapter over its own SDK client, whose +// transport refuses to follow redirects. +// +// It owns its client for that reason alone. The SDK's default client follows +// a 3xx — to a foreign host too, with the Authorization header stripped but +// the request still sent — and the seam's zero-egress obligation is broken +// before any continuation check here could run. Client options cannot fix +// that from outside (a custom *http.Client is replaced at construction), but +// the transport is honored, and a transport sees each redirect hop before it +// leaves the machine. +func NewLiveFeedAdapter(cfg *basecamp.Config, tokens basecamp.TokenProvider, accountID string, inner http.RoundTripper, opts ...basecamp.ClientOption) (*FeedAdapter, error) { + if cfg == nil || tokens == nil || accountID == "" { + return nil, errors.New("connector: the live feed adapter needs a config, a token provider and an account id") + } + if inner == nil { + inner = http.DefaultTransport + } + opts = append(opts, basecamp.WithTransport(RefuseRedirects(inner))) + client := basecamp.NewClient(cfg, tokens, opts...) + return NewFeedAdapter(client.ForAccount(accountID).EventFeed(), cfg.BaseURL) +} + +// ErrRedirectRefused is a redirect hop the feed's transport would not send. +var ErrRedirectRefused = errors.New("connector: the event feed does not follow redirects") + +// RefuseRedirects wraps a transport so that no redirect hop is ever sent. The +// client then reports the refusal as the request's failure. +func RefuseRedirects(inner http.RoundTripper) http.RoundTripper { + return refuseRedirects{inner: inner} +} + +type refuseRedirects struct{ inner http.RoundTripper } + +func (t refuseRedirects) RoundTrip(req *http.Request) (*http.Response, error) { + // Response is set exactly when the client created this request to follow + // a redirect. + if req.Response != nil { + return nil, ErrRedirectRefused + } + return t.inner.RoundTrip(req) +} + // FeedAdapter backs the TicketMinter and PollSource seams. type FeedAdapter struct { client FeedClient @@ -119,7 +162,7 @@ func NewFeedAdapter(client FeedClient, origin string) (*FeedAdapter, error) { func (a *FeedAdapter) MintStreamTicket(ctx context.Context) (eventfeed.StreamTicket, error) { ticket, err := a.client.CreateStreamTicket(ctx) if err != nil { - return eventfeed.StreamTicket{}, mintError(err) + return eventfeed.StreamTicket{}, mintError(ctx, err) } return eventfeed.StreamTicket{ Ticket: ticket.Ticket, @@ -137,7 +180,7 @@ func (a *FeedAdapter) Poll(ctx context.Context, cursor eventfeed.Cursor, filters page, err := a.client.PollEvents(ctx, opts) if err != nil { - return eventfeed.PollPage{}, pollError(err) + return eventfeed.PollPage{}, pollError(ctx, err) } events := make([]eventfeed.Event, 0, len(page.Events)) @@ -235,10 +278,19 @@ func idStrings(ids []int64) []string { // seam's taxonomy, and it is the one place the two 410s are told apart. // --------------------------------------------------------------------------- +// callerCanceled reports a failure that is the caller's own cancellation. The +// seam requires it to pass through unchanged: classified as transient, a +// shutdown or a reconnect would enter transport-retry handling. A deadline the +// client imposed on itself, with the caller's context still live, is not this +// and stays transient. +func callerCanceled(ctx context.Context, err error) bool { + return ctx.Err() != nil && errors.Is(err, ctx.Err()) +} + // pollError classifies a failed PollEvents call. -func pollError(err error) error { - if err == nil { - return nil +func pollError(ctx context.Context, err error) error { + if err == nil || callerCanceled(ctx, err) { + return err } // The inbox's 410 first, so it can never fall through to the feed's arm. @@ -265,6 +317,10 @@ func pollError(err error) error { } } + if errors.Is(err, ErrRedirectRefused) { + return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: err} + } + var mismatch *basecamp.FeedFilterMismatchError if errors.As(err, &mismatch) { return &eventfeed.PollError{ @@ -303,9 +359,12 @@ func pollError(err error) error { } // mintError classifies a failed CreateStreamTicket call. -func mintError(err error) error { - if err == nil { - return nil +func mintError(ctx context.Context, err error) error { + if err == nil || callerCanceled(ctx, err) { + return err + } + if errors.Is(err, ErrRedirectRefused) { + return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: err} } var apiErr *basecamp.Error if !errors.As(err, &apiErr) { diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 02a249474..b95321b1c 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -23,8 +23,8 @@ const ( // delay would call a still-committing event missing. DefaultRepairInterval = 60 * time.Second // DefaultRepairWindow is how long a loss stays open before the ids still - // missing are called unrecovered. Ten minutes is twenty repair polls past - // the safety delay. + // missing are called unrecovered. At the sixty-second cadence that is + // about ten repair polls, each well past the safety delay. DefaultRepairWindow = 10 * time.Minute // DefaultMembershipInterval is how often the agent's project list is // re-read. The cable snapshots the agent's buckets when it subscribes, so @@ -101,7 +101,10 @@ type Intake struct { // only when a page boundary confirms a poll page actually landed. pollCandidates []int64 snapshot map[int64]bool - reconnect chan struct{} + // learned holds buckets events proved visible that the lister did not + // name. + learned map[int64]bool + reconnect chan struct{} // cancelRun ends the current connection; nil between connections. cancelRun context.CancelFunc // promotedThisRun is whether this connection has recorded a poll-served @@ -229,6 +232,11 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { runCtx, cancel := context.WithCancel(ctx) defer cancel() + // This connection IS the reconnect any request made between connections + // asked for; a latch left standing would turn its first terminal error + // into a silent reconnect. + in.reconnectRequested() + in.mu.Lock() in.cancelRun = cancel in.promotedThisRun = false @@ -310,7 +318,12 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { feedErr = err break } - if err := in.ingest(runCtx, event, LaneOf(event)); err != nil { + // Run's context, not the connection's: a reconnect canceling the + // connection while this hand-off waits for queue room would leave the + // event seen in the ledger and never queued, since the re-served page + // no longer reports it new. The reconnect waits for the hand-off; the + // feed stays paused either way. + if err := in.ingest(ctx, event, LaneOf(event)); err != nil { if in.reconnectRequested() { return errReconnect } @@ -604,10 +617,13 @@ func (in *Intake) takeSnapshot(ctx context.Context) { } in.mu.Lock() defer in.mu.Unlock() - in.snapshot = make(map[int64]bool, len(buckets)) + in.snapshot = make(map[int64]bool, len(buckets)+len(in.learned)) for _, id := range buckets { in.snapshot[id] = true } + for id := range in.learned { + in.snapshot[id] = true + } } // noteBucket asks for a reconnect when an event arrives from a bucket the live @@ -621,6 +637,16 @@ func (in *Intake) noteBucket(bucketID int64) { return } in.log.Info("an event arrived from a project the live subscription does not hold", "bucket_id", bucketID) + // Learned for the life of the process, and merged into every later + // snapshot. A project the membership list never names — archived, or past + // the lister's page — would otherwise cost a reconnect per event, forever. + in.mu.Lock() + if in.learned == nil { + in.learned = make(map[int64]bool) + } + in.learned[bucketID] = true + in.snapshot[bucketID] = true + in.mu.Unlock() in.requestReconnect() } @@ -640,10 +666,17 @@ func (in *Intake) onPositionRejected(ctx context.Context) { // is at least what the ledger holds. return } - served, err := in.ledger.LineagePollServedID(ctx, in.key) - if err != nil || served == 0 { + // This filter set's own id first. Another set's may be past events this + // one never served, and re-entering there would skip them. + served, err := in.ledger.LastPollServedID(ctx, in.key) + if err != nil { return } + if served == 0 { + if served, err = in.ledger.LineagePollServedID(ctx, in.key); err != nil || served == 0 { + return + } + } in.mu.Lock() in.reentryAfter = served in.mu.Unlock() @@ -661,6 +694,7 @@ func (in *Intake) watchMembership(ctx context.Context) func() { done := make(chan struct{}) go func() { defer close(done) + defer stop() ticker := time.NewTicker(in.opts.MembershipInterval) defer ticker.Stop() for { @@ -702,15 +736,24 @@ func (in *Intake) membershipChanged(buckets []int64) bool { } return false } - if len(buckets) != len(in.snapshot) { - return true + listed := 0 + for id := range in.snapshot { + if !in.learned[id] { + listed++ + } } + fresh := 0 for _, id := range buckets { if !in.snapshot[id] { return true } + if !in.learned[id] { + fresh++ + } } - return false + // A bucket that dropped off the list is a change; a learned one showing + // up in the list is not. + return fresh != listed } // requestReconnect marks a reconnect due and ends the current connection now, diff --git a/internal/connector/intake_feed_test.go b/internal/connector/intake_feed_test.go index 8b4ef6287..a340afdd2 100644 --- a/internal/connector/intake_feed_test.go +++ b/internal/connector/intake_feed_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "path/filepath" "sync" "testing" "time" @@ -191,7 +192,7 @@ func TestIntakeRunsTheFeedThroughCatchUpAndStreaming(t *testing.T) { // position rather than the present. func TestIntakeSurvivesARestartWithoutDuplicating(t *testing.T) { dir := t.TempDir() - path := dir + "/connector.db" + path := filepath.Join(dir, "state", "connector.db") firstLedger, err := OpenLedger(path) require.NoError(t, err) diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index fc3dcadbc..2fa7c6e6a 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" "time" @@ -75,10 +76,8 @@ func OpenLedger(path string) (*Ledger, error) { // query, fragment or an escape, and open some other file. return nil, fmt.Errorf("connector: ledger path %q contains a character the SQLite URI cannot carry (?, # or %%)", path) } - if dir := filepath.Dir(path); dir != "" && dir != "." { - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, fmt.Errorf("connector: create ledger directory: %w", err) - } + if err := securePath(path); err != nil { + return nil, err } // _txlock=immediate takes the write lock when a transaction opens rather @@ -99,9 +98,72 @@ func OpenLedger(path string) (*Ledger, error) { _ = db.Close() return nil, err } + // The WAL and shared-memory sidecars exist now and were created under the + // process umask. The private directory already keeps other users out; + // tightening them too costs nothing. + for _, sidecar := range []string{path + "-wal", path + "-shm"} { + if err := os.Chmod(sidecar, 0o600); err != nil && !os.IsNotExist(err) { + _ = db.Close() + return nil, fmt.Errorf("connector: secure ledger sidecar: %w", err) + } + } return l, nil } +// securePath makes the ledger private or refuses it. +// +// The ledger holds feed positions — signed tokens that resume the account's +// feed — and every event's metadata. Its directory must be 0700 and the file +// 0600. A directory or file that already exists with looser permissions is +// refused rather than tightened: something else chose those permissions, and +// silently changing them could break it or hide that the ledger was exposed. +func securePath(path string) error { + dir := filepath.Dir(path) + switch info, err := os.Stat(dir); { + case os.IsNotExist(err): + if err := os.MkdirAll(dir, 0o700); err != nil { + return fmt.Errorf("connector: create ledger directory: %w", err) + } + if err := os.Chmod(dir, 0o700); err != nil { //nolint:gosec // a directory needs its search bit; 0700 is owner-only + return fmt.Errorf("connector: secure ledger directory: %w", err) + } + case err != nil: + return fmt.Errorf("connector: inspect ledger directory: %w", err) + case !info.IsDir(): + return fmt.Errorf("connector: ledger directory %s is not a directory", dir) + case looserThan(info.Mode(), 0o700): + return fmt.Errorf("connector: ledger directory %s is readable by other users (mode %04o); it must be 0700", dir, info.Mode().Perm()) + } + + switch info, err := os.Stat(path); { + case os.IsNotExist(err): + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("connector: create ledger: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("connector: create ledger: %w", err) + } + if err := os.Chmod(path, 0o600); err != nil { + return fmt.Errorf("connector: secure ledger: %w", err) + } + case err != nil: + return fmt.Errorf("connector: inspect ledger: %w", err) + case looserThan(info.Mode(), 0o600): + return fmt.Errorf("connector: ledger %s is readable by other users (mode %04o); it must be 0600", path, info.Mode().Perm()) + } + return nil +} + +// looserThan reports permission bits beyond limit. Windows has no POSIX bits +// to speak of; access there is the ACL of the user's profile directory. +func looserThan(mode os.FileMode, limit os.FileMode) bool { + if runtime.GOOS == "windows" { + return false + } + return mode.Perm()&^limit != 0 +} + // Close releases the ledger's handle. func (l *Ledger) Close() error { return l.db.Close() } diff --git a/internal/connector/ledger_test.go b/internal/connector/ledger_test.go index 47c0fe16a..c59203492 100644 --- a/internal/connector/ledger_test.go +++ b/internal/connector/ledger_test.go @@ -15,7 +15,7 @@ import ( func newTestLedger(t *testing.T) *Ledger { t.Helper() - ledger, err := OpenLedger(filepath.Join(t.TempDir(), "connector.db")) + ledger, err := OpenLedger(filepath.Join(t.TempDir(), "state", "connector.db")) require.NoError(t, err) t.Cleanup(func() { _ = ledger.Close() }) return ledger diff --git a/internal/connector/live_adapter_test.go b/internal/connector/live_adapter_test.go new file mode 100644 index 000000000..59c07ca66 --- /dev/null +++ b/internal/connector/live_adapter_test.go @@ -0,0 +1,43 @@ +package connector + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// The generated operation must not follow a 3xx to a server-supplied target: +// zero egress to a foreign redirect, before any continuation check can run. +func TestTheLiveAdapterRefusesRedirectsBeforeAnyEgress(t *testing.T) { + var foreignHits atomic.Int32 + foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + foreignHits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer foreign.Close() + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, foreign.URL+r.URL.Path, http.StatusFound) + })) + defer origin.Close() + + adapter, err := NewLiveFeedAdapter(&basecamp.Config{BaseURL: origin.URL}, &basecamp.StaticTokenProvider{Token: "token"}, "2914079", nil) + require.NoError(t, err) + + _, err = adapter.Poll(context.Background(), eventfeed.Cursor{Since: "1"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) + + _, err = adapter.MintStreamTicket(context.Background()) + require.Error(t, err) + + assert.Zero(t, foreignHits.Load(), "no request may reach the redirect target") +} diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 5afc08a1d..838990a0c 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -30,7 +30,10 @@ type Queue struct { warnAt int warned atomic.Bool - paused atomic.Bool + // waiting counts offers blocked for room. Intake and every open repair + // walk offer concurrently, so a single flag would be cleared by the first + // waiter to resume while another — perhaps the feed — still waits. + waiting atomic.Int32 // OnWarn fires when the depth first crosses the warning threshold, and // OnRecover when it falls back below. Both are optional. @@ -67,13 +70,11 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { default: } - q.paused.Store(true) - if q.OnPause != nil { + if q.waiting.Add(1) == 1 && q.OnPause != nil { q.OnPause(q.Depth()) } defer func() { - q.paused.Store(false) - if q.OnResume != nil { + if q.waiting.Add(-1) == 0 && q.OnResume != nil { q.OnResume(q.Depth()) } }() @@ -107,7 +108,7 @@ func (q *Queue) Depth() int { return len(q.ids) } // Paused reports whether an offer is currently waiting for room — which is to // say whether the feed is being consumed. -func (q *Queue) Paused() bool { return q.paused.Load() } +func (q *Queue) Paused() bool { return q.waiting.Load() > 0 } // noteDepth fires the warning edges. It is edge-triggered, not level: a // backlog that sits above the threshold for an hour is one warning, and the diff --git a/internal/connector/review2_test.go b/internal/connector/review2_test.go new file mode 100644 index 000000000..1f13156a4 --- /dev/null +++ b/internal/connector/review2_test.go @@ -0,0 +1,327 @@ +package connector + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// Round 2 of review. Each of these failed on 893c0d6. + +func TestARefusedPositionPrefersThisFilterSetsOwnPollServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + key := intake.CheckpointKey() + require.NoError(t, ledger.Save(ctx, key, "position-the-server-will-refuse")) + require.NoError(t, ledger.NotePollServed(ctx, key, 1000)) + other := key + other.FilterKey = "srv2-0000000000000000" + require.NoError(t, ledger.NotePollServed(ctx, other, 2000)) + + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "p2"}) + polls.ScriptPage(eventfeed.PollPage{Position: "p3"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + var reentry eventfeed.Cursor + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + calls := polls.Calls() + if len(calls) < 2 { + return false + } + reentry = calls[1].Cursor + return true + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "1000", reentry.Since, + "another filter set's id is past events this one never served; re-entering there skips them") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A reconnect that lands while intake is waiting for queue room must not +// strand the event it was handing over: it is already seen, so no re-served +// page would ever queue it again. +func TestAReconnectDuringABackloggedHandOffStrandsNothing(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 1) + require.NoError(t, err) + membership := &flakyMembership{buckets: []int64{48699913}} + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: membership, + MembershipInterval: 50 * time.Millisecond, + }) + intake.queue = queue + intake.opts.Queue = queue + for range 5 { + minter.ScriptTicket(ticket()) + } + page := eventfeed.PollPage{Events: []eventfeed.Event{testEvent(500), testEvent(501)}, Position: "p1"} + for range 5 { + polls.ScriptPage(page) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return queue.Paused() }, 5*time.Second, 5*time.Millisecond) + membership.mu.Lock() + membership.buckets = []int64{48699913, 1} + membership.mu.Unlock() + + // Nothing is taken until the reconnect has been asked for, so the hand-off + // is still waiting when it lands. + require.Eventually(t, func() bool { + return len(transport.Dials()) >= 2 || len(intake.reconnect) == 1 + }, 5*time.Second, time.Millisecond) + time.Sleep(20 * time.Millisecond) + + answered := 1 + var got []int64 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + ctxTake, stop := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer stop() + if id, err := queue.Take(ctxTake); err == nil { + got = append(got, id) + } + return len(got) >= 2 + }, 5*time.Second, 10*time.Millisecond, "every seen event must reach the queue") + assert.Equal(t, []int64{500, 501}, got) + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A project the membership list never names — archived, or past the lister's +// page — must cost one reconnect, not one per event. +func TestAnUnlistedProjectCostsOneReconnectNotOnePerEvent(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: &flakyMembership{buckets: []int64{1}}, + MembershipInterval: time.Hour, + }) + for range 20 { + minter.ScriptTicket(ticket()) + } + var events []eventfeed.Event + for id := int64(600); id < 605; id++ { + e := testEvent(id) + e.BucketID = 777 + events = append(events, e) + } + for range 20 { + polls.ScriptPage(eventfeed.PollPage{Events: events, Position: "p"}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + time.Sleep(10 * time.Millisecond) + } + assert.LessOrEqual(t, len(transport.Dials()), 2, "five events from one unlisted project are one reconnect") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +func TestSinceAppliesToTheFirstConnectionOnly(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + SinceEventID: 17099838000, + Membership: &flakyMembership{buckets: []int64{1}}, + MembershipInterval: time.Hour, + }) + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + e := testEvent(17099838001) + e.BucketID = 777 // unlisted: forces one reconnect + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{e}, Position: "after-since"}) + polls.ScriptPage(eventfeed.PollPage{Position: "second"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + return polls.CallCount() >= 2 + }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "17099838000", polls.Calls()[0].Cursor.Since) + assert.NotEqual(t, "17099838000", polls.Calls()[1].Cursor.Since, + "a reconnect resumes from what the run stored, not from --since again") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +func TestMembershipWatcherStopsWhenStopped(t *testing.T) { + var reads atomic.Int32 + intake, _, _ := newTestIntake(t, nil, nil) + intake.opts.Membership = countingMembership{&reads} + intake.opts.MembershipInterval = time.Millisecond + + stop := intake.watchMembership(context.Background()) + require.Eventually(t, func() bool { return reads.Load() > 0 }, time.Second, time.Millisecond) + stop() + after := reads.Load() + time.Sleep(20 * time.Millisecond) + assert.Equal(t, after, reads.Load(), "a stopped watcher reads nothing more") +} + +type countingMembership struct{ n *atomic.Int32 } + +func (c countingMembership) Buckets(context.Context) ([]int64, error) { + c.n.Add(1) + return nil, nil +} + +// The seam requires connector cancellation to pass through unchanged, or a +// shutdown enters transport-retry handling. +func TestCallerCancellationPassesThroughTheAdapterUnchanged(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + adapter := newTestAdapter(t, &fakeFeedClient{err: context.Canceled}) + _, err := adapter.Poll(ctx, eventfeed.Cursor{}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + assert.False(t, errors.As(err, &pollErr), "a canceled poll is not a transport failure") + assert.ErrorIs(t, err, context.Canceled) + + _, err = adapter.MintStreamTicket(ctx) + var mintErr *eventfeed.MintError + assert.False(t, errors.As(err, &mintErr), "a canceled mint is not a transport failure") + assert.ErrorIs(t, err, context.Canceled) + + // A client-owned timeout with the caller's context still live stays + // transient. + adapter = newTestAdapter(t, &fakeFeedClient{err: context.DeadlineExceeded}) + _, err = adapter.Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollTransient, pollErr.Kind) +} + +func TestQueuePauseTracksEveryBlockedOffer(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + var pauses, resumes atomic.Int32 + queue.OnPause = func(int) { pauses.Add(1) } + queue.OnResume = func(int) { resumes.Add(1) } + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + + first := make(chan error, 1) + second := make(chan error, 1) + go func() { first <- queue.Offer(ctx, 2) }() + go func() { second <- queue.Offer(ctx, 3) }() + require.Eventually(t, func() bool { return pauses.Load() >= 1 && len(queue.ids) == 1 }, time.Second, time.Millisecond) + time.Sleep(20 * time.Millisecond) + + _, err = queue.Take(ctx) + require.NoError(t, err) + select { + case <-first: + case <-second: + case <-time.After(time.Second): + t.Fatal("one offer should have resumed") + } + assert.True(t, queue.Paused(), "the other offer is still waiting, so the feed is still paused") + assert.Zero(t, resumes.Load(), "resume fires when the last waiter stops waiting") + + _, err = queue.Take(ctx) + require.NoError(t, err) + require.Eventually(t, func() bool { return !queue.Paused() }, time.Second, time.Millisecond) + assert.Equal(t, int32(1), pauses.Load()) + assert.Equal(t, int32(1), resumes.Load()) +} + +// The ledger holds feed positions — signed tokens — and account event +// metadata. It is private to the user or it is not opened. +func TestLedgerIsCreatedPrivate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits") + } + dir := filepath.Join(t.TempDir(), "state") + ledger, err := OpenLedger(filepath.Join(dir, "connector.db")) + require.NoError(t, err) + defer ledger.Close() + require.NoError(t, ledger.Save(context.Background(), testKey(), "p")) + + info, err := os.Stat(dir) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + for _, name := range []string{"connector.db", "connector.db-wal", "connector.db-shm"} { + info, err := os.Stat(filepath.Join(dir, name)) + if os.IsNotExist(err) { + continue + } + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm(), name) + } +} + +func TestLedgerRefusesALooseDirectory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits") + } + dir := filepath.Join(t.TempDir(), "shared") + require.NoError(t, os.Mkdir(dir, 0o755)) + require.NoError(t, os.Chmod(dir, 0o755)) + _, err := OpenLedger(filepath.Join(dir, "connector.db")) + assert.Error(t, err, "a directory other users can read exposes the ledger's sidecars") +} + +func TestLedgerRefusesALooseFile(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX permission bits") + } + dir := filepath.Join(t.TempDir(), "state") + require.NoError(t, os.Mkdir(dir, 0o700)) + require.NoError(t, os.Chmod(dir, 0o700)) + path := filepath.Join(dir, "connector.db") + require.NoError(t, os.WriteFile(path, nil, 0o644)) + require.NoError(t, os.Chmod(path, 0o644)) + _, err := OpenLedger(path) + assert.Error(t, err) +} From 224de40691aa0933772a3f0b4333c78ddb084e5f Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 15:50:02 +0200 Subject: [PATCH 05/49] Make the adapter the only policy boundary, and state the repair pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A design pass over intake as one state machine named 25 invariants and found seven unheld. The repeat findings all came from one place: the repair walk is a second feed walker outside the package's governance, and it had its own copies of cursor validation, error rendering and 410 classification. Those copies are gone. Every walker now reaches the wire through the adapter, and the adapter is where policy lives: - A followed URL must carry exactly one cursor. `PollEventsOptionsFromURL` allows a URL with neither, which enters the feed at the present — a silent skip on the one path whose purpose is to continue. Refused. - No error the adapter returns carries the generated one. A network error renders its request URL, which on this lane is a position token or a server-supplied continuation, and a refused redirect renders the server-chosen Location. Errors now leave as a fixed sentinel or a status and a code, and the repair walk logs a failure's kind, never its text. The repair walk carries explicit per-pass state instead of inferring it from the shape of its cursor: a 410 whose epoch happens to equal the walk's entry is followed like any other, so the ids above the fence stay recoverable, while the same resume answering 410 twice in one pass waits for the cadence. A position refused when the safe re-entry cannot be read now ends the run. Returning quietly let the package fall back to its own reset cursor, which is the present. The instance lock is named from the account number, not its spelling, so 02914079 and 2914079 cannot hold two locks on one account. Two processes opening one fresh ledger both get it. The queue's warning edges are serialized. A learned project that the membership list later names is listed from then on, so its revocation is seen. --- internal/connector/feed_adapter.go | 89 ++++++-- internal/connector/intake.go | 92 ++++++-- internal/connector/invariants_test.go | 288 ++++++++++++++++++++++++ internal/connector/ledger.go | 15 +- internal/connector/live_adapter_test.go | 5 +- internal/connector/lock.go | 9 +- internal/connector/queue.go | 16 +- internal/connector/repair.go | 105 +++++---- internal/connector/review2_test.go | 5 +- internal/connector/review_fixes_test.go | 4 +- 10 files changed, 523 insertions(+), 105 deletions(-) create mode 100644 internal/connector/invariants_test.go diff --git a/internal/connector/feed_adapter.go b/internal/connector/feed_adapter.go index 1cb32a507..b510e6612 100644 --- a/internal/connector/feed_adapter.go +++ b/internal/connector/feed_adapter.go @@ -112,7 +112,10 @@ func NewLiveFeedAdapter(cfg *basecamp.Config, tokens basecamp.TokenProvider, acc var ErrRedirectRefused = errors.New("connector: the event feed does not follow redirects") // RefuseRedirects wraps a transport so that no redirect hop is ever sent. The -// client then reports the refusal as the request's failure. +// client then reports the refusal as the request's failure. The SDK sees that +// failure as a network error and spends its retry budget re-sending the +// ORIGINAL, same-origin request before giving up; nothing reaches the target +// on any attempt. func RefuseRedirects(inner http.RoundTripper) http.RoundTripper { return refuseRedirects{inner: inner} } @@ -219,7 +222,14 @@ func (a *FeedAdapter) optionsFor(cursor eventfeed.Cursor, filters eventfeed.Filt } opts, err := basecamp.PollEventsOptionsFromURL(cursor.PageURL) if err != nil { - return nil, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: err} + return nil, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errMalformedContinuation} + } + if (opts.Since == "") == (opts.Position == "") { + // A followed URL carries exactly one cursor. With neither, the + // generated operation enters at the present — a silent skip of + // everything unserved, on the path whose whole purpose was to + // continue. With both, the server decides which one wins. + return nil, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errCursorlessContinuation} } // The URL carries the server's own canonical filter set. It is used // as served: re-imposing the local filters on a resume is how a @@ -278,19 +288,45 @@ func idStrings(ids []int64) []string { // seam's taxonomy, and it is the one place the two 410s are told apart. // --------------------------------------------------------------------------- +// Everything below builds errors that are safe to render. The rule, for every +// walker this adapter serves: no Err carries the generated error. A generated +// network error renders its request URL — a position token, a server-supplied +// continuation — and a refused redirect renders the server-chosen Location. An +// error leaves this file as a fixed sentinel or a status and a code, never as +// text the server or the request chose. + +var ( + errMalformedContinuation = errors.New("connector: continuation URL could not be read") + errCursorlessContinuation = errors.New("connector: continuation URL carries no single cursor") + errNetwork = errors.New("connector: event feed request failed before a response") +) + +// sanitized reduces a generated error to what is safe to render. +func sanitized(err error) error { + var apiErr *basecamp.Error + if errors.As(err, &apiErr) { + return fmt.Errorf("connector: event feed answered HTTP %d (%s)", apiErr.HTTPStatus, apiErr.Code) + } + return errNetwork +} + // callerCanceled reports a failure that is the caller's own cancellation. The // seam requires it to pass through unchanged: classified as transient, a // shutdown or a reconnect would enter transport-retry handling. A deadline the // client imposed on itself, with the caller's context still live, is not this -// and stays transient. +// and stays transient. The context's own error is returned, not the failure +// that wraps it, which may render the request URL. func callerCanceled(ctx context.Context, err error) bool { return ctx.Err() != nil && errors.Is(err, ctx.Err()) } // pollError classifies a failed PollEvents call. func pollError(ctx context.Context, err error) error { - if err == nil || callerCanceled(ctx, err) { - return err + if err == nil { + return nil + } + if callerCanceled(ctx, err) { + return ctx.Err() } // The inbox's 410 first, so it can never fall through to the feed's arm. @@ -306,19 +342,19 @@ func pollError(ctx context.Context, err error) error { if gone.EpochAfterID == nil { return &eventfeed.PollError{ Kind: eventfeed.PollUnrecoverable, - Err: &InboxRetentionGoneError{Resume: gone.Resume, Err: err}, + Err: &InboxRetentionGoneError{Resume: gone.Resume, Err: sanitized(err)}, } } return &eventfeed.PollError{ Kind: eventfeed.PollGone, EpochAfterID: *gone.EpochAfterID, ResumeURL: gone.Resume, - Err: &FeedEpochGoneError{EpochAfterID: *gone.EpochAfterID, Resume: gone.Resume, Err: err}, + Err: &FeedEpochGoneError{EpochAfterID: *gone.EpochAfterID, Resume: gone.Resume, Err: sanitized(err)}, } } if errors.Is(err, ErrRedirectRefused) { - return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: err} + return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: ErrRedirectRefused} } var mismatch *basecamp.FeedFilterMismatchError @@ -327,58 +363,61 @@ func pollError(ctx context.Context, err error) error { Kind: eventfeed.PollFilterChanged, PositionDigest: mismatch.PositionDigest, FiltersDigest: mismatch.FiltersDigest, - Err: err, + Err: sanitized(err), } } var apiErr *basecamp.Error if !errors.As(err, &apiErr) { - return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: err} + return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errNetwork} } switch { case apiErr.HTTPStatus == 400: // SWAP POINT for basecamp-sdk PR 912's *FeedRequestError: when it // carries reason=invalid_position this becomes PollPositionInvalid, - // and reason=invalid_filter becomes PollFilterInvalid. Until the + // and reason=invalid_filter becomes PollFilterInvalid (with the + // server's message in Msg, as the seam requires there). Until the // server names the reason, a 400 is undifferentiated and is surfaced // rather than guessed — see UndifferentiatedRequestError. return &eventfeed.PollError{ Kind: eventfeed.PollUnrecoverable, - Msg: apiErr.Message, - Err: &UndifferentiatedRequestError{Err: err}, + Err: &UndifferentiatedRequestError{Err: sanitized(err)}, } case apiErr.HTTPStatus == 401 || apiErr.HTTPStatus == 403: - return &eventfeed.PollError{Kind: eventfeed.PollUnauthorized, Err: err} + return &eventfeed.PollError{Kind: eventfeed.PollUnauthorized, Err: sanitized(err)} case apiErr.RetryAfter > 0: - return &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: retryAfter(apiErr), Err: err} + return &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: retryAfter(apiErr), Err: sanitized(err)} case apiErr.Retryable: - return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: err} + return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: sanitized(err)} } - return &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Msg: apiErr.Message, Err: err} + return &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: sanitized(err)} } // mintError classifies a failed CreateStreamTicket call. func mintError(ctx context.Context, err error) error { - if err == nil || callerCanceled(ctx, err) { - return err + if err == nil { + return nil + } + if callerCanceled(ctx, err) { + return ctx.Err() } if errors.Is(err, ErrRedirectRefused) { - return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: err} + return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: ErrRedirectRefused} } var apiErr *basecamp.Error if !errors.As(err, &apiErr) { - return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: err} + return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: errNetwork} } switch { case apiErr.HTTPStatus == 401 || apiErr.HTTPStatus == 403: - return &eventfeed.MintError{Kind: eventfeed.MintUnauthorized, Err: err} + return &eventfeed.MintError{Kind: eventfeed.MintUnauthorized, Err: sanitized(err)} case apiErr.RetryAfter > 0: - return &eventfeed.MintError{Kind: eventfeed.MintThrottled, RetryAfter: retryAfter(apiErr), Err: err} + return &eventfeed.MintError{Kind: eventfeed.MintThrottled, RetryAfter: retryAfter(apiErr), Err: sanitized(err)} case apiErr.Retryable: - return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: err} + return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: sanitized(err)} } - return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: err} + return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: sanitized(err)} } func retryAfter(apiErr *basecamp.Error) time.Duration { diff --git a/internal/connector/intake.go b/internal/connector/intake.go index b95321b1c..985d5ea31 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -94,6 +94,9 @@ type Intake struct { pointer *pointerWriter key eventfeed.CheckpointKey + // positions reads the safe re-entry ids; the ledger, except in tests that + // need the read to fail. + positions pollServedReader mu sync.Mutex // pollCandidates holds the ids delivered since the last page boundary that @@ -114,6 +117,9 @@ type Intake struct { // reentryAfter is the safe re-entry the next connection takes after a // refused stored position. reentryAfter int64 + // abortErr ends the run: set when continuing could only mean entering + // the feed somewhere unsafe. + abortErr error repairs sync.WaitGroup // lifetime is Run's context. Repair walks are bound to it rather than to @@ -181,9 +187,16 @@ func New(opts Options) (*Intake, error) { }, } in.ledger.now = opts.Clock + in.positions = opts.Ledger return in, nil } +// pollServedReader is the slice of the ledger re-entry reads. +type pollServedReader interface { + LastPollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) + LineagePollServedID(ctx context.Context, key eventfeed.CheckpointKey) (int64, error) +} + // CheckpointKey is the identity this intake's position is stored under. func (in *Intake) CheckpointKey() eventfeed.CheckpointKey { return in.key } @@ -308,8 +321,15 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { _ = feed.Close() feed.Wait() }() - stopMembership := in.watchMembership(runCtx) - defer stopMembership() + // The watcher's context is created and canceled here, in its owner, so + // the cancel is provably reached on every path out of this connection. + watchCtx, stopWatch := context.WithCancel(runCtx) + defer stopWatch() + awaitMembership := in.watchMembership(watchCtx) + defer func() { + stopWatch() + awaitMembership() + }() defer cancel() var feedErr error @@ -331,6 +351,12 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { } } + in.mu.Lock() + aborted := in.abortErr + in.mu.Unlock() + if aborted != nil { + return aborted + } if in.reconnectRequested() { return errReconnect } @@ -668,14 +694,21 @@ func (in *Intake) onPositionRejected(ctx context.Context) { } // This filter set's own id first. Another set's may be past events this // one never served, and re-entering there would skip them. - served, err := in.ledger.LastPollServedID(ctx, in.key) + served, err := in.positions.LastPollServedID(ctx, in.key) + if err == nil && served == 0 { + served, err = in.positions.LineagePollServedID(ctx, in.key) + } if err != nil { + // Returning here would let the package take its own reset cursor, + // which is the present. Not knowing where it is safe to re-enter is + // a reason to stop, not to guess. + in.abort(fmt.Errorf("connector: a position was refused and the safe re-entry could not be read: %w", err)) return } if served == 0 { - if served, err = in.ledger.LineagePollServedID(ctx, in.key); err != nil || served == 0 { - return - } + // The poll lane has never served this consumer anything: the present + // is all there is. + return } in.mu.Lock() in.reentryAfter = served @@ -683,18 +716,29 @@ func (in *Intake) onPositionRejected(ctx context.Context) { in.requestReconnect() } +// abort ends the current connection and the run with err. +func (in *Intake) abort(err error) { + in.mu.Lock() + if in.abortErr == nil { + in.abortErr = err + } + cancel := in.cancelRun + in.mu.Unlock() + if cancel != nil { + cancel() + } +} + // watchMembership re-reads the agent's projects on a timer and asks for a -// reconnect when the set changes. The returned stop function ends the watcher -// and waits for it, and is safe to call whatever state ctx is in. +// reconnect when the set changes. It runs until ctx ends; the returned +// function waits for it to have stopped. func (in *Intake) watchMembership(ctx context.Context) func() { if in.opts.Membership == nil { return func() {} } - ctx, stop := context.WithCancel(ctx) done := make(chan struct{}) go func() { defer close(done) - defer stop() ticker := time.NewTicker(in.opts.MembershipInterval) defer ticker.Stop() for { @@ -717,15 +761,16 @@ func (in *Intake) watchMembership(ctx context.Context) func() { } } }() - return func() { - stop() - <-done - } + return func() { <-done } } // membershipChanged compares a fresh read with the snapshot. With no snapshot // — the read at subscribe failed — the fresh read becomes the baseline; // otherwise a failed first read would disable change detection for good. +// +// Learned buckets (proved visible by an event, never named by the list) are +// ignored on the way out, so the list omitting them is not a change. Once the +// list names one, it is listed like any other, so its later revocation is. func (in *Intake) membershipChanged(buckets []int64) bool { in.mu.Lock() defer in.mu.Unlock() @@ -736,24 +781,21 @@ func (in *Intake) membershipChanged(buckets []int64) bool { } return false } - listed := 0 - for id := range in.snapshot { - if !in.learned[id] { - listed++ - } - } - fresh := 0 for _, id := range buckets { if !in.snapshot[id] { return true } + } + for _, id := range buckets { + delete(in.learned, id) + } + listed := 0 + for id := range in.snapshot { if !in.learned[id] { - fresh++ + listed++ } } - // A bucket that dropped off the list is a change; a learned one showing - // up in the list is not. - return fresh != listed + return listed != len(buckets) } // requestReconnect marks a reconnect due and ends the current connection now, diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go new file mode 100644 index 000000000..7774cb628 --- /dev/null +++ b/internal/connector/invariants_test.go @@ -0,0 +1,288 @@ +package connector + +import ( + "bytes" + "context" + "errors" + "log/slog" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// One test per invariant the design review found unenforced or untested. The +// invariant names match the review's list (A positions, C repair, D +// classification, E lifecycle, F queue, G lock, H confidentiality). + +// D2 / A3: a followed URL carries exactly its cursor. One with neither since +// nor position would silently enter at the present. +func TestInvariantD2ACursorlessContinuationIsRefused(t *testing.T) { + client := &fakeFeedClient{} + _, err := newTestAdapter(t, client).Poll(context.Background(), + eventfeed.Cursor{PageURL: "https://3.basecampapi.com/2914079/events.json?types=comment.created"}, + eventfeed.Filters{}) + + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.NotEqual(t, eventfeed.PollTransient, pollErr.Kind) + assert.Zero(t, client.calls, "a URL with no cursor is an entry at the present, and is never sent") +} + +// H1: nothing server-chosen — a redirect target, a URL carrying a position — +// is rendered into an error the adapter returns. +func TestInvariantH1AdapterErrorsRenderNoServerChosenURL(t *testing.T) { + leaky := errors.New(`Get "https://3.basecampapi.com/2914079/events.json?position=SECRET-POSITION": connection reset`) + + _, err := newTestAdapter(t, &fakeFeedClient{err: leaky}).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + require.Error(t, err) + assert.NotContains(t, err.Error(), "SECRET-POSITION") + + _, err = newTestAdapter(t, &fakeFeedClient{err: leaky}).MintStreamTicket(context.Background()) + require.Error(t, err) + assert.NotContains(t, err.Error(), "SECRET-POSITION") + + redirect := errors.Join(ErrRedirectRefused, errors.New(`Get "https://evil.example.com/steal?leak=SECRET-TARGET"`)) + _, err = newTestAdapter(t, &fakeFeedClient{err: redirect}).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) + assert.NotContains(t, err.Error(), "SECRET-TARGET") + + _, err = newTestAdapter(t, &fakeFeedClient{err: redirect}).MintStreamTicket(context.Background()) + require.Error(t, err) + assert.NotContains(t, err.Error(), "SECRET-TARGET") +} + +// H1, repair side: the walk's logs render a failure's kind, never its text. +func TestInvariantH1RepairLogsRenderNoFailureText(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Minute) + require.NoError(t, err) + + var logs bytes.Buffer + polls := &scriptedPolls{errs: []error{ + errors.New("position=SECRET-POSITION"), + &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errors.New("next=SECRET-NEXT")}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.log = slog.New(slog.NewTextHandler(&logs, nil)) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.NotContains(t, logs.String(), "SECRET") +} + +type failingPositions struct{} + +func (failingPositions) LastPollServedID(context.Context, eventfeed.CheckpointKey) (int64, error) { + return 0, errors.New("disk I/O error") +} + +func (failingPositions) LineagePollServedID(context.Context, eventfeed.CheckpointKey) (int64, error) { + return 0, errors.New("disk I/O error") +} + +// A3: when the safe re-entry after a refused position cannot be read, the +// connection ends rather than letting the package enter at the present. +func TestInvariantA3AnUnreadableReentryEndsTheFeedInsteadOfEnteringAtThePresent(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "position-the-server-will-refuse")) + intake.positions = failingPositions{} + + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "present-entry"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + err := awaitReturn(t, done, "a feed whose safe re-entry cannot be read must end") + assert.Error(t, err) + for _, call := range polls.Calls() { + assert.NotEqual(t, "now", call.Cursor.Since, "no poll may enter at the present") + } +} + +// G1: one lock per canonical account and agent. +func TestInvariantG1OneLockPerCanonicalAccount(t *testing.T) { + dir := t.TempDir() + first, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + require.NoError(t, err) + t.Cleanup(func() { _ = first.Release() }) + + _, err = AcquireInstanceLock(dir, "02914079", 52007412, time.Now()) + assert.Error(t, err, "02914079 is the same account and must meet the same lock") + _, err = AcquireInstanceLock(dir, "0", 52007412, time.Now()) + assert.Error(t, err) +} + +// C2: a 410 on a pass that has not yet followed that 410's resume is +// followed, whatever the walk's entry happens to equal. Ids above the fence +// stay recoverable. +func TestInvariantC2A410OnAStoredPositionIsFollowedEvenAtTheFence(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + require.NoError(t, err) + _, err = ledger.MarkUnrecoveredThrough(ctx, loss.ID, 150) + require.NoError(t, err) + require.NoError(t, ledger.SetRepairEntry(ctx, loss.ID, 150)) + require.NoError(t, ledger.SaveRepairCursor(ctx, loss.ID, "position-above-the-fence")) + loss.RepairSince = 150 + loss.RepairCursor = "position-above-the-fence" + + polls := fencedPolls{resume: "https://3.basecampapi.com/2914079/events.json?since=150"} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{200}, recovered) +} + +// fencedPolls answers every stored position with the epoch's 410, and serves +// the event above the fence only from the resume. +type fencedPolls struct{ resume string } + +func (f fencedPolls) Poll(_ context.Context, cursor eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { + if cursor.PageURL == f.resume || cursor.Since == "150" { + return eventfeed.PollPage{Events: []eventfeed.Event{testEvent(200)}, Position: "above"}, nil + } + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, ResumeURL: f.resume} +} + +// F2: warning edges settle on the true state under concurrent offers and +// takes. +func TestInvariantF2WarningEdgesSettleOnTheTrueState(t *testing.T) { + for range 50 { + queue, err := NewQueue(1, 64) + require.NoError(t, err) + var warns, recovers atomic.Int32 + queue.OnWarn = func(int) { warns.Add(1) } + queue.OnRecover = func(int) { recovers.Add(1) } + + ctx := context.Background() + var wg sync.WaitGroup + for g := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 200 { + require.NoError(t, queue.Offer(ctx, int64(g*1000+i))) + _, err := queue.Take(ctx) + require.NoError(t, err) + } + }() + } + wg.Wait() + require.Zero(t, queue.Depth()) + require.Equal(t, warns.Load(), recovers.Load(), "every warning is answered once the backlog is gone") + } +} + +// F1: while intake is paused on a full queue, the page it is in the middle of +// is not checkpointed. +func TestInvariantF1APausedFeedDoesNotMoveTheCheckpoint(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 1) + require.NoError(t, err) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + intake.queue = queue + intake.opts.Queue = queue + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(500), testEvent(501)}, Position: "after-both"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, queue.Paused, 5*time.Second, time.Millisecond) + time.Sleep(50 * time.Millisecond) + _, ok, err := ledger.Load(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.False(t, ok, "a crash now must resume from before the page, not after it") + + _, err = queue.Take(ctx) + require.NoError(t, err) + require.Eventually(t, func() bool { + position, ok, err := ledger.Load(ctx, intake.CheckpointKey()) + return err == nil && ok && position == "after-both" + }, 5*time.Second, 5*time.Millisecond) + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// E2: a reconnect asked for between connections is answered by the next +// connection, not carried into it to turn its terminal error into a silent +// reconnect. +func TestInvariantE2AStaleReconnectDoesNotSwallowATerminalError(t *testing.T) { + ledger := newTestLedger(t) + intake, _, minter, _ := newFeedIntake(t, ledger, Options{}) + for range 4 { + minter.ScriptError(&eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: errors.New("gone")}) + } + // Asked for while no connection was running: the next connection IS the + // answer to it. + intake.requestReconnect() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := awaitReturn(t, runInBackground(ctx, t, intake), "the terminal error must end Run") + assert.Error(t, err) + assert.Equal(t, 1, minter.Calls(), + "a latch carried into the connection turns its terminal error into a reconnect") +} + +// E3: membership detection catches every change to the listed set, ignores +// learned buckets, and notices a learned bucket's revocation once the list +// has named it. +func TestInvariantE3MembershipComparison(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.snapshot = map[int64]bool{1: true, 2: true, 9: true} + intake.learned = map[int64]bool{9: true} + + assert.False(t, intake.membershipChanged([]int64{1, 2}), "a learned bucket the list omits is not a change") + assert.True(t, intake.membershipChanged([]int64{1}), "a listed bucket dropping off is a change") + + intake.snapshot = map[int64]bool{1: true, 2: true, 9: true} + intake.learned = map[int64]bool{9: true} + assert.False(t, intake.membershipChanged([]int64{1, 2, 9}), "the list catching up with a learned bucket is not a change") + assert.True(t, intake.membershipChanged([]int64{1, 2}), "and its later revocation is") +} + +// H2 availability: two processes opening one fresh ledger both get it. +func TestInvariantH2ConcurrentFreshOpensAllSucceed(t *testing.T) { + path := filepath.Join(t.TempDir(), "state", "connector.db") + var wg sync.WaitGroup + errs := make(chan error, 8) + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + ledger, err := OpenLedger(path) + if err == nil { + err = ledger.Close() + } + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + assert.NoError(t, err) + } +} diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index 2fa7c6e6a..c10069e1c 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -135,18 +135,21 @@ func securePath(path string) error { return fmt.Errorf("connector: ledger directory %s is readable by other users (mode %04o); it must be 0700", dir, info.Mode().Perm()) } - switch info, err := os.Stat(path); { - case os.IsNotExist(err): - f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - if err != nil { - return fmt.Errorf("connector: create ledger: %w", err) - } + switch f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600); { + case err == nil: if err := f.Close(); err != nil { return fmt.Errorf("connector: create ledger: %w", err) } if err := os.Chmod(path, 0o600); err != nil { return fmt.Errorf("connector: secure ledger: %w", err) } + return nil + case !os.IsExist(err): + return fmt.Errorf("connector: create ledger: %w", err) + } + // It exists — perhaps created a moment ago by another process opening the + // same fresh ledger. Its permissions decide, not who created it. + switch info, err := os.Stat(path); { case err != nil: return fmt.Errorf("connector: inspect ledger: %w", err) case looserThan(info.Mode(), 0o600): diff --git a/internal/connector/live_adapter_test.go b/internal/connector/live_adapter_test.go index 59c07ca66..05703d5cf 100644 --- a/internal/connector/live_adapter_test.go +++ b/internal/connector/live_adapter_test.go @@ -28,7 +28,10 @@ func TestTheLiveAdapterRefusesRedirectsBeforeAnyEgress(t *testing.T) { })) defer origin.Close() - adapter, err := NewLiveFeedAdapter(&basecamp.Config{BaseURL: origin.URL}, &basecamp.StaticTokenProvider{Token: "token"}, "2914079", nil) + adapter, err := NewLiveFeedAdapter(&basecamp.Config{BaseURL: origin.URL}, &basecamp.StaticTokenProvider{Token: "token"}, "2914079", nil, + // One attempt: the SDK retries a refused hop as a network error, which + // only repeats the same-origin request and slows the test down. + basecamp.WithMaxRetries(0)) require.NoError(t, err) _, err = adapter.Poll(context.Background(), eventfeed.Cursor{Since: "1"}, eventfeed.Filters{}) diff --git a/internal/connector/lock.go b/internal/connector/lock.go index c12ba2262..2d417818b 100644 --- a/internal/connector/lock.go +++ b/internal/connector/lock.go @@ -43,11 +43,14 @@ type instanceHolder struct { // AcquireInstanceLock takes the lock for one account and agent, or refuses. func AcquireInstanceLock(dir, accountID string, agentPersonID int64, now time.Time) (*InstanceLock, error) { - if _, err := strconv.ParseUint(accountID, 10, 64); err != nil || agentPersonID <= 0 { - // The account id becomes part of a file name, so it is held to what an - // account id is: digits. + account, err := strconv.ParseUint(accountID, 10, 64) + if err != nil || account == 0 || agentPersonID <= 0 { return nil, errors.New("connector: the instance lock needs a numeric account id and an agent person id") } + // The file is named from the NUMBER, not the spelling: "02914079" and + // "2914079" are one account and must meet one lock, or the refusal of a + // second connector is a matter of how the id was typed. + accountID = strconv.FormatUint(account, 10) if err := os.MkdirAll(dir, 0o700); err != nil { return nil, fmt.Errorf("connector: create state directory: %w", err) } diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 838990a0c..e79501369 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -3,6 +3,7 @@ package connector import ( "context" "errors" + "sync" "sync/atomic" ) @@ -29,7 +30,12 @@ type Queue struct { ids chan int64 warnAt int - warned atomic.Bool + // edges serializes the depth sample with the warning transition and its + // callback. Unserialized, an offer can sample a warning depth, a take can + // drain and find nothing to recover from, and the offer then raises a + // warning that no later event will clear. + edges sync.Mutex + warned bool // waiting counts offers blocked for room. Intake and every open repair // walk offer concurrently, so a single flag would be cleared by the first // waiter to resume while another — perhaps the feed — still waits. @@ -115,13 +121,17 @@ func (q *Queue) Paused() bool { return q.waiting.Load() > 0 } // recovery is the other half of the pair, so a warning is never left standing // after the thing it warned about went away. func (q *Queue) noteDepth() { + q.edges.Lock() + defer q.edges.Unlock() depth := q.Depth() switch { - case depth >= q.warnAt && q.warned.CompareAndSwap(false, true): + case depth >= q.warnAt && !q.warned: + q.warned = true if q.OnWarn != nil { q.OnWarn(depth) } - case depth < q.warnAt && q.warned.CompareAndSwap(true, false): + case depth < q.warnAt && q.warned: + q.warned = false if q.OnRecover != nil { q.OnRecover(depth) } diff --git a/internal/connector/repair.go b/internal/connector/repair.go index e6de09543..f19be8d8f 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -108,11 +108,11 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { } last := loss.RepairCursor - reentered := false + pass := &repairPass{followed: map[string]bool{}} for { page, err := w.polls.Poll(ctx, cursor, w.filters) if err != nil { - next, err := w.pollFailure(ctx, loss, cursor, err, &reentered) + next, err := w.pollFailure(ctx, loss, cursor, err, pass) if err != nil || next == nil { return last, err } @@ -158,10 +158,35 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { // full recovery. var errReconciliationEnded = errors.New("connector: reconciliation ended inside the repair walk") +// repairPass is what one pass has already done, stated rather than inferred +// from the shape of the cursor. +type repairPass struct { + // followed holds the resume URLs this pass has already followed. A 410 on + // one of them is the same 410 again. + followed map[string]bool + // reentered is whether this pass has already dropped a refused cursor + // for the explicit id. + reentered bool +} + +// failureKind is the only rendering of a failed poll the walk logs. A poll's +// error text can carry its request URL — a position token or a +// server-supplied continuation — and the walk logs at the operator's terminal. +func failureKind(err error) string { + var pollErr *eventfeed.PollError + if errors.As(err, &pollErr) { + return pollErr.Kind.String() + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "canceled" + } + return "unclassified" +} + // pollFailure decides what one failed repair poll means. It returns the cursor // to continue this pass at, nil with no error to end the pass and wait for the // next repair poll, or an error. -func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor eventfeed.Cursor, err error, reentered *bool) (*eventfeed.Cursor, error) { +func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor eventfeed.Cursor, err error, pass *repairPass) (*eventfeed.Cursor, error) { var retention *InboxRetentionGoneError if errors.As(err, &retention) { // The inbox lane's 410 on the account lane: the same foreign loss the @@ -186,68 +211,68 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event var pollErr *eventfeed.PollError if !errors.As(err, &pollErr) { - w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "error", err) + w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) return nil, nil } switch pollErr.Kind { case eventfeed.PollGone: - if (cursor.PageURL != "" && cursor.PageURL == pollErr.ResumeURL) || pollErr.EpochAfterID == loss.RepairSince { - // The resume itself answered with the same 410. The gap is - // already recorded; following it again would loop. Leave it to the - // repair cadence. + epoch := pollErr.EpochAfterID + if pass.followed[pollErr.ResumeURL] { + // This pass already followed this resume, and it answered 410 + // again. Following it again would loop; the next pass tries. w.log.Warn("the repair walk's resume answered 410 again; retrying on the repair cadence", "loss_id", loss.ID) return nil, nil } - // History up to the epoch is gone, and the ids there with it. Ids - // above the epoch are still servable, so the resume is followed as - // served rather than condemning them too. - epoch := pollErr.EpochAfterID - if _, recordErr := w.ledger.RecordGap(ctx, Gap{ - DetectedAt: w.now(), - Class: GapEpoch, - EpochAfterID: &epoch, - EntryClass: entryClassOf(pollErr.ResumeURL), - Note: "the repair walk was seeded below the feed's epoch", - }); recordErr != nil { - return nil, recordErr - } - behind, markErr := w.ledger.MarkUnrecoveredThrough(ctx, loss.ID, epoch) - if markErr != nil { - return nil, markErr + if epoch != loss.RepairSince { + // A fence this loss has not seen: history up to the epoch is + // gone, and the ids there with it. + if _, recordErr := w.ledger.RecordGap(ctx, Gap{ + DetectedAt: w.now(), + Class: GapEpoch, + EpochAfterID: &epoch, + EntryClass: entryClassOf(pollErr.ResumeURL), + Note: "the repair walk was seeded below the feed's epoch", + }); recordErr != nil { + return nil, recordErr + } + behind, markErr := w.ledger.MarkUnrecoveredThrough(ctx, loss.ID, epoch) + if markErr != nil { + return nil, markErr + } + w.log.Error("the repair walk fell below the feed's epoch; the dropped events behind it are gone", + "loss_id", loss.ID, "epoch_after_id", epoch, "unrecovered", behind) + // Later passes start at the fence. + loss.RepairSince = epoch + loss.RepairCursor = "" + if setErr := w.ledger.SetRepairEntry(ctx, loss.ID, epoch); setErr != nil { + return nil, setErr + } } - w.log.Error("the repair walk fell below the feed's epoch; the dropped events behind it are gone", - "loss_id", loss.ID, "epoch_after_id", epoch, "unrecovered", behind) stillMissing, missErr := w.ledger.MissingIDs(ctx, loss.ID, LossMissing) if missErr != nil { return nil, missErr } - // Everything at or below the epoch is settled, so later passes start - // at the fence rather than walking into the same 410 again. - loss.RepairSince = epoch - loss.RepairCursor = "" - if setErr := w.ledger.SetRepairEntry(ctx, loss.ID, epoch); setErr != nil { - return nil, setErr - } if len(stillMissing) == 0 || pollErr.ResumeURL == "" { - // Every id was behind the fence, or there is no resume to follow: - // no repeat will serve anything more. if _, closeErr := w.ledger.CloseLoss(ctx, loss.ID, w.now()); closeErr != nil { return nil, closeErr } return nil, errReconciliationEnded } + // Ids above the epoch are still servable: the resume is followed as + // served, whether or not this loss had met the fence before. + pass.followed[pollErr.ResumeURL] = true return &eventfeed.Cursor{PageURL: pollErr.ResumeURL}, nil case eventfeed.PollFilterChanged, eventfeed.PollPositionInvalid: // The walk's own cursor was refused — minted under another filter // set, or no longer honored. Its explicit id is still good. Once per // pass: a refusal of the explicit id too is left to the cadence. - if *reentered || (cursor.Position == "" && cursor.PageURL == "") { - w.log.Warn("a repair poll was refused; retrying on the repair cadence", "loss_id", loss.ID, "error", err) + if pass.reentered || (cursor.Position == "" && cursor.PageURL == "") { + w.log.Warn("a repair poll was refused; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) return nil, nil } - *reentered = true + pass.reentered = true loss.RepairCursor = "" if saveErr := w.ledger.SaveRepairCursor(ctx, loss.ID, ""); saveErr != nil { return nil, saveErr @@ -257,7 +282,7 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event case eventfeed.PollTransient, eventfeed.PollThrottled, eventfeed.PollUnauthorized: // A reason to try again on the next repair poll, not a reason to call // the ids unrecovered. A slow or throttled walk delays nothing else. - w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "error", err) + w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) return nil, nil case eventfeed.PollFilterInvalid, eventfeed.PollRedirectRefused, eventfeed.PollUnrecoverable: @@ -268,7 +293,7 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event // The loss stays open, so the next start — perhaps after the cause is // fixed — tries again instead of finding the ids already condemned. w.log.Error("a repair poll failed in a way retrying will not fix; the loss stays open for the next start", - "loss_id", loss.ID, "error", err) + "loss_id", loss.ID, "failure", failureKind(err)) return nil, errReconciliationEnded } diff --git a/internal/connector/review2_test.go b/internal/connector/review2_test.go index 1f13156a4..c32a2bf09 100644 --- a/internal/connector/review2_test.go +++ b/internal/connector/review2_test.go @@ -202,9 +202,12 @@ func TestMembershipWatcherStopsWhenStopped(t *testing.T) { intake.opts.Membership = countingMembership{&reads} intake.opts.MembershipInterval = time.Millisecond - stop := intake.watchMembership(context.Background()) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + await := intake.watchMembership(ctx) require.Eventually(t, func() bool { return reads.Load() > 0 }, time.Second, time.Millisecond) stop() + await() after := reads.Load() time.Sleep(20 * time.Millisecond) assert.Equal(t, after, reads.Load(), "a stopped watcher reads nothing more") diff --git a/internal/connector/review_fixes_test.go b/internal/connector/review_fixes_test.go index 98a62df0f..4e4f70635 100644 --- a/internal/connector/review_fixes_test.go +++ b/internal/connector/review_fixes_test.go @@ -376,7 +376,9 @@ func TestARepeated410OnTheResumeEndsTheWalk(t *testing.T) { walker, _ := newTestWalker(t, ledger, polls, clock) require.NoError(t, walker.reconcile(ctx, loss)) - assert.LessOrEqual(t, polls.calls, 12, "a 410 on the resume is retried on the cadence, not in a tight loop") + // Eleven passes over the ten-minute window, each at most its entry and + // that entry's resume: bounded by the cadence, not a tight loop. + assert.LessOrEqual(t, polls.calls, 24, "a 410 on the resume is retried on the cadence, not in a tight loop") gaps, err := ledger.Gaps(ctx) require.NoError(t, err) assert.Len(t, gaps, 1, "one 410, one gap") From 99359c0b865eb72b061436128c1ebe633191e2cf Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 16:06:10 +0200 Subject: [PATCH 06/49] Re-enter a refused position safely even from empty pages, and serve nothing new over MCP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A position can be saved from empty pages alone, leaving the filter set's poll-served id at zero. If the server then refused it, intake fell back to another filter set's id — past events this one never served — or, with none, let the package re-enter at the present. Either lost the events committed after the checkpoint. The re-entry is now this filter set's own id, or the beginning of served history: a replay the ledger's dedupe absorbs, rather than a skip. The lineage id stays where it belongs, on the first entry after a filter change. This PR no longer registers an MCP domain. The SDK bump re-syncs the vendored model, which the provenance test requires, and that brought the feed operations into the catalogue. They are now withheld by the same policy exclusion basecamp-cli PR 726 introduces: the stream ticket is a replayable bearer credential the generic dispatcher would return verbatim, and the two poll operations lose their 409 and 410 fields through it. Serving them is PR 725's and 726's to do. Also from review: a repair pass follows at most two 410 resumes, so a server that nonces its resume URLs cannot make it loop; a ledger read that failed because the connection was canceled no longer aborts the run; queue callbacks run outside the lock that serializes the transitions, so a callback may use the queue; the ledger refuses a state outside its lifecycle, and a reason on a state that takes none. --- internal/commands/mcp_test.go | 2 +- internal/connector/feed_adapter.go | 4 +- internal/connector/intake.go | 58 ++++++---- internal/connector/ledger_events.go | 13 +++ internal/connector/queue.go | 18 +-- internal/connector/repair.go | 7 +- internal/connector/round4_test.go | 172 ++++++++++++++++++++++++++++ internal/mcpserver/server_test.go | 2 +- 8 files changed, 241 insertions(+), 35 deletions(-) create mode 100644 internal/connector/round4_test.go diff --git a/internal/commands/mcp_test.go b/internal/commands/mcp_test.go index 2caaf42c2..8933a2e1a 100644 --- a/internal/commands/mcp_test.go +++ b/internal/commands/mcp_test.go @@ -141,7 +141,7 @@ func TestMCPCommandServesMCP(t *testing.T) { require.NoError(t, err) names = append(names, tool.Name) } - assert.Len(t, names, 17, "tools = %v", names) + assert.Len(t, names, 16, "tools = %v", names) result, err := session.CallTool(context.Background(), &mcp.CallToolParams{ Name: "basecamp_projects", diff --git a/internal/connector/feed_adapter.go b/internal/connector/feed_adapter.go index b510e6612..f756803ff 100644 --- a/internal/connector/feed_adapter.go +++ b/internal/connector/feed_adapter.go @@ -228,7 +228,9 @@ func (a *FeedAdapter) optionsFor(cursor eventfeed.Cursor, filters eventfeed.Filt // A followed URL carries exactly one cursor. With neither, the // generated operation enters at the present — a silent skip of // everything unserved, on the path whose whole purpose was to - // continue. With both, the server decides which one wins. + // continue. With both, which one the server honors is not ours to + // know. Both refusals are deliberate: a server that starts sending + // either shape ends the feed loudly rather than moving it silently. return nil, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errCursorlessContinuation} } // The URL carries the server's own canonical filter set. It is used diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 985d5ea31..f5b0a65a7 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -114,9 +114,11 @@ type Intake struct { // id. Until it has, the package's own reset cursor is zero, and a refused // position would send it to the present. promotedThisRun bool - // reentryAfter is the safe re-entry the next connection takes after a - // refused stored position. - reentryAfter int64 + // reentry is the safe entry the next connection takes after a refused + // stored position; hasReentry says whether one is pending. + reentry eventfeed.Start + hasReentry bool + reentryLog string // abortErr ends the run: set when continuing could only mean entering // the feed somewhere unsafe. abortErr error @@ -253,8 +255,8 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { in.mu.Lock() in.cancelRun = cancel in.promotedThisRun = false - reentry := in.reentryAfter - in.reentryAfter = 0 + reentry, hasReentry, reentryLog := in.reentry, in.hasReentry, in.reentryLog + in.hasReentry = false in.mu.Unlock() defer func() { in.mu.Lock() @@ -278,9 +280,9 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { case since > 0: start = eventfeed.StartAfter(since) in.log.Info("entering the feed after an explicit event id", "since", since) - case reentry > 0: - start = eventfeed.StartAfter(reentry) - in.log.Warn("the stored position was refused; re-entering after the last poll-served id", "since", reentry) + case hasReentry: + start = reentry + in.log.Warn("the stored position was refused; " + reentryLog) case hadPosition: in.log.Info("resuming the feed from the stored position", "filter_key", in.key.FilterKey) case lineageServed > 0: @@ -681,8 +683,14 @@ func (in *Intake) noteBucket(bucketID int64) { // // The package re-enters from an in-memory id that starts at zero each run, so // right after a restart it would take the present and skip everything since -// the refused position. The ledger holds the id the poll lane had reached, so -// the connection is ended before that re-entry polls and remade after it. +// the refused position. The connection is ended before that re-entry polls, +// and remade at a safe entry: +// +// - this filter set's own last poll-served id, when it has one; +// - otherwise the beginning of served history. A position can be saved +// from empty pages alone, so "no poll-served id" does not mean "nothing +// to skip". Replaying costs reads the ledger's dedupe absorbs; entering at +// the present, or at another filter set's id, costs events. func (in *Intake) onPositionRejected(ctx context.Context) { in.mu.Lock() promoted := in.promotedThisRun @@ -692,26 +700,28 @@ func (in *Intake) onPositionRejected(ctx context.Context) { // is at least what the ledger holds. return } - // This filter set's own id first. Another set's may be past events this - // one never served, and re-entering there would skip them. served, err := in.positions.LastPollServedID(ctx, in.key) - if err == nil && served == 0 { - served, err = in.positions.LineagePollServedID(ctx, in.key) - } if err != nil { - // Returning here would let the package take its own reset cursor, - // which is the present. Not knowing where it is safe to re-enter is - // a reason to stop, not to guess. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // The connection is already ending — a reconnect or a shutdown — + // so the package's re-entry will not poll. Nothing to decide. + return + } + // Not knowing where it is safe to re-enter is a reason to stop, not + // to guess. in.abort(fmt.Errorf("connector: a position was refused and the safe re-entry could not be read: %w", err)) return } - if served == 0 { - // The poll lane has never served this consumer anything: the present - // is all there is. - return - } + in.mu.Lock() - in.reentryAfter = served + if served > 0 { + in.reentry = eventfeed.StartAfter(served) + in.reentryLog = "re-entering after this filter set's last poll-served id " + strconv.FormatInt(served, 10) + } else { + in.reentry = eventfeed.StartBeginning() + in.reentryLog = "no poll-served id for this filter set; re-entering at the beginning of served history" + } + in.hasReentry = true in.mu.Unlock() in.requestReconnect() } diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go index 72d61d01d..37c53ea49 100644 --- a/internal/connector/ledger_events.go +++ b/internal/connector/ledger_events.go @@ -145,6 +145,19 @@ func (l *Ledger) CountInState(ctx context.Context, state RecordState) (int, erro // every state but blocked and discarded — those two are the only ones a reason // explains. func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reason string) error { + switch state { + case StateBlocked, StateDiscarded: + if reason == "" { + return fmt.Errorf("connector: set state of %d: a %s record needs a reason", id, state) + } + case StateSeen, StateAdmitted, StateQueued, StateDispatched, StateCompleted: + if reason != "" { + return fmt.Errorf("connector: set state of %d: a %s record takes no reason", id, state) + } + default: + // A state outside the lifecycle is a row no recovery scan looks for. + return fmt.Errorf("connector: set state of %d: %q is not a ledger state", id, state) + } res, err := l.db.ExecContext(ctx, `UPDATE events SET state = ?, reason = ?, updated_at = ? WHERE id = ?`, string(state), reason, l.timestamp(), id) diff --git a/internal/connector/queue.go b/internal/connector/queue.go index e79501369..354034b65 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -121,19 +121,23 @@ func (q *Queue) Paused() bool { return q.waiting.Load() > 0 } // recovery is the other half of the pair, so a warning is never left standing // after the thing it warned about went away. func (q *Queue) noteDepth() { + // The transition is decided under the lock; the callback runs after it, + // so a callback may observe or use the queue without deadlocking against + // the operation that raised it. Callbacks can therefore arrive out of + // order across goroutines, but the state they report on never is. q.edges.Lock() - defer q.edges.Unlock() depth := q.Depth() + var fire func(int) switch { case depth >= q.warnAt && !q.warned: q.warned = true - if q.OnWarn != nil { - q.OnWarn(depth) - } + fire = q.OnWarn case depth < q.warnAt && q.warned: q.warned = false - if q.OnRecover != nil { - q.OnRecover(depth) - } + fire = q.OnRecover + } + q.edges.Unlock() + if fire != nil { + fire(depth) } } diff --git a/internal/connector/repair.go b/internal/connector/repair.go index f19be8d8f..0ddb6bc49 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -158,6 +158,11 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { // full recovery. var errReconciliationEnded = errors.New("connector: reconciliation ended inside the repair walk") +// maxResumesPerPass bounds the 410 resumes one pass follows. Keyed by URL +// alone, a server that signs or nonces its resume URLs would make every answer +// look new; the bound does not depend on the server choosing stable URLs. +const maxResumesPerPass = 2 + // repairPass is what one pass has already done, stated rather than inferred // from the shape of the cursor. type repairPass struct { @@ -218,7 +223,7 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event switch pollErr.Kind { case eventfeed.PollGone: epoch := pollErr.EpochAfterID - if pass.followed[pollErr.ResumeURL] { + if pass.followed[pollErr.ResumeURL] || len(pass.followed) >= maxResumesPerPass { // This pass already followed this resume, and it answered 410 // again. Following it again would loop; the next pass tries. w.log.Warn("the repair walk's resume answered 410 again; retrying on the repair cadence", "loss_id", loss.ID) diff --git a/internal/connector/round4_test.go b/internal/connector/round4_test.go new file mode 100644 index 000000000..64cc26144 --- /dev/null +++ b/internal/connector/round4_test.go @@ -0,0 +1,172 @@ +package connector + +import ( + "context" + + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// headPolls models a feed where one event was committed after the stored +// position. The stored position is refused; an entry at the present serves +// nothing; any entry in served history serves the event. +type headPolls struct { + mu sync.Mutex + cursors []eventfeed.Cursor +} + +func (h *headPolls) Poll(_ context.Context, cursor eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { + h.mu.Lock() + h.cursors = append(h.cursors, cursor) + h.mu.Unlock() + switch { + case cursor.Position == "checkpoint-from-empty-pages": + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollPositionInvalid} + case cursor.Since == "now" || (cursor.Since == "" && cursor.Position == "" && cursor.PageURL == ""): + return eventfeed.PollPage{Position: "head"}, nil + default: + return eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838600)}, Position: "after-the-event"}, nil + } +} + +// A position saved from empty pages leaves this filter set's poll-served id at +// zero. If the server then refuses it, the safe re-entry is the beginning of +// served history — not another filter set's id, and not the present. +func TestARefusedPositionWithNoPollServedIDReentersAtTheBeginningNotThePresent(t *testing.T) { + for _, lineage := range []int64{0, 17099838700} { + t.Run("lineage "+strconv.FormatInt(lineage, 10), func(t *testing.T) { + ledger := newTestLedger(t) + polls := &headPolls{} + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + intake, transport, minter, _ := newFeedIntake(t, ledger, Options{}) + intake.opts.Polls = polls + intake.queue = queue + for range 4 { + minter.ScriptTicket(ticket()) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "checkpoint-from-empty-pages")) + if lineage > 0 { + other := intake.CheckpointKey() + other.FilterKey = "srv2-0000000000000000" + require.NoError(t, ledger.NotePollServed(ctx, other, lineage)) + } + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 3*time.Second, 10*time.Millisecond, "the event committed after the refused checkpoint is lost") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") + }) + } +} + +// The ledger's lifecycle is closed; a state outside it, or a reason on a state +// that takes none, would be a durable row no recovery scan can find. +func TestSetStateRefusesAnythingOutsideTheLifecycle(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + + assert.Error(t, ledger.SetState(ctx, 1, RecordState("sen"), "")) + assert.Error(t, ledger.SetState(ctx, 1, StateAdmitted, "because")) + assert.Error(t, ledger.SetState(ctx, 1, StateBlocked, ""), "a blocked record says why") + assert.NoError(t, ledger.SetState(ctx, 1, StateBlocked, "read_failed")) + assert.NoError(t, ledger.SetState(ctx, 1, StateAdmitted, "")) +} + +// noncePolls answers every request with the epoch's 410, and each answer's +// resume differs only by a nonce — a server that signs its resume URLs. +type noncePolls struct { + mu sync.Mutex + calls int +} + +func (n *noncePolls) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + n.mu.Lock() + defer n.mu.Unlock() + n.calls++ + if n.calls > 500 { + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable} + } + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150&nonce=" + strconv.Itoa(n.calls)} +} + +// C3: a pass is bounded whatever URLs the server chooses. +func TestARepairPassIsBoundedWhenEveryResumeURLDiffers(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, time.Minute) + require.NoError(t, err) + + polls := &noncePolls{} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + assert.LessOrEqual(t, polls.calls, 10, "two passes, each a handful of polls, not a hot loop") +} + +// A ledger read that failed because the connection was canceled — by a +// reconnect — is not a reason to end the run. +func TestACanceledReadDuringAReconnectIsNotAnAbort(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.positions = canceledPositions{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + intake.onPositionRejected(ctx) + assert.NoError(t, intake.abortErr) +} + +type canceledPositions struct{} + +func (canceledPositions) LastPollServedID(ctx context.Context, _ eventfeed.CheckpointKey) (int64, error) { + return 0, ctx.Err() +} + +func (canceledPositions) LineagePollServedID(ctx context.Context, _ eventfeed.CheckpointKey) (int64, error) { + return 0, ctx.Err() +} + +// Queue callbacks may observe the queue. None of them runs while the queue +// holds a lock its own operations need. +func TestQueueCallbacksMayTouchTheQueue(t *testing.T) { + queue, err := NewQueue(1, 4) + require.NoError(t, err) + var depths []int + queue.OnWarn = func(int) { + ctxTake, stop := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer stop() + _, _ = queue.Take(ctxTake) + depths = append(depths, queue.Depth()) + } + + done := make(chan error, 1) + go func() { done <- queue.Offer(context.Background(), 1) }() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("a callback that takes from the queue deadlocked the offer") + } +} diff --git a/internal/mcpserver/server_test.go b/internal/mcpserver/server_test.go index 71bbf0673..cfdeceee8 100644 --- a/internal/mcpserver/server_test.go +++ b/internal/mcpserver/server_test.go @@ -41,7 +41,7 @@ func TestServerListsDomainTools(t *testing.T) { session := mcptest.Connect(t, srv.BuildMCPServer(slog.New(slog.DiscardHandler))) tools := mcptest.ListTools(t, session) - assert.Len(t, tools, 17) + assert.Len(t, tools, 16) require.Contains(t, tools, "basecamp_projects") projects := tools["basecamp_projects"] assert.Contains(t, projects.Description, "list_projects") From e1a91fc569af4ead8b9b7022d7aa064877741916 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 16:08:09 +0200 Subject: [PATCH 07/49] Retry the opening migration while SQLite reports the ledger busy Switching a fresh database into WAL mode takes a lock the busy handler is not consulted for, so two processes opening one new ledger at once could fail with SQLITE_BUSY. Seen three times in the concurrent-open test before this change; not reproducible on demand, so this carries no red-to-green proof. The migration is idempotent, and the retry is bounded at five seconds. --- internal/connector/invariants_test.go | 4 ++-- internal/connector/ledger.go | 26 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go index 7774cb628..b1c2bcec2 100644 --- a/internal/connector/invariants_test.go +++ b/internal/connector/invariants_test.go @@ -268,8 +268,8 @@ func TestInvariantE3MembershipComparison(t *testing.T) { func TestInvariantH2ConcurrentFreshOpensAllSucceed(t *testing.T) { path := filepath.Join(t.TempDir(), "state", "connector.db") var wg sync.WaitGroup - errs := make(chan error, 8) - for range 8 { + errs := make(chan error, 32) + for range 32 { wg.Add(1) go func() { defer wg.Done() diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index c10069e1c..7b98f44cf 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -94,7 +94,7 @@ func OpenLedger(path string) (*Ledger, error) { db.SetMaxOpenConns(1) l := &Ledger{db: db, now: time.Now} - if err := l.migrate(context.Background()); err != nil { + if err := retryBusy(func() error { return l.migrate(context.Background()) }); err != nil { _ = db.Close() return nil, err } @@ -110,6 +110,30 @@ func OpenLedger(path string) (*Ledger, error) { return l, nil } +// retryBusy retries fn while SQLite reports the database busy, for up to five +// seconds. +// +// The busy timeout covers ordinary contention, but not all of it: switching a +// fresh database into WAL mode takes a lock the busy handler is not consulted +// for, so two processes opening one new ledger at once — `status` beside a +// starting connector — can get SQLITE_BUSY immediately. The migration is +// idempotent, so trying again is safe. +func retryBusy(fn func() error) error { + deadline := time.Now().Add(5 * time.Second) + for { + err := fn() + if err == nil || !isBusy(err) || time.Now().After(deadline) { + return err + } + time.Sleep(20 * time.Millisecond) + } +} + +func isBusy(err error) bool { + msg := err.Error() + return strings.Contains(msg, "SQLITE_BUSY") || strings.Contains(msg, "database is locked") +} + // securePath makes the ledger private or refuses it. // // The ledger holds feed positions — signed tokens that resume the account's From b7cc7cf3e2d07cfd87172892c8a5dc001c76349b Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 16:14:07 +0200 Subject: [PATCH 08/49] Count what the poll lane served, and replay rather than skip on a filter change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last poll-served id counted delivered events. The package suppresses the poll copy of anything the live lane already delivered, and the poll lane runs about thirty seconds behind by design, so in steady state almost every poll page delivers nothing and the id stayed at zero. Every re-entry that needed it fell back to a full replay. Worse, a filter change with a position but no id entered the new filter set at the present, losing whatever followed the old position — the same loss the refused-position path was fixed for, on the other entry. The feed's poll source is now wrapped to record the highest id each page served, promoted when the package confirms that page delivered: served, not delivered, as the package counts for its own reset cursor. A filter change from a lineage with a position and no id replays from the beginning of served history. A 410 met by such a deliberate replay is recorded as expected, so status does not present it as a loss. The busy retry now reads SQLite's result code rather than matching message text. --- internal/connector/intake.go | 143 ++++++++++++++++++------ internal/connector/intake_test.go | 16 ++- internal/connector/ledger.go | 15 ++- internal/connector/ledger_checkpoint.go | 12 ++ internal/connector/round5_test.go | 93 +++++++++++++++ 5 files changed, 239 insertions(+), 40 deletions(-) create mode 100644 internal/connector/round5_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index f5b0a65a7..ba293d0b9 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -99,11 +99,9 @@ type Intake struct { positions pollServedReader mu sync.Mutex - // pollCandidates holds the ids delivered since the last page boundary that - // carry no push-lane transport fields. They become the last poll-served id - // only when a page boundary confirms a poll page actually landed. - pollCandidates []int64 - snapshot map[int64]bool + // served is the current connection's wrapped poll source. + served *servedPolls + snapshot map[int64]bool // learned holds buckets events proved visible that the lister did not // name. learned map[int64]bool @@ -118,7 +116,12 @@ type Intake struct { // stored position; hasReentry says whether one is pending. reentry eventfeed.Start hasReentry bool - reentryLog string + // replaying is whether this connection entered at the beginning of served + // history on purpose, to recover. + replaying bool + // reentryReplays is whether the pending re-entry is a replay. + reentryReplays bool + reentryLog string // abortErr ends the run: set when continuing could only mean entering // the feed somewhere unsafe. abortErr error @@ -255,8 +258,9 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { in.mu.Lock() in.cancelRun = cancel in.promotedThisRun = false - reentry, hasReentry, reentryLog := in.reentry, in.hasReentry, in.reentryLog + reentry, hasReentry, reentryLog, reentryReplays := in.reentry, in.hasReentry, in.reentryLog, in.reentryReplays in.hasReentry = false + in.replaying = false in.mu.Unlock() defer func() { in.mu.Lock() @@ -274,6 +278,10 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { if err != nil { return err } + lineageHasPosition, err := in.ledger.LineageHasPosition(runCtx, in.key) + if err != nil { + return err + } start := eventfeed.StartResume() switch { @@ -282,6 +290,7 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { in.log.Info("entering the feed after an explicit event id", "since", since) case hasReentry: start = reentry + in.setReplaying(reentryReplays) in.log.Warn("the stored position was refused; " + reentryLog) case hadPosition: in.log.Info("resuming the feed from the stored position", "filter_key", in.key.FilterKey) @@ -292,6 +301,15 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { start = eventfeed.StartAfter(lineageServed) in.log.Warn("no position for this filter set; re-entering after the last poll-served id under the previous one", "since", lineageServed, "filter_key", in.key.FilterKey) + case lineageHasPosition: + // A filter change from a set that had read up to a position but + // recorded no served id. Entering at the present would skip what + // followed that position; replaying from the beginning costs reads + // the ledger absorbs. + start = eventfeed.StartBeginning() + in.setReplaying(true) + in.log.Warn("no position or poll-served id for this filter set, but the previous one had read up to a position; replaying from the beginning of served history", + "filter_key", in.key.FilterKey) default: // Said out loud because it is a real loss of history, not a neutral // default: everything committed before this moment is never served. @@ -312,7 +330,12 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { options = append(options, eventfeed.WithTransport(in.opts.Transport)) } - feed, err := eventfeed.New(in.key.Origin, in.opts.AccountID, in.opts.Minter, in.opts.Polls, options...) + served := &servedPolls{inner: in.opts.Polls} + in.mu.Lock() + in.served = served + in.mu.Unlock() + + feed, err := eventfeed.New(in.key.Origin, in.opts.AccountID, in.opts.Minter, served, options...) if err != nil { return fmt.Errorf("connector: build feed: %w", err) } @@ -374,9 +397,6 @@ func (in *Intake) ingest(ctx context.Context, event eventfeed.Event, lane Lane) // had been. return err } - if lane == LanePoll { - in.notePollCandidate(event.ID) - } if !fresh { // The ordinary case: the poll lane serving what the live lane already // delivered, or a restart re-walking a page. Dedupe is the point. @@ -404,11 +424,9 @@ func (in *Intake) ingest(ctx context.Context, event eventfeed.Event, lane Lane) // the vocabulary, and a *bool — rather than defaulting them, precisely so this // is answerable. // -// It is used for one thing that matters, the last poll-served id, and it is -// paired there with a page boundary: absence of the push fields nominates an -// id, a delivered poll page confirms it. Erring towards "live" only costs -// duplicates the ledger absorbs; erring towards "poll" would move the re-entry -// past events the poll lane had not served. +// It labels records and pointer lines. It decides nothing about re-entry: the +// last poll-served id is taken from what the poll source served, not from +// guessing which lane a delivery came from. func LaneOf(event eventfeed.Event) Lane { if event.ActorType == "" && event.VisibleToClients == nil { return LanePoll @@ -416,25 +434,69 @@ func LaneOf(event eventfeed.Event) Lane { return LaneLive } -func (in *Intake) notePollCandidate(id int64) { - in.mu.Lock() - defer in.mu.Unlock() - in.pollCandidates = append(in.pollCandidates, id) -} +// servedPolls wraps the feed's poll source to record what each page SERVED. +// +// The last poll-served id has to count served events, not delivered ones. The +// package suppresses the poll copy of any event the live lane already +// delivered, and the poll lane runs about thirty seconds behind the live one +// by design, so in steady state nearly every poll page delivers nothing. +// Counting deliveries would leave the id at zero, and every re-entry that +// needs it would fall back to a full replay. This is what the package counts +// for its own reset cursor too. +// +// The repair walk does not go through here: its pages are not the feed's +// position and must never advance the feed's re-entry. +type servedPolls struct { + inner eventfeed.PollSource -// confirmPollServed promotes the candidates a delivered poll page confirms. -func (in *Intake) confirmPollServed(ctx context.Context) { - in.mu.Lock() - candidates := in.pollCandidates - in.pollCandidates = nil - in.mu.Unlock() + mu sync.Mutex + // byPosition holds the highest id a page served, keyed by the position + // that page issued, until the package confirms the page was delivered. + byPosition map[string]int64 +} +func (s *servedPolls) Poll(ctx context.Context, cursor eventfeed.Cursor, filters eventfeed.Filters) (eventfeed.PollPage, error) { + page, err := s.inner.Poll(ctx, cursor, filters) + if err != nil || page.Position == "" { + return page, err + } var highest int64 - for _, id := range candidates { - if id > highest { - highest = id + for _, event := range page.Events { + highest = max(highest, event.ID) + } + if highest > 0 { + s.mu.Lock() + if s.byPosition == nil { + s.byPosition = make(map[string]int64) } + s.byPosition[page.Position] = max(s.byPosition[page.Position], highest) + s.mu.Unlock() } + return page, err +} + +// confirmed returns the highest id served by the page that issued position, +// and forgets every page served before it: pages are polled and delivered one +// at a time, so nothing earlier can still be waiting. +func (s *servedPolls) confirmed(position string) int64 { + s.mu.Lock() + defer s.mu.Unlock() + highest := s.byPosition[position] + clear(s.byPosition) + return highest +} + +// confirmPollServed records the served id of a page the package has just +// finished delivering. Only then: an ingest that failed partway through a page +// leaves the page undelivered and its ids unrecorded. +func (in *Intake) confirmPollServed(ctx context.Context, position string) { + in.mu.Lock() + served := in.served + in.mu.Unlock() + if served == nil { + return + } + highest := served.confirmed(position) if highest == 0 { // An empty page. Ordinary — the walk crossed rows the filters exclude // — and it serves no id, so it advances nothing here. @@ -457,11 +519,11 @@ func (in *Intake) observer(ctx context.Context) eventfeed.Observer { in.log.Warn("feed socket disconnected", "reason", reason, "error", err) }, CatchUpStarted: func(eventfeed.Cursor) { in.log.Info("feed catch-up walk started") }, - PageDelivered: func(int, string) { + PageDelivered: func(_ int, position string) { // Detached deliberately: the position the page just moved must be // recorded even if the run's context is on its way down, or a // shutdown mid-page loses the id the next re-entry needs. - in.confirmPollServed(context.WithoutCancel(ctx)) //nolint:contextcheck // detached on purpose, see above + in.confirmPollServed(context.WithoutCancel(ctx), position) //nolint:contextcheck // detached on purpose, see above }, CaughtUp: func() { // "Caught up with the walk", not "caught up with the account": @@ -528,12 +590,20 @@ func (in *Intake) handleSignal(signal eventfeed.Signal) eventfeed.Disposition { // is followed exactly as served — the entry class is the server's // decision, read out of its cursor, never substituted. epoch := s.EpochAfterID + note := "the feed's served history before the epoch is gone" + in.mu.Lock() + if in.replaying { + // Expected, not a loss: this connection chose to replay from the + // beginning to recover, and the beginning is below the epoch. + note = "a recovery replay from the beginning of served history met the epoch, as expected; not a loss" + } + in.mu.Unlock() if _, err := in.ledger.RecordGap(ctx, Gap{ DetectedAt: in.now(), Class: GapEpoch, EpochAfterID: &epoch, EntryClass: entryClassOf(s.ResumeURL), - Note: "the feed's served history before the epoch is gone", + Note: note, }); err != nil { // A gap we cannot write down is a gap nothing will ever report. in.log.Error("could not record the feed gap; refusing to continue past it", "error", err) @@ -721,11 +791,18 @@ func (in *Intake) onPositionRejected(ctx context.Context) { in.reentry = eventfeed.StartBeginning() in.reentryLog = "no poll-served id for this filter set; re-entering at the beginning of served history" } + in.reentryReplays = served == 0 in.hasReentry = true in.mu.Unlock() in.requestReconnect() } +func (in *Intake) setReplaying(replaying bool) { + in.mu.Lock() + in.replaying = replaying + in.mu.Unlock() +} + // abort ends the current connection and the run with err. func (in *Intake) abort(err error) { in.mu.Lock() diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go index 5f8d9a376..933f8d18d 100644 --- a/internal/connector/intake_test.go +++ b/internal/connector/intake_test.go @@ -122,7 +122,12 @@ func TestPointerLineCarriesNoContent(t *testing.T) { } func TestPollServedIDAdvancesOnlyOnAPageBoundary(t *testing.T) { + polls := &scriptedPolls{pages: []eventfeed.PollPage{{ + Events: []eventfeed.Event{testEvent(17099838500)}, + Position: "p1", + }}} intake, ledger, _ := newTestIntake(t, nil, nil) + intake.served = &servedPolls{inner: polls} ctx := context.Background() live := testEvent(17099999999) @@ -130,17 +135,20 @@ func TestPollServedIDAdvancesOnlyOnAPageBoundary(t *testing.T) { visible := true live.VisibleToClients = &visible require.NoError(t, intake.ingest(ctx, live, LaneOf(live))) + intake.confirmPollServed(ctx, "some-other-position") - intake.confirmPollServed(ctx) served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) require.NoError(t, err) assert.Zero(t, served, "a live id is far ahead of the poll lane; re-entering at one skips everything inside the safety delay") - polled := testEvent(17099838500) - require.NoError(t, intake.ingest(ctx, polled, LaneOf(polled))) - intake.confirmPollServed(ctx) + page, err := intake.served.Poll(ctx, eventfeed.Cursor{}, eventfeed.Filters{}) + require.NoError(t, err) + served, err = ledger.LastPollServedID(ctx, intake.CheckpointKey()) + require.NoError(t, err) + assert.Zero(t, served, "a page served but not yet delivered records nothing") + intake.confirmPollServed(ctx, page.Position) served, err = ledger.LastPollServedID(ctx, intake.CheckpointKey()) require.NoError(t, err) assert.Equal(t, int64(17099838500), served) diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index 7b98f44cf..f475ba5fb 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -11,7 +11,8 @@ import ( "strings" "time" - _ "modernc.org/sqlite" // database/sql driver "sqlite", pure Go: no cgo on any of the five release targets. + "modernc.org/sqlite" // database/sql driver "sqlite", pure Go: no cgo on any of the five release targets. + sqlite3 "modernc.org/sqlite/lib" ) // RecordState is where an event sits in the ledger's lifecycle. @@ -130,8 +131,16 @@ func retryBusy(fn func() error) error { } func isBusy(err error) bool { - msg := err.Error() - return strings.Contains(msg, "SQLITE_BUSY") || strings.Contains(msg, "database is locked") + var sqliteErr *sqlite.Error + if !errors.As(err, &sqliteErr) { + return false + } + // The primary result code, without the extended bits. + switch sqliteErr.Code() & 0xff { + case sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED: + return true + } + return false } // securePath makes the ledger private or refuses it. diff --git a/internal/connector/ledger_checkpoint.go b/internal/connector/ledger_checkpoint.go index 7650b3040..2a973dc03 100644 --- a/internal/connector/ledger_checkpoint.go +++ b/internal/connector/ledger_checkpoint.go @@ -108,6 +108,18 @@ func (l *Ledger) LineagePollServedID(ctx context.Context, key eventfeed.Checkpoi return id, nil } +// LineageHasPosition reports whether this consumer holds a position under any +// filter set. +func (l *Ledger) LineageHasPosition(ctx context.Context, key eventfeed.CheckpointKey) (bool, error) { + var n int + err := l.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM checkpoints WHERE lineage = ? AND position <> ''`, lineageOf(key)).Scan(&n) + if err != nil { + return false, fmt.Errorf("connector: read lineage positions: %w", err) + } + return n > 0, nil +} + // lineageOf is the checkpoint identity without its filter digest. func lineageOf(key eventfeed.CheckpointKey) string { key.FilterKey = "" diff --git a/internal/connector/round5_test.go b/internal/connector/round5_test.go new file mode 100644 index 000000000..94dde52d4 --- /dev/null +++ b/internal/connector/round5_test.go @@ -0,0 +1,93 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// A3: the last poll-served id counts what the poll lane SERVED, not what it +// delivered. In steady state the poll copy of nearly every event is +// suppressed, because the live lane delivered it thirty seconds earlier; +// counting deliveries leaves the id at zero and every later re-entry falls +// back to a full replay — or, on a filter change, to the present. +func TestThePollServedIDCountsServedEventsTheLiveLaneAlreadyDelivered(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{RepairInterval: 50 * time.Millisecond}) + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838600)}, Position: "p2"}) + for range 20 { + polls.ScriptPage(eventfeed.PollPage{Position: "p2"}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + conn, identifier := subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() >= 1 }, 5*time.Second, time.Millisecond) + conn.Serve(pushFrame(t, identifier, testEvent(17099838600))) + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 5*time.Second, time.Millisecond) + + require.Eventually(t, func() bool { + served, err := ledger.LastPollServedID(ctx, intake.CheckpointKey()) + return err == nil && served == 17099838600 + }, 5*time.Second, 5*time.Millisecond, "the repair poll served the event; its delivery being suppressed does not unserve it") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A3: a filter change enters under the new digest at the old lineage's +// poll-served id — and, when the lineage has a position but no id, at the +// beginning of served history. Never at the present. +func TestAFilterChangeFromAPositionWithNoServedIDReplaysInsteadOfEnteringAtThePresent(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Filters: eventfeed.Filters{Types: []string{"comment.created"}}, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + old := intake.CheckpointKey() + old.FilterKey = eventfeed.Filters{}.FilterKey() + require.NoError(t, ledger.Save(ctx, old, "old-digest-position-from-empty-pages")) + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond) + first := polls.Calls()[0].Cursor + assert.Equal(t, "0", first.Since, "the old filter set had read up to a position; entering at the present skips what followed it") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// A gap met by a deliberate recovery replay is labeled as one, so status does +// not present a replay's expected 410 as a loss. +func TestAGapMetByARecoveryReplayIsLabeledAsOne(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + intake.replaying = true + + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: 17099838487, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487", + })) + gaps, err := ledger.Gaps(context.Background()) + require.NoError(t, err) + require.Len(t, gaps, 1) + assert.Contains(t, gaps[0].Note, "replay") +} From f58d62f78ccc9c1f952491b902922359fc1dea8a Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 16:18:41 +0200 Subject: [PATCH 09/49] Let the replay label cover only the replay A connection that entered by a recovery replay kept the label for its whole life, so a genuine epoch move days later was recorded as an expected, harmless gap. The label now ends with the first gap it explains, or when the replay reaches the head. --- internal/connector/intake.go | 8 +++++++- internal/connector/round5_test.go | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index ba293d0b9..12c2bd700 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -526,6 +526,9 @@ func (in *Intake) observer(ctx context.Context) eventfeed.Observer { in.confirmPollServed(context.WithoutCancel(ctx), position) //nolint:contextcheck // detached on purpose, see above }, CaughtUp: func() { + // A replay that reached the head without meeting the epoch is + // over; nothing after this is part of it. + in.setReplaying(false) // "Caught up with the walk", not "caught up with the account": // delivery has write-time brakes that write no addressing and say // nothing, so a quiet feed is never proof of a quiet project. @@ -594,8 +597,11 @@ func (in *Intake) handleSignal(signal eventfeed.Signal) eventfeed.Disposition { in.mu.Lock() if in.replaying { // Expected, not a loss: this connection chose to replay from the - // beginning to recover, and the beginning is below the epoch. + // beginning to recover, and the beginning is below the epoch. The + // label covers that one entry; any later 410 on this connection is + // a real gap and is recorded as one. note = "a recovery replay from the beginning of served history met the epoch, as expected; not a loss" + in.replaying = false } in.mu.Unlock() if _, err := in.ledger.RecordGap(ctx, Gap{ diff --git a/internal/connector/round5_test.go b/internal/connector/round5_test.go index 94dde52d4..9beeaa3c5 100644 --- a/internal/connector/round5_test.go +++ b/internal/connector/round5_test.go @@ -2,6 +2,7 @@ package connector import ( "context" + "strconv" "testing" "time" @@ -91,3 +92,22 @@ func TestAGapMetByARecoveryReplayIsLabeledAsOne(t *testing.T) { require.Len(t, gaps, 1) assert.Contains(t, gaps[0].Note, "replay") } + +// The replay label covers the replay's own entry, not the life of the +// connection: a later, genuine epoch move must not be recorded as expected. +func TestTheReplayLabelCoversOnlyTheReplaysOwnGap(t *testing.T) { + intake, ledger, _ := newTestIntake(t, nil, nil) + intake.replaying = true + + for _, epoch := range []int64{100, 17099838000} { + assert.Equal(t, eventfeed.Accept, intake.handleSignal(eventfeed.FeedGap{ + EpochAfterID: epoch, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=" + strconv.FormatInt(epoch, 10), + })) + } + gaps, err := ledger.Gaps(context.Background()) + require.NoError(t, err) + require.Len(t, gaps, 2) + assert.Contains(t, gaps[0].Note, "replay") + assert.NotContains(t, gaps[1].Note, "not a loss", "a later epoch move is a real gap") +} From 4bf12ba299f784ddeb9012a0f509fe6dad221704 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 16:23:40 +0200 Subject: [PATCH 10/49] Deliver queue edges in order, report partial reconciliation honestly, keep timestamp precision Queue warning callbacks ran off the lock in whatever order goroutines reached them, so a drained queue could end on a warning. Edges are now decided into one ordered list and drained by one goroutine at a time, with the lock released around each callback, so a callback may still use the queue. A loss whose ids were partly fenced off by an epoch was logged as fully reconciled once nothing was missing. The settle step now says when ids were left unrecovered. The pointer line wrote the event's timestamp at second precision; it now keeps what the feed gave. --- internal/connector/intake.go | 2 +- internal/connector/queue.go | 42 ++++++++++---- internal/connector/repair.go | 12 +++- internal/connector/round6_test.go | 95 +++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 12 deletions(-) create mode 100644 internal/connector/round6_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 12c2bd700..768825fd7 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -955,7 +955,7 @@ func (p *pointerWriter) write(event eventfeed.Event, lane Lane) error { CreatorID: event.CreatorID, PerformedByID: event.PerformedByID, RecordingID: event.RecordingID, - CreatedAt: event.CreatedAt.UTC().Format(time.RFC3339), + CreatedAt: event.CreatedAt.UTC().Format(time.RFC3339Nano), Lane: lane, State: string(StateSeen), }) diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 354034b65..afef468e8 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -36,6 +36,10 @@ type Queue struct { // warning that no later event will clear. edges sync.Mutex warned bool + // pending holds edges decided but not yet delivered, in the order they + // were decided; delivering says a goroutine is already draining them. + pending []queueEdge + delivering bool // waiting counts offers blocked for room. Intake and every open repair // walk offer concurrently, so a single flag would be cleared by the first // waiter to resume while another — perhaps the feed — still waits. @@ -121,23 +125,41 @@ func (q *Queue) Paused() bool { return q.waiting.Load() > 0 } // recovery is the other half of the pair, so a warning is never left standing // after the thing it warned about went away. func (q *Queue) noteDepth() { - // The transition is decided under the lock; the callback runs after it, - // so a callback may observe or use the queue without deadlocking against - // the operation that raised it. Callbacks can therefore arrive out of - // order across goroutines, but the state they report on never is. + // The transition is decided under the lock, and appended to one ordered + // list of edges. Exactly one goroutine at a time delivers that list, in + // order, with the lock released around each callback: an operator sees the + // edges in the order the state took them, and a callback may use the queue + // — its own edge is queued behind, and delivered by, the drain already + // running. q.edges.Lock() depth := q.Depth() - var fire func(int) switch { case depth >= q.warnAt && !q.warned: q.warned = true - fire = q.OnWarn + q.pending = append(q.pending, queueEdge{fire: q.OnWarn, depth: depth}) case depth < q.warnAt && q.warned: q.warned = false - fire = q.OnRecover + q.pending = append(q.pending, queueEdge{fire: q.OnRecover, depth: depth}) } - q.edges.Unlock() - if fire != nil { - fire(depth) + if q.delivering { + q.edges.Unlock() + return + } + q.delivering = true + for len(q.pending) > 0 { + edge := q.pending[0] + q.pending = q.pending[1:] + q.edges.Unlock() + if edge.fire != nil { + edge.fire(edge.depth) + } + q.edges.Lock() } + q.delivering = false + q.edges.Unlock() +} + +type queueEdge struct { + fire func(int) + depth int } diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 0ddb6bc49..8b1f2502b 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -95,7 +95,17 @@ func (w *repairWalker) settled(ctx context.Context, loss Loss) (bool, error) { if _, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()); err != nil { return false, err } - w.log.Info("a buffer overflow was fully reconciled", "loss_id", loss.ID) + // "Nothing missing" is not "everything recovered": an epoch can have + // fenced some ids off as unrecovered already. Say which it was. + unrecovered, err := w.ledger.MissingIDs(ctx, loss.ID, LossUnrecovered) + if err != nil { + return false, err + } + if len(unrecovered) > 0 { + w.log.Warn("a buffer overflow's reconciliation ended with events unrecovered", "loss_id", loss.ID, "unrecovered", len(unrecovered)) + } else { + w.log.Info("a buffer overflow was fully reconciled", "loss_id", loss.ID) + } return true, nil } diff --git a/internal/connector/round6_test.go b/internal/connector/round6_test.go new file mode 100644 index 000000000..16bc273df --- /dev/null +++ b/internal/connector/round6_test.go @@ -0,0 +1,95 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// F2: an operator sees warning edges in the order the state took them. +// Delivered out of order, a drained queue can end on a warning. +func TestQueueWarningEdgesArriveInTheOrderTheyHappened(t *testing.T) { + for range 30 { + queue, err := NewQueue(1, 64) + require.NoError(t, err) + var mu sync.Mutex + var seen []string + queue.OnWarn = func(int) { mu.Lock(); seen = append(seen, "warn"); mu.Unlock() } + queue.OnRecover = func(int) { mu.Lock(); seen = append(seen, "recover"); mu.Unlock() } + + ctx := context.Background() + var wg sync.WaitGroup + for g := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 300 { + require.NoError(t, queue.Offer(ctx, int64(g*1000+i))) + _, err := queue.Take(ctx) + require.NoError(t, err) + } + }() + } + wg.Wait() + + mu.Lock() + for i, edge := range seen { + want := "warn" + if i%2 == 1 { + want = "recover" + } + require.Equal(t, want, edge, "edge %d of %v", i, len(seen)) + } + if len(seen) > 0 { + require.Equal(t, "recover", seen[len(seen)-1], "a drained queue's last word is a recovery") + } + mu.Unlock() + } +} + +// C2: a loss that ended with ids behind the epoch is not reported as fully +// reconciled. +func TestAPartlyUnrecoveredLossIsNotReportedAsFullyReconciled(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=150"}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(200)}, Position: "p"}}, + } + var logs bytes.Buffer + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.log = slog.New(slog.NewTextHandler(&logs, nil)) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.NotContains(t, logs.String(), "fully reconciled", "id 100 is behind the epoch and was never recovered") +} + +// The pointer line keeps the event's timestamp at the precision the feed gave. +func TestPointerLineKeepsTheTimestampsPrecision(t *testing.T) { + var pointers bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &pointers) + event := testEvent(1) + event.CreatedAt = time.Date(2026, 9, 16, 10, 0, 0, 120_000_000, time.UTC) + require.NoError(t, intake.ingest(context.Background(), event, LanePoll)) + + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(pointers.String())), &pointer)) + parsed, err := time.Parse(time.RFC3339Nano, pointer.CreatedAt) + require.NoError(t, err) + assert.True(t, parsed.Equal(event.CreatedAt), "got %s", pointer.CreatedAt) +} From 3279414373f7b82859cfd362d1e347b98284582c Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 16:30:46 +0200 Subject: [PATCH 11/49] Store ledger timestamps at fixed width, so retention compares them in time order RFC3339Nano trims trailing zeros, so as SQLite text a whole-second stamp sorted after a later fractional one in the same second, and DropContent kept content past its window. Every ledger timestamp is now written as UTC with all nine fractional digits. The schema has not shipped, so no migration is needed. Also rejoins a standard-library import group split by a blank line. gofmt accepts that, and no enabled linter checks import grouping, which is why make check passed. --- internal/connector/ledger.go | 9 ++++++++- internal/connector/ledger_events.go | 6 +++--- internal/connector/ledger_recovery.go | 2 +- internal/connector/round4_test.go | 1 - internal/connector/round7_test.go | 29 +++++++++++++++++++++++++++ 5 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 internal/connector/round7_test.go diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index f475ba5fb..149687442 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -316,4 +316,11 @@ func (l *Ledger) SchemaVersion(ctx context.Context) (int, error) { return version, err } -func (l *Ledger) timestamp() string { return l.now().UTC().Format(time.RFC3339Nano) } +func (l *Ledger) timestamp() string { return stamp(l.now()) } + +// ledgerTime is the one format every ledger timestamp is stored in: UTC, with +// all nine fractional digits. Fixed width is what makes SQLite's text +// comparison agree with time order. time.RFC3339Nano trims trailing zeros, so +// "12:00:00Z" would sort after the later "12:00:00.5Z" and retention would +// keep a record past its window. +const ledgerTime = "2006-01-02T15:04:05.000000000Z07:00" diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go index 37c53ea49..3ef86b43f 100644 --- a/internal/connector/ledger_events.go +++ b/internal/connector/ledger_events.go @@ -75,7 +75,7 @@ ON CONFLICT (id) DO NOTHING`, ev.ID, string(StateSeen), string(lane), ev.EventType, ev.Kind, ev.Action, ev.BucketID, ev.CreatorID, ev.PerformedByID, ev.RecordingID, detailsArg(ev.Details), ev.ActorType, ev.VisibleToClients, - ev.CreatedAt.UTC().Format(time.RFC3339Nano), now, now) + stamp(ev.CreatedAt), now, now) if err != nil { return false, fmt.Errorf("connector: record seen %d: %w", ev.ID, err) } @@ -192,8 +192,8 @@ SET details = NULL, event_type = '', kind = '', action = '', bucket_id = 0, visible_to_clients = NULL, content_dropped = 1, updated_at = updated_at WHERE content_dropped = 0 AND ((state = ? AND updated_at < ?) OR (state = ? AND updated_at < ?))`, - string(StateDiscarded), discardedBefore.UTC().Format(time.RFC3339Nano), - string(StateCompleted), completedBefore.UTC().Format(time.RFC3339Nano)) + string(StateDiscarded), stamp(discardedBefore), + string(StateCompleted), stamp(completedBefore)) if err != nil { return 0, fmt.Errorf("connector: drop content: %w", err) } diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go index e393f5984..ee107db7f 100644 --- a/internal/connector/ledger_recovery.go +++ b/internal/connector/ledger_recovery.go @@ -389,7 +389,7 @@ func (l *Ledger) Gaps(ctx context.Context) ([]Gap, error) { return gaps, rows.Err() } -func stamp(t time.Time) string { return t.UTC().Format(time.RFC3339Nano) } +func stamp(t time.Time) string { return t.UTC().Format(ledgerTime) } func nullableStamp(t *time.Time) any { if t == nil { diff --git a/internal/connector/round4_test.go b/internal/connector/round4_test.go index 64cc26144..f000fe798 100644 --- a/internal/connector/round4_test.go +++ b/internal/connector/round4_test.go @@ -2,7 +2,6 @@ package connector import ( "context" - "strconv" "sync" "testing" diff --git a/internal/connector/round7_test.go b/internal/connector/round7_test.go new file mode 100644 index 000000000..eb77b57c3 --- /dev/null +++ b/internal/connector/round7_test.go @@ -0,0 +1,29 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Retention compares timestamps. Stored as variable-width text, a whole-second +// stamp ("…00Z") sorts after a later fractional one ("…00.5Z") in the same +// second, and a record past its window is kept. +func TestDropContentDropsARecordExpiredWithinTheSameSecond(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + whole := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + ledger.now = func() time.Time { return whole } + + _, err := ledger.RecordSeen(ctx, testEvent(42), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 42, StateDiscarded, "untrusted_author")) + + cutoff := whole.Add(500 * time.Millisecond) + dropped, err := ledger.DropContent(ctx, cutoff, cutoff) + require.NoError(t, err) + assert.Equal(t, 1, dropped, "updated at 12:00:00.000, cutoff 12:00:00.500: expired") +} From 8226ec96a3154fba0707ded3ea2b5127565a753a Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 16:37:53 +0200 Subject: [PATCH 12/49] Test ledger retention in both directions across fractional-second widths A record updated just after a whole-second cutoff must be kept, as well as one updated just before a fractional cutoff being dropped; both failed with the variable-width stamps f728b71 replaced. DropContent's updated_at comparison is the only timestamp the package compares in SQL; every ORDER BY is on an integer id. --- internal/connector/round7_test.go | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/internal/connector/round7_test.go b/internal/connector/round7_test.go index eb77b57c3..2b5e50945 100644 --- a/internal/connector/round7_test.go +++ b/internal/connector/round7_test.go @@ -27,3 +27,37 @@ func TestDropContentDropsARecordExpiredWithinTheSameSecond(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, dropped, "updated at 12:00:00.000, cutoff 12:00:00.500: expired") } + +// The other direction: a record updated half a second AFTER a whole-second +// cutoff is inside its window and must be kept. As variable-width text, +// "…00.5Z" < "…00Z", so it would be purged. +func TestDropContentKeepsARecordUpdatedJustAfterAWholeSecondCutoff(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + cutoff := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + ledger.now = func() time.Time { return cutoff.Add(500 * time.Millisecond) } + + _, err := ledger.RecordSeen(ctx, testEvent(43), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 43, StateCompleted, "")) + + dropped, err := ledger.DropContent(ctx, cutoff, cutoff) + require.NoError(t, err) + assert.Zero(t, dropped, "updated at 12:00:00.500, cutoff 12:00:00.000: still inside the window") +} + +// Every stored timestamp has one width, which is the property both directions +// rest on. +func TestLedgerTimestampsHaveOneWidth(t *testing.T) { + for _, at := range []time.Time{ + time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + time.Date(2026, 9, 16, 12, 0, 0, 500_000_000, time.FixedZone("CEST", 2*3600)), + time.Date(2026, 9, 16, 12, 0, 0, 123_456_789, time.UTC), + } { + s := stamp(at) + assert.Len(t, s, len("2006-01-02T15:04:05.000000000Z"), s) + parsed, err := parseStamp(s) + require.NoError(t, err) + assert.True(t, parsed.Equal(at)) + } +} From 725871197b6eaa83ea239357f08f61054f257295 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 21:43:54 +0200 Subject: [PATCH 13/49] Take the SDK's live feed seams now that they have landed basecamp-sdk 897, 899 and 912 are on main, so the two swap points this PR marked are taken, and the temporary adapter is deleted. Intake binds to eventfeed.NewLive through LiveOptions. The SDK's seams now own what the adapter did: - a redirect guard that answers every 3xx of a feed call at the wire, so nothing reaches a redirect target - the two 410s as two typed errors: FeedPositionGoneError with its epoch required, InboxPositionGoneError with its since=0 resume, refused on the account lane rather than flattened to epoch 0, and a resume that does not re-enter at its declared fence refused too - the 400's reason, with an unnamed reason surfaced rather than guessed - exactly one cursor on every followed URL, and caller cancellation passed through unclassified Tests at the real NewLive seam pin that contract. Each fails against a scratch copy of the SDK with the handling removed. One check stays in intake: the repair walk follows next and resume URLs outside the SDK connector, whose same-origin validation is unexported, so the walk holds them to the API origin itself. The SDK's live poll source keeps walk state, so the feed's connection and each repair walk now get their own. SDK bumped to main at 12cbad5b; the vendored MCP model, still withholding the feed operations, and the Nix vendorHash are re-synced. --- internal/connector/feed_adapter.go | 442 ----------------------- internal/connector/feed_adapter_test.go | 256 ------------- internal/connector/intake.go | 61 ++-- internal/connector/intake_test.go | 20 - internal/connector/invariants_test.go | 39 -- internal/connector/live_adapter_test.go | 46 --- internal/connector/repair.go | 60 +-- internal/connector/repair_test.go | 1 + internal/connector/review2_test.go | 26 -- internal/connector/review_fixes_test.go | 67 +++- internal/connector/round4_test.go | 2 +- internal/connector/seam_contract_test.go | 152 ++++++++ 12 files changed, 274 insertions(+), 898 deletions(-) delete mode 100644 internal/connector/feed_adapter.go delete mode 100644 internal/connector/feed_adapter_test.go delete mode 100644 internal/connector/live_adapter_test.go create mode 100644 internal/connector/seam_contract_test.go diff --git a/internal/connector/feed_adapter.go b/internal/connector/feed_adapter.go deleted file mode 100644 index f756803ff..000000000 --- a/internal/connector/feed_adapter.go +++ /dev/null @@ -1,442 +0,0 @@ -package connector - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" - "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" -) - -// This file is the Layer-1 adapter the eventfeed package leaves open: the -// TicketMinter and PollSource seams, backed by the SDK's generated -// EventFeedService. It is the package's own documented path — "a host that -// wants the live feed supplies its own over the generated operations" — not a -// stand-in for one. -// -// SWAP POINT. basecamp-sdk PR 899 lands these adapters in the SDK, and PR 897 -// an eventfeed.NewLive that wires them. When both are in, NewFeedAdapter's two -// seam methods are replaced by that constructor and everything below the -// translation line goes; the translation of the SDK's error shapes into this -// package's two 410 types stays, because it is what keeps the two recoveries -// apart, and nothing upstream owns that. - -// FeedEpochGoneError is the ACCOUNT feed's 410: the held position fell below -// the feed's epoch. EpochAfterID names where servable history begins, and the -// served resume URL re-enters above that fence. -// -// Recovery: accept, follow the resume exactly as served, record the gap with -// its epoch, and classify the entry by the cursor the server chose. -type FeedEpochGoneError struct { - EpochAfterID int64 - Resume string - Err error -} - -func (e *FeedEpochGoneError) Error() string { - return fmt.Sprintf("event feed position is below the feed epoch %d: %v", e.EpochAfterID, e.Err) -} -func (e *FeedEpochGoneError) Unwrap() error { return e.Err } - -// InboxRetentionGoneError is the INBOX lane's 410: the held position fell out -// of the 30-day retention window. There is no epoch — not a zero one — and the -// resume re-enters at since=0, the earliest retained item. -// -// Recovery: it is a different loss with a different shape, and the account -// lane must never produce one. Intake surfaces it rather than resuming, so -// that a contract change announces itself instead of being absorbed as "the -// epoch is zero". -type InboxRetentionGoneError struct { - Resume string - Err error -} - -func (e *InboxRetentionGoneError) Error() string { - return fmt.Sprintf("inbox position is outside the retention window: %v", e.Err) -} -func (e *InboxRetentionGoneError) Unwrap() error { return e.Err } - -// UndifferentiatedRequestError is a feed 400 the server gave no reason for. -// -// The two reasons need opposite recoveries — an invalid position re-enters -// with since=, an invalid filter is terminal — so a 400 -// that names neither is surfaced rather than guessed. Guessing the position -// turns a bad filter into an endless re-entry loop; guessing the filter kills -// a feed a re-entry would have fixed. -type UndifferentiatedRequestError struct { - Err error -} - -func (e *UndifferentiatedRequestError) Error() string { - return fmt.Sprintf("event feed refused the request without naming a reason: %v", e.Err) -} -func (e *UndifferentiatedRequestError) Unwrap() error { return e.Err } - -// FeedClient is the slice of the SDK's EventFeedService this adapter needs. -// An interface so the adapter's error translation can be tested without a -// network, which is the only part of it that carries judgement. -type FeedClient interface { - PollEvents(ctx context.Context, opts *basecamp.PollEventsOptions) (*basecamp.EventFeedPage, error) - CreateStreamTicket(ctx context.Context) (*basecamp.StreamTicket, error) -} - -// NewLiveFeedAdapter builds the adapter over its own SDK client, whose -// transport refuses to follow redirects. -// -// It owns its client for that reason alone. The SDK's default client follows -// a 3xx — to a foreign host too, with the Authorization header stripped but -// the request still sent — and the seam's zero-egress obligation is broken -// before any continuation check here could run. Client options cannot fix -// that from outside (a custom *http.Client is replaced at construction), but -// the transport is honored, and a transport sees each redirect hop before it -// leaves the machine. -func NewLiveFeedAdapter(cfg *basecamp.Config, tokens basecamp.TokenProvider, accountID string, inner http.RoundTripper, opts ...basecamp.ClientOption) (*FeedAdapter, error) { - if cfg == nil || tokens == nil || accountID == "" { - return nil, errors.New("connector: the live feed adapter needs a config, a token provider and an account id") - } - if inner == nil { - inner = http.DefaultTransport - } - opts = append(opts, basecamp.WithTransport(RefuseRedirects(inner))) - client := basecamp.NewClient(cfg, tokens, opts...) - return NewFeedAdapter(client.ForAccount(accountID).EventFeed(), cfg.BaseURL) -} - -// ErrRedirectRefused is a redirect hop the feed's transport would not send. -var ErrRedirectRefused = errors.New("connector: the event feed does not follow redirects") - -// RefuseRedirects wraps a transport so that no redirect hop is ever sent. The -// client then reports the refusal as the request's failure. The SDK sees that -// failure as a network error and spends its retry budget re-sending the -// ORIGINAL, same-origin request before giving up; nothing reaches the target -// on any attempt. -func RefuseRedirects(inner http.RoundTripper) http.RoundTripper { - return refuseRedirects{inner: inner} -} - -type refuseRedirects struct{ inner http.RoundTripper } - -func (t refuseRedirects) RoundTrip(req *http.Request) (*http.Response, error) { - // Response is set exactly when the client created this request to follow - // a redirect. - if req.Response != nil { - return nil, ErrRedirectRefused - } - return t.inner.RoundTrip(req) -} - -// FeedAdapter backs the TicketMinter and PollSource seams. -type FeedAdapter struct { - client FeedClient - origin *url.URL -} - -var ( - _ eventfeed.TicketMinter = (*FeedAdapter)(nil) - _ eventfeed.PollSource = (*FeedAdapter)(nil) -) - -// NewFeedAdapter builds the adapter. origin is the API base the continuation -// and resume URLs are validated against. -func NewFeedAdapter(client FeedClient, origin string) (*FeedAdapter, error) { - if client == nil { - return nil, errors.New("connector: feed adapter needs a client") - } - canonical, err := eventfeed.CanonicalOrigin(origin) - if err != nil { - return nil, fmt.Errorf("connector: feed adapter origin: %w", err) - } - parsed, err := url.Parse(canonical) - if err != nil { - return nil, fmt.Errorf("connector: feed adapter origin: %w", err) - } - return &FeedAdapter{client: client, origin: parsed}, nil -} - -// MintStreamTicket mints one ticket. Neither the ticket nor the URL it rides -// in is ever rendered into an error here: the URL's query string carries the -// bearer. -func (a *FeedAdapter) MintStreamTicket(ctx context.Context) (eventfeed.StreamTicket, error) { - ticket, err := a.client.CreateStreamTicket(ctx) - if err != nil { - return eventfeed.StreamTicket{}, mintError(ctx, err) - } - return eventfeed.StreamTicket{ - Ticket: ticket.Ticket, - ExpiresIn: ticket.ExpiresIn, - URL: ticket.URL, - }, nil -} - -// Poll fetches one page at cursor under filters. -func (a *FeedAdapter) Poll(ctx context.Context, cursor eventfeed.Cursor, filters eventfeed.Filters) (eventfeed.PollPage, error) { - opts, err := a.optionsFor(cursor, filters) - if err != nil { - return eventfeed.PollPage{}, err - } - - page, err := a.client.PollEvents(ctx, opts) - if err != nil { - return eventfeed.PollPage{}, pollError(ctx, err) - } - - events := make([]eventfeed.Event, 0, len(page.Events)) - for _, e := range page.Events { - events = append(events, eventfeed.Event{ - ID: e.ID, - Kind: e.Kind, - EventType: e.EventType, - Action: e.Action, - CreatedAt: e.CreatedAt, - BucketID: e.BucketID, - CreatorID: e.CreatorID, - PerformedByID: e.PerformedByID, - RecordingID: e.RecordingID, - Details: e.Details, - }) - } - // An empty Events with a Next is ordinary, not an end: a request crosses - // up to a thousand ledger rows and serves at most a hundred matches, and - // the rows the filters excluded still moved the cursor. The run loop - // follows Next; nothing here shortcuts on len(events) == 0. - return eventfeed.PollPage{Events: events, Position: page.Position, Next: page.Next}, nil -} - -// optionsFor turns one cursor into the generated operation's options. Exactly -// one of the cursor's three fields is set; the zero cursor is the bare present -// entry. -func (a *FeedAdapter) optionsFor(cursor eventfeed.Cursor, filters eventfeed.Filters) (*basecamp.PollEventsOptions, error) { - if cursor.PageURL != "" { - // A continuation or a 410 resume. It is server-supplied, so it is - // validated against the configured origin before it is used for - // anything: the SPEC's zero-egress-to-a-foreign-target obligation - // rides on this adapter, not on the package. - if err := a.checkContinuation(cursor.PageURL); err != nil { - return nil, err - } - opts, err := basecamp.PollEventsOptionsFromURL(cursor.PageURL) - if err != nil { - return nil, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errMalformedContinuation} - } - if (opts.Since == "") == (opts.Position == "") { - // A followed URL carries exactly one cursor. With neither, the - // generated operation enters at the present — a silent skip of - // everything unserved, on the path whose whole purpose was to - // continue. With both, which one the server honors is not ours to - // know. Both refusals are deliberate: a server that starts sending - // either shape ends the feed loudly rather than moving it silently. - return nil, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: errCursorlessContinuation} - } - // The URL carries the server's own canonical filter set. It is used - // as served: re-imposing the local filters on a resume is how a - // resume stops being the server's. - return opts, nil - } - - opts := &basecamp.PollEventsOptions{ - Since: cursor.Since, - Position: cursor.Position, - Types: filters.Types, - Buckets: filters.Buckets, - Creators: filters.Creators, - Performers: idStrings(filters.Performers), - ExcludePerformers: idStrings(filters.ExcludePerformers), - ActorTypes: filters.ActorTypes, - } - return opts, nil -} - -// checkContinuation enforces same-origin and no-downgrade on a server-supplied -// URL before it is followed. -func (a *FeedAdapter) checkContinuation(raw string) error { - parsed, err := url.Parse(raw) - if err != nil { - return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: fmt.Errorf("connector: unparseable continuation URL")} - } - if !parsed.IsAbs() { - return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: errors.New("connector: continuation URL is not absolute")} - } - if !strings.EqualFold(parsed.Scheme, a.origin.Scheme) || !strings.EqualFold(parsed.Host, a.origin.Host) { - // LocationOrigin is DATA and is deliberately not rendered: a hostile - // target can reflect the bearer into a host label. - return &eventfeed.PollError{ - Kind: eventfeed.PollRedirectRefused, - LocationOrigin: parsed.Scheme + "://" + parsed.Host, - Err: errors.New("connector: continuation URL leaves the configured API origin"), - } - } - return nil -} - -func idStrings(ids []int64) []string { - if len(ids) == 0 { - return nil - } - out := make([]string, 0, len(ids)) - for _, id := range ids { - out = append(out, strconv.FormatInt(id, 10)) - } - return out -} - -// --------------------------------------------------------------------------- -// The translation line. Everything below maps the SDK's error shapes onto the -// seam's taxonomy, and it is the one place the two 410s are told apart. -// --------------------------------------------------------------------------- - -// Everything below builds errors that are safe to render. The rule, for every -// walker this adapter serves: no Err carries the generated error. A generated -// network error renders its request URL — a position token, a server-supplied -// continuation — and a refused redirect renders the server-chosen Location. An -// error leaves this file as a fixed sentinel or a status and a code, never as -// text the server or the request chose. - -var ( - errMalformedContinuation = errors.New("connector: continuation URL could not be read") - errCursorlessContinuation = errors.New("connector: continuation URL carries no single cursor") - errNetwork = errors.New("connector: event feed request failed before a response") -) - -// sanitized reduces a generated error to what is safe to render. -func sanitized(err error) error { - var apiErr *basecamp.Error - if errors.As(err, &apiErr) { - return fmt.Errorf("connector: event feed answered HTTP %d (%s)", apiErr.HTTPStatus, apiErr.Code) - } - return errNetwork -} - -// callerCanceled reports a failure that is the caller's own cancellation. The -// seam requires it to pass through unchanged: classified as transient, a -// shutdown or a reconnect would enter transport-retry handling. A deadline the -// client imposed on itself, with the caller's context still live, is not this -// and stays transient. The context's own error is returned, not the failure -// that wraps it, which may render the request URL. -func callerCanceled(ctx context.Context, err error) bool { - return ctx.Err() != nil && errors.Is(err, ctx.Err()) -} - -// pollError classifies a failed PollEvents call. -func pollError(ctx context.Context, err error) error { - if err == nil { - return nil - } - if callerCanceled(ctx, err) { - return ctx.Err() - } - - // The inbox's 410 first, so it can never fall through to the feed's arm. - // - // On sdk main today both lanes answer one *basecamp.FeedPositionGoneError - // with EpochAfterID *int64, nil on the inbox. basecamp-sdk PR 912 splits - // them into two types with the epoch required on the feed's. Either way - // the discrimination happens here and once: a nil epoch flattened into - // eventfeed.PollError's plain int64 EpochAfterID would present a - // retention loss as "the feed's epoch is 0" and send it down the epoch's - // recovery path, which is the failure this arm exists to prevent. - if gone := asFeedGone(err); gone != nil { - if gone.EpochAfterID == nil { - return &eventfeed.PollError{ - Kind: eventfeed.PollUnrecoverable, - Err: &InboxRetentionGoneError{Resume: gone.Resume, Err: sanitized(err)}, - } - } - return &eventfeed.PollError{ - Kind: eventfeed.PollGone, - EpochAfterID: *gone.EpochAfterID, - ResumeURL: gone.Resume, - Err: &FeedEpochGoneError{EpochAfterID: *gone.EpochAfterID, Resume: gone.Resume, Err: sanitized(err)}, - } - } - - if errors.Is(err, ErrRedirectRefused) { - return &eventfeed.PollError{Kind: eventfeed.PollRedirectRefused, Err: ErrRedirectRefused} - } - - var mismatch *basecamp.FeedFilterMismatchError - if errors.As(err, &mismatch) { - return &eventfeed.PollError{ - Kind: eventfeed.PollFilterChanged, - PositionDigest: mismatch.PositionDigest, - FiltersDigest: mismatch.FiltersDigest, - Err: sanitized(err), - } - } - - var apiErr *basecamp.Error - if !errors.As(err, &apiErr) { - return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: errNetwork} - } - - switch { - case apiErr.HTTPStatus == 400: - // SWAP POINT for basecamp-sdk PR 912's *FeedRequestError: when it - // carries reason=invalid_position this becomes PollPositionInvalid, - // and reason=invalid_filter becomes PollFilterInvalid (with the - // server's message in Msg, as the seam requires there). Until the - // server names the reason, a 400 is undifferentiated and is surfaced - // rather than guessed — see UndifferentiatedRequestError. - return &eventfeed.PollError{ - Kind: eventfeed.PollUnrecoverable, - Err: &UndifferentiatedRequestError{Err: sanitized(err)}, - } - case apiErr.HTTPStatus == 401 || apiErr.HTTPStatus == 403: - return &eventfeed.PollError{Kind: eventfeed.PollUnauthorized, Err: sanitized(err)} - case apiErr.RetryAfter > 0: - return &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: retryAfter(apiErr), Err: sanitized(err)} - case apiErr.Retryable: - return &eventfeed.PollError{Kind: eventfeed.PollTransient, Err: sanitized(err)} - } - return &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable, Err: sanitized(err)} -} - -// mintError classifies a failed CreateStreamTicket call. -func mintError(ctx context.Context, err error) error { - if err == nil { - return nil - } - if callerCanceled(ctx, err) { - return ctx.Err() - } - if errors.Is(err, ErrRedirectRefused) { - return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: ErrRedirectRefused} - } - var apiErr *basecamp.Error - if !errors.As(err, &apiErr) { - return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: errNetwork} - } - switch { - case apiErr.HTTPStatus == 401 || apiErr.HTTPStatus == 403: - return &eventfeed.MintError{Kind: eventfeed.MintUnauthorized, Err: sanitized(err)} - case apiErr.RetryAfter > 0: - return &eventfeed.MintError{Kind: eventfeed.MintThrottled, RetryAfter: retryAfter(apiErr), Err: sanitized(err)} - case apiErr.Retryable: - return &eventfeed.MintError{Kind: eventfeed.MintTransient, Err: sanitized(err)} - } - return &eventfeed.MintError{Kind: eventfeed.MintUnrecoverable, Err: sanitized(err)} -} - -func retryAfter(apiErr *basecamp.Error) time.Duration { - return time.Duration(apiErr.RetryAfter) * time.Second -} - -// asFeedGone reads the SDK's feed 410 into a shape this package owns, so the -// rest of the file does not move when the SDK's does. -func asFeedGone(err error) *feedGone { - var gone *basecamp.FeedPositionGoneError - if !errors.As(err, &gone) { - return nil - } - return &feedGone{EpochAfterID: gone.EpochAfterID, Resume: gone.Resume} -} - -type feedGone struct { - EpochAfterID *int64 - Resume string -} diff --git a/internal/connector/feed_adapter_test.go b/internal/connector/feed_adapter_test.go deleted file mode 100644 index 03f8dae91..000000000 --- a/internal/connector/feed_adapter_test.go +++ /dev/null @@ -1,256 +0,0 @@ -package connector - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" - "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" -) - -type fakeFeedClient struct { - page *basecamp.EventFeedPage - ticket *basecamp.StreamTicket - err error - lastOpt *basecamp.PollEventsOptions - calls int -} - -func (f *fakeFeedClient) PollEvents(_ context.Context, opts *basecamp.PollEventsOptions) (*basecamp.EventFeedPage, error) { - f.calls++ - f.lastOpt = opts - if f.err != nil { - return nil, f.err - } - return f.page, nil -} - -func (f *fakeFeedClient) CreateStreamTicket(context.Context) (*basecamp.StreamTicket, error) { - f.calls++ - if f.err != nil { - return nil, f.err - } - return f.ticket, nil -} - -func newTestAdapter(t *testing.T, client FeedClient) *FeedAdapter { - t.Helper() - adapter, err := NewFeedAdapter(client, "https://3.basecampapi.com") - require.NoError(t, err) - return adapter -} - -// The card's hardest constraint: the feed's two 410s mean different things and -// must not share one recovery path. PR 898 models both with one -// FeedPositionGoneError discriminated by a nil EpochAfterID, while the -// connector seam's PollError.EpochAfterID is a plain int64 — so the naive -// conversion silently turns "the inbox's retention window closed" into "the -// feed's epoch is event 0". -func TestFeedEpoch410BecomesTheGapSignal(t *testing.T) { - epoch := int64(17099838487) - client := &fakeFeedClient{err: &basecamp.FeedPositionGoneError{ - Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 410, Message: "position below epoch"}, - EpochAfterID: &epoch, - Resume: "https://3.basecampapi.com/2914079/events.json?since=17099838487", - }} - - _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollGone, pollErr.Kind) - assert.Equal(t, epoch, pollErr.EpochAfterID) - assert.Equal(t, "https://3.basecampapi.com/2914079/events.json?since=17099838487", pollErr.ResumeURL) - - var epochGone *FeedEpochGoneError - assert.ErrorAs(t, err, &epochGone) -} - -func TestRetention410NeverBecomesTheGapSignal(t *testing.T) { - client := &fakeFeedClient{err: &basecamp.FeedPositionGoneError{ - Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 410, Message: "position outside the retention window"}, - EpochAfterID: nil, // the inbox lane's shape: no epoch, not a zero one - Resume: "https://3.basecampapi.com/2914079/my/inbox.json?since=0", - }} - - _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.NotEqual(t, eventfeed.PollGone, pollErr.Kind, - "a retention 410 dispatched as FeedGap resumes the feed down the epoch's path") - assert.Zero(t, pollErr.EpochAfterID, - "no epoch may be invented for a 410 that carried none") - - var retention *InboxRetentionGoneError - require.ErrorAs(t, err, &retention) - assert.Equal(t, "https://3.basecampapi.com/2914079/my/inbox.json?since=0", retention.Resume) - - var epochGone *FeedEpochGoneError - assert.False(t, errors.As(err, &epochGone), - "the two 410s must not both satisfy the epoch arm") -} - -func TestFilterMismatchCarriesBothDigests(t *testing.T) { - client := &fakeFeedClient{err: &basecamp.FeedFilterMismatchError{ - Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 409, Message: "filters changed"}, - PositionDigest: "9f2ab04e5c11d3a7", - FiltersDigest: "0011223344556677", - }} - - _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollFilterChanged, pollErr.Kind) - assert.Equal(t, "9f2ab04e5c11d3a7", pollErr.PositionDigest) - assert.Equal(t, "0011223344556677", pollErr.FiltersDigest) -} - -// The two reasons behind a feed 400 need opposite recoveries, so a 400 that -// names neither is surfaced rather than guessed. -func TestUnreasonedBadRequestIsSurfacedNotGuessed(t *testing.T) { - client := &fakeFeedClient{err: &basecamp.Error{ - Code: basecamp.CodeValidation, HTTPStatus: 400, Message: "bad request", - }} - - _, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.NotEqual(t, eventfeed.PollPositionInvalid, pollErr.Kind, - "guessing the position turns a bad filter into an endless re-entry loop") - assert.NotEqual(t, eventfeed.PollFilterInvalid, pollErr.Kind, - "guessing the filter kills a feed a re-entry would have fixed") - - var undifferentiated *UndifferentiatedRequestError - assert.ErrorAs(t, err, &undifferentiated) -} - -func TestThrottleAndTransientAreClassifiedApart(t *testing.T) { - throttled := &fakeFeedClient{err: &basecamp.Error{ - Code: basecamp.CodeRateLimit, HTTPStatus: 429, Retryable: true, RetryAfter: 7, - }} - _, err := newTestAdapter(t, throttled).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollThrottled, pollErr.Kind) - assert.Equal(t, 7*time.Second, pollErr.RetryAfter) - - transient := &fakeFeedClient{err: &basecamp.Error{ - Code: basecamp.CodeAPI, HTTPStatus: 503, Retryable: true, - }} - _, err = newTestAdapter(t, transient).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollTransient, pollErr.Kind) -} - -func TestForeignContinuationIsRefusedBeforeAnyRequest(t *testing.T) { - client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} - - _, err := newTestAdapter(t, client).Poll(context.Background(), - eventfeed.Cursor{PageURL: "https://evil.example.com/2914079/events.json?position=x"}, - eventfeed.Filters{}) - - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) - assert.Equal(t, "https://evil.example.com", pollErr.LocationOrigin) - assert.NotContains(t, pollErr.Error(), "evil.example.com", - "a hostile target can reflect the bearer into a host label, so the origin is data and never a rendering") - assert.Zero(t, client.calls, "zero egress to a foreign target") -} - -func TestSchemeDowngradeIsRefused(t *testing.T) { - client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} - - _, err := newTestAdapter(t, client).Poll(context.Background(), - eventfeed.Cursor{PageURL: "http://3.basecampapi.com/2914079/events.json?position=x"}, - eventfeed.Filters{}) - - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) - assert.Zero(t, client.calls) -} - -func TestEmptyPageWithANextIsNotAnEnd(t *testing.T) { - client := &fakeFeedClient{page: &basecamp.EventFeedPage{ - Events: nil, - Position: "opaque-position-2", - Next: "https://3.basecampapi.com/2914079/events.json?position=opaque-position-2", - }} - - page, err := newTestAdapter(t, client).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - require.NoError(t, err) - assert.Empty(t, page.Events) - assert.NotEmpty(t, page.Next, "a page that served no match still advanced the cursor") - assert.Equal(t, "opaque-position-2", page.Position) -} - -func TestPollPassesTheFilterSetAndCursor(t *testing.T) { - client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} - adapter := newTestAdapter(t, client) - - _, err := adapter.Poll(context.Background(), - eventfeed.Cursor{Since: "17099838487"}, - eventfeed.Filters{ - Types: []string{"comment.created"}, - Buckets: []int64{48699913}, - ExcludePerformers: []int64{52007412}, - ActorTypes: []string{"person"}, - }) - require.NoError(t, err) - - require.NotNil(t, client.lastOpt) - assert.Equal(t, "17099838487", client.lastOpt.Since) - assert.Equal(t, []string{"comment.created"}, client.lastOpt.Types) - assert.Equal(t, []int64{48699913}, client.lastOpt.Buckets) - assert.Equal(t, []string{"52007412"}, client.lastOpt.ExcludePerformers, - "the loop guard is the agent's own resolved id") - assert.Equal(t, []string{"person"}, client.lastOpt.ActorTypes) -} - -func TestResumeURLIsUsedAsServed(t *testing.T) { - client := &fakeFeedClient{page: &basecamp.EventFeedPage{}} - adapter := newTestAdapter(t, client) - - _, err := adapter.Poll(context.Background(), - eventfeed.Cursor{PageURL: "https://3.basecampapi.com/2914079/events.json?since=17099838487&types=comment.created"}, - eventfeed.Filters{Types: []string{"card.created"}}) - require.NoError(t, err) - - require.NotNil(t, client.lastOpt) - assert.Equal(t, "17099838487", client.lastOpt.Since) - assert.Equal(t, []string{"comment.created"}, client.lastOpt.Types, - "re-imposing the local filters on a resume is how a resume stops being the server's") -} - -func TestMintClassifiesUnauthorized(t *testing.T) { - client := &fakeFeedClient{err: &basecamp.Error{Code: basecamp.CodeAuth, HTTPStatus: 401}} - - _, err := newTestAdapter(t, client).MintStreamTicket(context.Background()) - - var mintErr *eventfeed.MintError - require.ErrorAs(t, err, &mintErr) - assert.Equal(t, eventfeed.MintUnauthorized, mintErr.Kind) -} - -func TestMintNeverRendersTheTicket(t *testing.T) { - client := &fakeFeedClient{ticket: &basecamp.StreamTicket{ - Ticket: "s3cr3t-bearer", - ExpiresIn: 120, - URL: "wss://cable.basecamp.com/cable?ticket=s3cr3t-bearer", - }} - - ticket, err := newTestAdapter(t, client).MintStreamTicket(context.Background()) - require.NoError(t, err) - assert.Equal(t, "s3cr3t-bearer", ticket.Ticket) - assert.Equal(t, "wss://cable.basecamp.com/cable?ticket=s3cr3t-bearer", ticket.URL, - "the URL is connected to verbatim; the connector never assembles cable topology") -} diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 768825fd7..d4dfeceec 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -58,7 +58,12 @@ type Options struct { Ledger *Ledger Queue *Queue Minter eventfeed.TicketMinter - Polls eventfeed.PollSource + // PollsFor returns a fresh poll source per walk. The SDK's live source + // holds walk state (the order a continuation must follow), so the feed's + // connection and each repair walk get their own. Polls, when set instead, + // is shared by all of them — for test fakes that hold no walk state. + PollsFor func() eventfeed.PollSource + Polls eventfeed.PollSource // Pointers receives one NDJSON line per newly seen event. Writes are // serialized: an interleaved write tears a line and breaks the watcher @@ -80,6 +85,18 @@ type Options struct { Transport eventfeed.CableTransport } +// LiveOptions binds intake to the live feed: the SDK's own seams over the +// generated operations, with their redirect guard, their two-410 mapping and +// their continuation checks. The caller fills in the rest (account, namespace, +// ledger, queue, filters). +func LiveOptions(live *eventfeed.Live) Options { + return Options{ + Origin: live.Origin(), + Minter: live.Minter(), + PollsFor: live.Polls, + } +} + // Intake is the feed's delivery path: write the pointer, hand over the id. // // Everything else — reading the recording, judging it, dispatching it — is @@ -144,7 +161,7 @@ func New(opts Options) (*Intake, error) { return nil, errors.New("connector: intake needs a ledger") case opts.Queue == nil: return nil, errors.New("connector: intake needs a queue") - case opts.Minter == nil || opts.Polls == nil: + case opts.Minter == nil || (opts.Polls == nil && opts.PollsFor == nil): return nil, errors.New("connector: intake needs the feed's two seams") case opts.AccountID == "": return nil, errors.New("connector: intake needs an account id") @@ -152,6 +169,11 @@ func New(opts Options) (*Intake, error) { return nil, errors.New("connector: intake needs a consumer namespace") } + if opts.PollsFor == nil { + shared := opts.Polls + opts.PollsFor = func() eventfeed.PollSource { return shared } + } + origin, err := eventfeed.CanonicalOrigin(opts.Origin) if err != nil { return nil, fmt.Errorf("connector: intake origin: %w", err) @@ -330,7 +352,7 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { options = append(options, eventfeed.WithTransport(in.opts.Transport)) } - served := &servedPolls{inner: in.opts.Polls} + served := &servedPolls{inner: in.opts.PollsFor()} in.mu.Lock() in.served = served in.mu.Unlock() @@ -385,7 +407,12 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { if in.reconnectRequested() { return errReconnect } - return in.classifyTerminal(ctx, feedErr) + // The two 410s are told apart below this package: the SDK's live seam + // maps the feed's FeedPositionGoneError (epoch required) to the gap + // signal, and refuses the inbox's InboxPositionGoneError shape on the + // account lane as unrecoverable, so it ends the feed here as a terminal + // rather than resuming as if it were the epoch's. + return feedErr } // ingest is the whole of intake: one pointer written, one id handed over. @@ -557,29 +584,6 @@ func (in *Intake) observer(ctx context.Context) eventfeed.Observer { } } -// classifyTerminal turns the feed's terminal error into the connector's own -// verdict. The one case that needs saying is the inbox's 410 arriving on the -// account lane: it is a different loss with a different resume, and it is -// surfaced rather than absorbed. -func (in *Intake) classifyTerminal(ctx context.Context, err error) error { - if err == nil { - return nil - } - var retention *InboxRetentionGoneError - if errors.As(err, &retention) { - if _, recordErr := in.ledger.RecordGap(ctx, Gap{ - DetectedAt: in.now(), - Class: GapRetention, - EntryClass: EntryUnknown, - Note: "a retention 410 was served on the account lane, which has an epoch instead; not resumed", - }); recordErr != nil { - in.log.Error("could not record the retention gap", "error", recordErr) - } - in.log.Error("the account feed answered the inbox lane's 410: its resume re-enters at the earliest retained item, not above an epoch, so it is not followed here") - } - return err -} - // handleSignal decides what a semantic signal means for this connector. It // runs synchronously on the delivery path, so it does only what must happen // before the disposition takes effect and starts the rest elsewhere. @@ -688,7 +692,8 @@ func (in *Intake) startRepair(ctx context.Context, loss Loss) { // Off the delivery path: nothing about live intake waits for this. walker := &repairWalker{ ledger: in.ledger, - polls: in.opts.Polls, + polls: in.opts.PollsFor(), + origin: in.key.Origin, filters: in.opts.Filters, ingest: in.ingest, now: in.now, diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go index 933f8d18d..7879ccac0 100644 --- a/internal/connector/intake_test.go +++ b/internal/connector/intake_test.go @@ -255,26 +255,6 @@ func TestFeedGapIsRefusedWhenItCannotBeRecorded(t *testing.T) { }), "a gap nothing wrote down is a gap nothing will ever report") } -// A retention 410 reaching the account lane means the server said something -// this connector does not model. It is surfaced and recorded as its own class, -// never resumed as if it had been the epoch's. -func TestRetentionGoneOnTheAccountLaneIsRecordedAsItsOwnClass(t *testing.T) { - intake, ledger, _ := newTestIntake(t, nil, nil) - ctx := context.Background() - - err := intake.classifyTerminal(ctx, &eventfeed.PollError{ - Kind: eventfeed.PollUnrecoverable, - Err: &InboxRetentionGoneError{Resume: "https://3.basecampapi.com/2914079/my/inbox.json?since=0"}, - }) - require.Error(t, err) - - gaps, err := ledger.Gaps(ctx) - require.NoError(t, err) - require.Len(t, gaps, 1) - assert.Equal(t, GapRetention, gaps[0].Class) - assert.Nil(t, gaps[0].EpochAfterID, "there is no epoch here, and zero is not one") -} - func TestLedgerRefusesAGapWhoseClassAndEpochDisagree(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go index b1c2bcec2..3927c326f 100644 --- a/internal/connector/invariants_test.go +++ b/internal/connector/invariants_test.go @@ -21,45 +21,6 @@ import ( // invariant names match the review's list (A positions, C repair, D // classification, E lifecycle, F queue, G lock, H confidentiality). -// D2 / A3: a followed URL carries exactly its cursor. One with neither since -// nor position would silently enter at the present. -func TestInvariantD2ACursorlessContinuationIsRefused(t *testing.T) { - client := &fakeFeedClient{} - _, err := newTestAdapter(t, client).Poll(context.Background(), - eventfeed.Cursor{PageURL: "https://3.basecampapi.com/2914079/events.json?types=comment.created"}, - eventfeed.Filters{}) - - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.NotEqual(t, eventfeed.PollTransient, pollErr.Kind) - assert.Zero(t, client.calls, "a URL with no cursor is an entry at the present, and is never sent") -} - -// H1: nothing server-chosen — a redirect target, a URL carrying a position — -// is rendered into an error the adapter returns. -func TestInvariantH1AdapterErrorsRenderNoServerChosenURL(t *testing.T) { - leaky := errors.New(`Get "https://3.basecampapi.com/2914079/events.json?position=SECRET-POSITION": connection reset`) - - _, err := newTestAdapter(t, &fakeFeedClient{err: leaky}).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - require.Error(t, err) - assert.NotContains(t, err.Error(), "SECRET-POSITION") - - _, err = newTestAdapter(t, &fakeFeedClient{err: leaky}).MintStreamTicket(context.Background()) - require.Error(t, err) - assert.NotContains(t, err.Error(), "SECRET-POSITION") - - redirect := errors.Join(ErrRedirectRefused, errors.New(`Get "https://evil.example.com/steal?leak=SECRET-TARGET"`)) - _, err = newTestAdapter(t, &fakeFeedClient{err: redirect}).Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) - assert.NotContains(t, err.Error(), "SECRET-TARGET") - - _, err = newTestAdapter(t, &fakeFeedClient{err: redirect}).MintStreamTicket(context.Background()) - require.Error(t, err) - assert.NotContains(t, err.Error(), "SECRET-TARGET") -} - // H1, repair side: the walk's logs render a failure's kind, never its text. func TestInvariantH1RepairLogsRenderNoFailureText(t *testing.T) { ledger := newTestLedger(t) diff --git a/internal/connector/live_adapter_test.go b/internal/connector/live_adapter_test.go deleted file mode 100644 index 05703d5cf..000000000 --- a/internal/connector/live_adapter_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package connector - -import ( - "context" - "net/http" - "net/http/httptest" - "sync/atomic" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" - "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" -) - -// The generated operation must not follow a 3xx to a server-supplied target: -// zero egress to a foreign redirect, before any continuation check can run. -func TestTheLiveAdapterRefusesRedirectsBeforeAnyEgress(t *testing.T) { - var foreignHits atomic.Int32 - foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - foreignHits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer foreign.Close() - origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Redirect(w, r, foreign.URL+r.URL.Path, http.StatusFound) - })) - defer origin.Close() - - adapter, err := NewLiveFeedAdapter(&basecamp.Config{BaseURL: origin.URL}, &basecamp.StaticTokenProvider{Token: "token"}, "2914079", nil, - // One attempt: the SDK retries a refused hop as a network error, which - // only repeats the same-origin request and slows the test down. - basecamp.WithMaxRetries(0)) - require.NoError(t, err) - - _, err = adapter.Poll(context.Background(), eventfeed.Cursor{Since: "1"}, eventfeed.Filters{}) - var pollErr *eventfeed.PollError - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) - - _, err = adapter.MintStreamTicket(context.Background()) - require.Error(t, err) - - assert.Zero(t, foreignHits.Load(), "no request may reach the redirect target") -} diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 8b1f2502b..766c95210 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "net/url" "strconv" "time" @@ -21,8 +22,10 @@ import ( // // It runs off the delivery path. Nothing about live intake waits for it. type repairWalker struct { - ledger *Ledger - polls eventfeed.PollSource + ledger *Ledger + polls eventfeed.PollSource + // origin is the API origin every URL the walk follows must stay on. + origin string filters eventfeed.Filters ingest func(ctx context.Context, event eventfeed.Event, lane Lane) error now func() time.Time @@ -158,6 +161,10 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { } // An empty page with a `next` is ordinary — the walk crossed rows the // filters excluded — so the loop never stops on len(Events) == 0. + if err := sameOrigin(w.origin, page.Next); err != nil { + w.log.Error("a repair page's next URL leaves the API origin; the loss stays open for the next start", "loss_id", loss.ID) + return last, errReconciliationEnded + } cursor = eventfeed.Cursor{PageURL: page.Next} } } @@ -202,28 +209,6 @@ func failureKind(err error) string { // to continue this pass at, nil with no error to end the pass and wait for the // next repair poll, or an error. func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor eventfeed.Cursor, err error, pass *repairPass) (*eventfeed.Cursor, error) { - var retention *InboxRetentionGoneError - if errors.As(err, &retention) { - // The inbox lane's 410 on the account lane: the same foreign loss the - // feed surfaces, recorded as its own class and never resumed as if it - // were the epoch's. Nothing on this lane will serve the ids. - if _, recordErr := w.ledger.RecordGap(ctx, Gap{ - DetectedAt: w.now(), - Class: GapRetention, - EntryClass: EntryUnknown, - Note: "a retention 410 was served to the repair walk on the account lane; not resumed", - }); recordErr != nil { - return nil, recordErr - } - unrecovered, closeErr := w.ledger.CloseLoss(ctx, loss.ID, w.now()) - if closeErr != nil { - return nil, closeErr - } - w.log.Error("the repair walk was answered with the inbox lane's 410; not resumed", - "loss_id", loss.ID, "unrecovered", unrecovered) - return nil, errReconciliationEnded - } - var pollErr *eventfeed.PollError if !errors.As(err, &pollErr) { w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) @@ -276,6 +261,10 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event } // Ids above the epoch are still servable: the resume is followed as // served, whether or not this loss had met the fence before. + if err := sameOrigin(w.origin, pollErr.ResumeURL); err != nil { + w.log.Error("a repair 410's resume URL leaves the API origin; the loss stays open for the next start", "loss_id", loss.ID) + return nil, errReconciliationEnded + } pass.followed[pollErr.ResumeURL] = true return &eventfeed.Cursor{PageURL: pollErr.ResumeURL}, nil @@ -325,3 +314,26 @@ func (w *repairWalker) wait(ctx context.Context, d time.Duration) error { return ctx.Err() } } + +// sameOrigin holds a URL the repair walk is about to follow to the API origin, +// with no scheme downgrade. +// +// This is the one check intake keeps from its temporary adapter. The SDK's +// live poll source re-issues a followed URL's cursor against its own client +// and never requests the URL itself, but it leaves origin validation to the +// caller — its connector runs it before every followed URL, unexported. The +// repair walk follows next and resume URLs outside that connector, so it +// runs the check itself rather than following a foreign URL's cursor. +func sameOrigin(origin, raw string) error { + parsed, err := url.Parse(raw) + if err != nil || !parsed.IsAbs() || parsed.User != nil { + return errForeignContinuation + } + canonical, err := eventfeed.CanonicalOrigin(parsed.Scheme + "://" + parsed.Host) + if err != nil || canonical != origin { + return errForeignContinuation + } + return nil +} + +var errForeignContinuation = errors.New("connector: a followed URL leaves the API origin") diff --git a/internal/connector/repair_test.go b/internal/connector/repair_test.go index 533b34265..948101e27 100644 --- a/internal/connector/repair_test.go +++ b/internal/connector/repair_test.go @@ -28,6 +28,7 @@ func newTestWalker(t *testing.T, ledger *Ledger, polls eventfeed.PollSource, clo walker := &repairWalker{ ledger: ledger, polls: polls, + origin: "https://3.basecampapi.com", now: clock.now, log: slog.New(slog.DiscardHandler), sleep: clock.advance(time.Minute), diff --git a/internal/connector/review2_test.go b/internal/connector/review2_test.go index c32a2bf09..619e76d6d 100644 --- a/internal/connector/review2_test.go +++ b/internal/connector/review2_test.go @@ -2,7 +2,6 @@ package connector import ( "context" - "errors" "os" "path/filepath" "runtime" @@ -220,31 +219,6 @@ func (c countingMembership) Buckets(context.Context) ([]int64, error) { return nil, nil } -// The seam requires connector cancellation to pass through unchanged, or a -// shutdown enters transport-retry handling. -func TestCallerCancellationPassesThroughTheAdapterUnchanged(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - adapter := newTestAdapter(t, &fakeFeedClient{err: context.Canceled}) - _, err := adapter.Poll(ctx, eventfeed.Cursor{}, eventfeed.Filters{}) - var pollErr *eventfeed.PollError - assert.False(t, errors.As(err, &pollErr), "a canceled poll is not a transport failure") - assert.ErrorIs(t, err, context.Canceled) - - _, err = adapter.MintStreamTicket(ctx) - var mintErr *eventfeed.MintError - assert.False(t, errors.As(err, &mintErr), "a canceled mint is not a transport failure") - assert.ErrorIs(t, err, context.Canceled) - - // A client-owned timeout with the caller's context still live stays - // transient. - adapter = newTestAdapter(t, &fakeFeedClient{err: context.DeadlineExceeded}) - _, err = adapter.Poll(context.Background(), eventfeed.Cursor{}, eventfeed.Filters{}) - require.ErrorAs(t, err, &pollErr) - assert.Equal(t, eventfeed.PollTransient, pollErr.Kind) -} - func TestQueuePauseTracksEveryBlockedOffer(t *testing.T) { queue, err := NewQueue(1, 1) require.NoError(t, err) diff --git a/internal/connector/review_fixes_test.go b/internal/connector/review_fixes_test.go index 4e4f70635..b18bcecf4 100644 --- a/internal/connector/review_fixes_test.go +++ b/internal/connector/review_fixes_test.go @@ -5,15 +5,16 @@ import ( "context" "encoding/json" "errors" + "net/http" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed/feedtest" ) @@ -257,24 +258,65 @@ func answerSubscription(conn *feedtest.Conn) { } } -// A retention 410 reaching the repair walk is the same foreign loss it is on -// the feed: recorded as its own class and not retried as if it were transient. -func TestARetention410InTheRepairWalkIsRecordedNotRetried(t *testing.T) { +// The inbox's 410 shape reaching the repair walk on the account lane is +// refused at the seam, never taken as the epoch's: no epoch gap is recorded +// for it, it is not retried for the window, and the loss stays open. +func TestAnInboxShaped410InTheRepairWalkIsNotTheEpochsPath(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) require.NoError(t, err) - adapter := newTestAdapter(t, &fakeFeedClient{err: retentionGone()}) - walker, _ := newTestWalker(t, ledger, adapter, clock) + var hits atomic.Int32 + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { + hits.Add(1) + s.json(w, 410, `{"error":"gone","resume":"ORIGIN/2914079/events.json?since=0"}`) + }) + polls, _ := livePolls(t, s) + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.origin = mustOrigin(t, s.URL) require.NoError(t, walker.reconcile(ctx, loss)) gaps, err := ledger.Gaps(ctx) require.NoError(t, err) - require.Len(t, gaps, 1, "the retention 410 is a fact about the feed, recorded once") - assert.Equal(t, GapRetention, gaps[0].Class) - assert.Nil(t, gaps[0].EpochAfterID) + assert.Empty(t, gaps, "no epoch was served, so no epoch gap is recorded") + assert.Equal(t, int32(1), hits.Load(), "a refusal retrying will not fix is not retried for ten minutes") + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Len(t, open, 1) +} + +// The repair walk follows next and resume URLs outside the SDK connector, so +// it holds them to the API origin itself. +func TestTheRepairWalkDoesNotFollowAForeignURL(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Position: "p1", Next: "https://evil.example.com/2914079/events.json?position=p1"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + assert.Equal(t, 1, polls.calls, "the foreign next is not followed") + + loss2, err := ledger.RecordLoss(ctx, []int64{17099838600}, clock.at, 10*time.Minute) + require.NoError(t, err) + polls2 := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 17099838550, + ResumeURL: "http://3.basecampapi.com/2914079/events.json?since=17099838550"}}} + walker2, _ := newTestWalker(t, ledger, polls2, clock) + require.NoError(t, walker2.reconcile(ctx, loss2)) + assert.Equal(t, 1, polls2.calls, "a downgraded resume is not followed") +} + +func mustOrigin(t *testing.T, raw string) string { + t.Helper() + origin, err := eventfeed.CanonicalOrigin(raw) + require.NoError(t, err) + return origin } // A 410 in the walk fences off the ids below the epoch. Ids above it are still @@ -354,13 +396,6 @@ func TestAnUnrecoverableRepairPollLeavesTheLossOpen(t *testing.T) { assert.Len(t, open, 1) } -func retentionGone() error { - return &basecamp.FeedPositionGoneError{ - Err: &basecamp.Error{Code: basecamp.CodeAPI, HTTPStatus: 410, Message: "outside retention"}, - Resume: "https://3.basecampapi.com/2914079/my/inbox.json?since=0", - } -} - // A resume that answers with the same 410 again must not become an endless // loop that writes a gap per turn. func TestARepeated410OnTheResumeEndsTheWalk(t *testing.T) { diff --git a/internal/connector/round4_test.go b/internal/connector/round4_test.go index f000fe798..994dcc904 100644 --- a/internal/connector/round4_test.go +++ b/internal/connector/round4_test.go @@ -46,7 +46,7 @@ func TestARefusedPositionWithNoPollServedIDReentersAtTheBeginningNotThePresent(t queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) require.NoError(t, err) intake, transport, minter, _ := newFeedIntake(t, ledger, Options{}) - intake.opts.Polls = polls + intake.opts.PollsFor = func() eventfeed.PollSource { return polls } intake.queue = queue for range 4 { minter.ScriptTicket(ticket()) diff --git a/internal/connector/seam_contract_test.go b/internal/connector/seam_contract_test.go new file mode 100644 index 000000000..905b9ece6 --- /dev/null +++ b/internal/connector/seam_contract_test.go @@ -0,0 +1,152 @@ +package connector + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp" + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// These pin, at the real seam intake is built on (eventfeed.NewLive over the +// generated operations), the contract its recovery paths depend on. The +// handling lives in basecamp-sdk now; if the SDK ever changes it, intake's +// assumptions break here rather than silently in production. + +type feedServer struct { + *httptest.Server + foreignURL string + foreignHits atomic.Int32 +} + +func newFeedServer(t *testing.T, handler func(w http.ResponseWriter, r *http.Request, s *feedServer)) *feedServer { + t.Helper() + s := &feedServer{} + foreign := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + s.foreignHits.Add(1) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(foreign.Close) + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r, s) + })) + t.Cleanup(s.Close) + s.foreignURL = foreign.URL + return s +} + +func (s *feedServer) json(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(strings.ReplaceAll(body, "ORIGIN", s.URL))) +} + +func livePolls(t *testing.T, s *feedServer) (eventfeed.PollSource, eventfeed.TicketMinter) { + t.Helper() + live, err := eventfeed.NewLive(&basecamp.Config{BaseURL: s.URL}, &basecamp.StaticTokenProvider{Token: "token"}, + "2914079", eventfeed.AccountLane, basecamp.WithMaxRetries(0)) + require.NoError(t, err) + opts := LiveOptions(live) + return opts.PollsFor(), opts.Minter +} + +func pollOnce(t *testing.T, status int, body string) *eventfeed.PollError { + t.Helper() + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { s.json(w, status, body) }) + polls, _ := livePolls(t, s) + _, err := polls.Poll(context.Background(), eventfeed.Cursor{Position: "p"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + return pollErr +} + +// D1: the feed's 410 names its epoch and becomes the gap signal. +func TestSeamTheFeedsEpoch410IsTheGapSignal(t *testing.T) { + pollErr := pollOnce(t, 410, `{"error":"gone","epoch_after_id":17099838487,"resume":"ORIGIN/2914079/events.json?since=17099838487"}`) + assert.Equal(t, eventfeed.PollGone, pollErr.Kind) + assert.Equal(t, int64(17099838487), pollErr.EpochAfterID) + assert.True(t, strings.HasSuffix(pollErr.ResumeURL, "since=17099838487")) +} + +// D1: the inbox's 410 shape — no epoch, a since=0 resume — arriving on the +// account lane never becomes the gap signal, and no epoch is invented for it. +func TestSeamAnInboxShaped410OnTheAccountLaneIsNeverTheGapSignal(t *testing.T) { + pollErr := pollOnce(t, 410, `{"error":"gone","resume":"ORIGIN/2914079/events.json?since=0"}`) + assert.NotEqual(t, eventfeed.PollGone, pollErr.Kind, "a retention loss must not take the epoch's recovery path") + assert.Zero(t, pollErr.EpochAfterID) +} + +// D1: a 410 whose resume does not re-enter at the fence it declares is refused +// rather than followed into a skip. +func TestSeamA410WhoseResumeSkipsTheFenceIsRefused(t *testing.T) { + pollErr := pollOnce(t, 410, `{"error":"gone","epoch_after_id":7,"resume":"ORIGIN/2914079/events.json?since=now"}`) + assert.NotEqual(t, eventfeed.PollGone, pollErr.Kind) +} + +// D3: the 400's reason keys recover-versus-stop. +func TestSeamThe400sReasonDecidesRecoverOrStop(t *testing.T) { + assert.Equal(t, eventfeed.PollPositionInvalid, + pollOnce(t, 400, `{"error":"x","reason":"invalid_position"}`).Kind) + assert.Equal(t, eventfeed.PollFilterInvalid, + pollOnce(t, 400, `{"error":"x","reason":"invalid_filter"}`).Kind) + unknown := pollOnce(t, 400, `{"error":"Unrecognized position","reason":"invalid_something"}`).Kind + assert.NotEqual(t, eventfeed.PollPositionInvalid, unknown, "an unnamed reason is surfaced, never guessed") + assert.NotEqual(t, eventfeed.PollFilterInvalid, unknown) +} + +// D2: a followed URL carries exactly one cursor; with neither it would enter at +// the present. +func TestSeamACursorlessContinuationIsRefusedBeforeAnyRequest(t *testing.T) { + var hits atomic.Int32 + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { + hits.Add(1) + s.json(w, 200, `{"events":[],"position":"p"}`) + }) + polls, _ := livePolls(t, s) + _, err := polls.Poll(context.Background(), eventfeed.Cursor{PageURL: s.URL + "/2914079/events.json"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Zero(t, hits.Load()) +} + +// D4 / H1: a 3xx is never followed, same origin or not, and the target is not +// rendered. +func TestSeamRedirectsAreRefusedWithZeroEgress(t *testing.T) { + s := newFeedServer(t, func(w http.ResponseWriter, r *http.Request, s *feedServer) { + http.Redirect(w, r, s.foreignURL+"/steal?leak=SECRET-TARGET", http.StatusFound) + }) + polls, minter := livePolls(t, s) + + _, err := polls.Poll(context.Background(), eventfeed.Cursor{Position: "p"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + require.ErrorAs(t, err, &pollErr) + assert.Equal(t, eventfeed.PollRedirectRefused, pollErr.Kind) + assert.NotContains(t, err.Error(), "SECRET-TARGET") + + _, err = minter.MintStreamTicket(context.Background()) + require.Error(t, err) + assert.NotContains(t, err.Error(), "SECRET-TARGET") + + assert.Zero(t, s.foreignHits.Load(), "no request may reach the redirect target") +} + +// E4: the caller's own cancellation passes through unclassified. +func TestSeamCallerCancellationIsNotATransportFailure(t *testing.T) { + s := newFeedServer(t, func(w http.ResponseWriter, _ *http.Request, s *feedServer) { + s.json(w, 200, `{"events":[],"position":"p"}`) + }) + polls, _ := livePolls(t, s) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := polls.Poll(ctx, eventfeed.Cursor{Position: "p"}, eventfeed.Filters{}) + var pollErr *eventfeed.PollError + assert.False(t, errors.As(err, &pollErr)) +} From 81b47973c924d9c15d89dabf34204328cd413d31 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 21:57:09 +0200 Subject: [PATCH 14/49] Re-enter a restart at this filter set's own id, stop on a refused re-entry, close expired losses The package confirms a page before it saves the page's position, so a failed save leaves this filter set's poll-served id recorded with no position. A restart then fell through to the lineage maximum, which can be another filter set's larger id, past events this one never served. It now re-enters at this filter set's own id when it has one. The SDK now classifies a 400 with no reason by its message, so a 400 that says "Unrecognized position" but is not cured by re-entering could send intake round an unbounded mint-dial-poll loop. A connection that is itself the safe re-entry and is refused before serving a page now ends the run. A loss whose repair walk always ended in a failure no retry fixes stayed open forever. Past its window it now gets one last pass and closes, its missing ids recorded as unrecovered. --- internal/connector/intake.go | 29 ++++++++- internal/connector/repair.go | 46 ++++++++++++++- internal/connector/round8_test.go | 97 +++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 internal/connector/round8_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index d4dfeceec..b5e5c50ea 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -89,6 +89,10 @@ type Options struct { // generated operations, with their redirect guard, their two-410 mapping and // their continuation checks. The caller fills in the rest (account, namespace, // ledger, queue, filters). +// +// Build the Live without a debug logger or request hooks in production: the +// SDK client logs request URLs through them, and a poll URL carries the feed +// position. func LiveOptions(live *eventfeed.Live) Options { return Options{ Origin: live.Origin(), @@ -138,7 +142,10 @@ type Intake struct { replaying bool // reentryReplays is whether the pending re-entry is a replay. reentryReplays bool - reentryLog string + // enteredByReentry is whether this connection is itself the safe + // re-entry after a refused position. + enteredByReentry bool + reentryLog string // abortErr ends the run: set when continuing could only mean entering // the feed somewhere unsafe. abortErr error @@ -283,6 +290,7 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { reentry, hasReentry, reentryLog, reentryReplays := in.reentry, in.hasReentry, in.reentryLog, in.reentryReplays in.hasReentry = false in.replaying = false + in.enteredByReentry = hasReentry in.mu.Unlock() defer func() { in.mu.Lock() @@ -296,6 +304,10 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { if err != nil { return err } + ownServed, err := in.ledger.LastPollServedID(runCtx, in.key) + if err != nil { + return err + } lineageServed, err := in.ledger.LineagePollServedID(runCtx, in.key) if err != nil { return err @@ -316,6 +328,13 @@ func (in *Intake) runOnce(ctx context.Context, since int64) error { in.log.Warn("the stored position was refused; " + reentryLog) case hadPosition: in.log.Info("resuming the feed from the stored position", "filter_key", in.key.FilterKey) + case ownServed > 0: + // This filter set has progress but no position: the package confirms + // a page before it saves the page's position, so a failed save leaves + // exactly this. Its own id, never another filter set's. + start = eventfeed.StartAfter(ownServed) + in.log.Warn("no stored position for this filter set; re-entering after its own last poll-served id", + "since", ownServed, "filter_key", in.key.FilterKey) case lineageServed > 0: // A filter change: this digest has no position, but the consumer's // poll lane had reached this id under another. Entering at the @@ -775,7 +794,15 @@ func (in *Intake) noteBucket(bucketID int64) { func (in *Intake) onPositionRejected(ctx context.Context) { in.mu.Lock() promoted := in.promotedThisRun + reentered := in.enteredByReentry in.mu.Unlock() + if !promoted && reentered { + // The safe re-entry was itself refused before it served a page. + // Whatever the server objects to, another re-entry will not cure it, + // and reconnecting again would mint, dial and poll in a tight loop. + in.abort(errors.New("connector: the feed refused the safe re-entry after a refused position; not retrying")) + return + } if promoted { // The package's own reset cursor is this run's poll-served id, which // is at least what the ledger holds. diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 766c95210..e780bb5c1 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -40,6 +40,14 @@ type repairWalker struct { func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { for { + if !w.now().Before(loss.DeadlineAt) { + // Past the window already — perhaps across restarts whose walks + // each ended in a failure no retry fixes. One last pass is still + // worth trying; after it, the loss closes either way. + err := w.finalPass(ctx, &loss) + return err + } + done, err := w.settled(ctx, loss) if err != nil || done { return err @@ -47,7 +55,7 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { cursor, err := w.walk(ctx, &loss) if errors.Is(err, errReconciliationEnded) { - return nil + return w.closeIfExpired(ctx, loss) } if err != nil { return err @@ -86,6 +94,42 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { } } +// finalPass runs one walk for a loss past its window and then closes it, +// whatever the walk managed: its missing ids become unrecovered. +func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { + if done, err := w.settled(ctx, *loss); err != nil || done { + return err + } + if _, err := w.walk(ctx, loss); err != nil && !errors.Is(err, errReconciliationEnded) { + return err + } + if done, err := w.settled(ctx, *loss); err != nil || done { + return err + } + return w.closeExpired(ctx, *loss) +} + +// closeIfExpired closes a loss whose walk ended early, if its window is over. +// A walk that ends in a failure no retry fixes leaves the loss open for the +// next start — but not forever. +func (w *repairWalker) closeIfExpired(ctx context.Context, loss Loss) error { + if w.now().Before(loss.DeadlineAt) { + return nil + } + return w.closeExpired(ctx, loss) +} + +func (w *repairWalker) closeExpired(ctx context.Context, loss Loss) error { + unrecovered, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()) + if err != nil { + return err + } + if unrecovered > 0 { + w.log.Error("a buffer overflow's window closed with events unrecovered", "loss_id", loss.ID, "unrecovered", unrecovered) + } + return nil +} + // settled closes the loss and reports true when nothing is missing any more. func (w *repairWalker) settled(ctx context.Context, loss Loss) (bool, error) { missing, err := w.ledger.MissingIDs(ctx, loss.ID, LossMissing) diff --git a/internal/connector/round8_test.go b/internal/connector/round8_test.go new file mode 100644 index 000000000..0167075ce --- /dev/null +++ b/internal/connector/round8_test.go @@ -0,0 +1,97 @@ +package connector + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// A3: a missing position does not mean a new filter. The package confirms a +// page before it saves the position, so a failed save leaves this filter's +// poll-served id recorded with no position. A restart re-enters at that id, +// not at another filter set's larger one. +func TestARestartWithoutAPositionPrefersThisFilterSetsOwnServedID(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + require.NoError(t, ledger.NotePollServed(ctx, intake.CheckpointKey(), 1000)) + other := intake.CheckpointKey() + other.FilterKey = "srv2-0000000000000000" + require.NoError(t, ledger.NotePollServed(ctx, other, 2000)) + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond) + assert.Equal(t, "1000", polls.Calls()[0].Cursor.Since, + "another filter set's id is past events this one never served") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// E2: a safe re-entry that is itself refused ends the run. Reconnecting again +// would mint, dial and poll in a tight loop against the API. +func TestARefusedReentryEndsTheRunInsteadOfLooping(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "refused")) + require.NoError(t, ledger.NotePollServed(ctx, intake.CheckpointKey(), 1000)) + + for range 50 { + minter.ScriptTicket(ticket()) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + } + + done := runInBackground(ctx, t, intake) + answered := 0 + var err error + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + select { + case err = <-done: + return true + default: + return false + } + }, 5*time.Second, 10*time.Millisecond, "a refused re-entry must end the run") + assert.Error(t, err) + assert.LessOrEqual(t, minter.Calls(), 3, "one refusal, one re-entry, one refusal of that: then stop") +} + +// C2: a loss whose repair can never finish — every pass ends in a failure no +// retry fixes — is still closed once its window has passed, so status shows its +// ids as unrecovered rather than open forever. +func TestALossPastItsWindowClosesEvenWhenItsWalkCannotFinish(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-24*time.Hour), 10*time.Minute) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollFilterInvalid, Err: errors.New("x")}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open) + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered) +} From ae21ccaa8e41b90fa404eda58a5dcb497d46b6e9 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 22:56:49 +0200 Subject: [PATCH 15/49] Bump the SDK to main for its live feed seams, and re-sync the model and vendorHash on the rebase basecamp-sdk 12cbad5b carries 897 and 899 (eventfeed.NewLive), which intake binds to. The vendored MCP model and the Nix vendorHash are regenerated against it with main's sync policy, which after 725 serves the feed's poll operations and withholds only the stream-ticket mint. --- internal/commands/mcp_test.go | 2 +- internal/mcpserver/server_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/commands/mcp_test.go b/internal/commands/mcp_test.go index 8933a2e1a..2caaf42c2 100644 --- a/internal/commands/mcp_test.go +++ b/internal/commands/mcp_test.go @@ -141,7 +141,7 @@ func TestMCPCommandServesMCP(t *testing.T) { require.NoError(t, err) names = append(names, tool.Name) } - assert.Len(t, names, 16, "tools = %v", names) + assert.Len(t, names, 17, "tools = %v", names) result, err := session.CallTool(context.Background(), &mcp.CallToolParams{ Name: "basecamp_projects", diff --git a/internal/mcpserver/server_test.go b/internal/mcpserver/server_test.go index cfdeceee8..71bbf0673 100644 --- a/internal/mcpserver/server_test.go +++ b/internal/mcpserver/server_test.go @@ -41,7 +41,7 @@ func TestServerListsDomainTools(t *testing.T) { session := mcptest.Connect(t, srv.BuildMCPServer(slog.New(slog.DiscardHandler))) tools := mcptest.ListTools(t, session) - assert.Len(t, tools, 16) + assert.Len(t, tools, 17) require.Contains(t, tools, "basecamp_projects") projects := tools["basecamp_projects"] assert.Contains(t, projects.Description, "list_projects") From f999fbb2050137c5655f74b8361200a19f17c65c Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 23:16:06 +0200 Subject: [PATCH 16/49] Regenerate the vendored model on main's policy A hunk from an earlier sync on this branch kept CreateStreamTicket in the behavior model through the rebase. main withholds it; regenerating against SDK 4523eac restores main's file exactly. --- internal/mcpserver/model/behavior-model.json | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/mcpserver/model/behavior-model.json b/internal/mcpserver/model/behavior-model.json index 775b63ad8..8913ae7e7 100644 --- a/internal/mcpserver/model/behavior-model.json +++ b/internal/mcpserver/model/behavior-model.json @@ -294,18 +294,6 @@ ] } }, - "CreateStreamTicket": { - "idempotent": true, - "retry": { - "max": 3, - "base_delay_ms": 1000, - "backoff": "exponential", - "retry_on": [ - 429, - 503 - ] - } - }, "CreateTemplate": { "retry": { "max": 2, From 94a9d1e81b26209ddd42eaf1e76a750330b9e932 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 23:17:24 +0200 Subject: [PATCH 17/49] Restore the SQLite driver to go.mod and refresh the vendorHash on main's SDK pin The rebase took main's go.mod, which predates this branch's dependencies. modernc.org/sqlite is added back at v1.59.0, go.sum is tidied against SDK 4523eac, and the Nix vendorHash is recomputed. --- go.mod | 9 ++++++++- go.sum | 43 ++++++++++++++++++++++++++++++++++++++++--- nix/package.nix | 2 +- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d84cae6bd..d93a02396 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( golang.org/x/sys v0.48.0 golang.org/x/text v0.42.0 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.59.0 ) require ( @@ -54,6 +55,7 @@ require ( github.com/charmbracelet/x/windows v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/coder/websocket v1.8.15 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect github.com/danieljoos/wincred v1.2.3 // indirect github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect @@ -99,7 +101,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/itchyny/timefmt-go v0.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.27 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect @@ -108,10 +110,12 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oapi-codegen/runtime v1.7.0 // indirect github.com/oklog/ulid/v2 v2.1.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/secure-systems-lab/go-securesystemslib v0.11.0 // indirect github.com/segmentio/asm v1.1.3 // indirect @@ -144,4 +148,7 @@ require ( google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.11 // indirect k8s.io/klog/v2 v2.140.0 // indirect + modernc.org/libc v1.75.7 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index bc9a9b162..367115dd3 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,8 @@ github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJ github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -254,6 +256,8 @@ github.com/google/go-containerregistry v0.21.7 h1:/vPFuVXDjtFREsVArW+0h1CIl5urnO github.com/google/go-containerregistry v0.21.7/go.mod h1:kjSbt7/zMsKLWfnHrIvKvhXHUw91jbe9DNjPPJ32gXE= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/trillian v1.7.3 h1:hziW+vo4czis48tzx2GK5xRBl/ZxBA9B0/UR5avXOro= @@ -286,6 +290,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/vault/api v1.22.0 h1:+HYFquE35/B74fHoIeXlZIP2YADVboaPjaSicHEZiH0= @@ -319,8 +325,8 @@ github.com/letsencrypt/boulder v0.20260309.0 h1:kZynrxK3QfqLGx6hhoz+Rfs3hgltJs1p github.com/letsencrypt/boulder v0.20260309.0/go.mod h1:yG8lj8pNPZ8taq3oNdTpfBS+eC74IaEuiewqzVpXiWE= github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -346,6 +352,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/oapi-codegen/nullable v1.1.0 h1:eAh8JVc5430VtYVnq00Hrbpag9PFRGWLjxR1/3KntMs= github.com/oapi-codegen/nullable v1.1.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/oapi-codegen/runtime v1.7.0 h1:t7358VYPvNbWJ9gdAkIK/smVeHpBf6yp8VTsaZsb/7k= @@ -360,6 +368,8 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= @@ -487,7 +497,6 @@ golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7 golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= @@ -519,6 +528,34 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= diff --git a/nix/package.nix b/nix/package.nix index c68146eaf..8a67a0b58 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -8,7 +8,7 @@ buildGoModule.override { go = go_1_26; } (finalAttrs: { src = lib.cleanSource ./..; # To update: set to lib.fakeHash, run `nix build`, use the hash from the error. - vendorHash = "sha256-mT0oVMMRcmpbszb0zVYd1k8cZfMHKHZRbhsy/+8GHxo="; + vendorHash = "sha256-wRIfLiw4cOs8y9NxSAuPB390ZV1q8YALuQb6EbEVGsI="; subPackages = [ "cmd/basecamp" ]; From de87973a9e36cab71da8ee19c47a3dfa6ed48dfb Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 23:32:17 +0200 Subject: [PATCH 18/49] Strip terminal controls from pointer lines, and share one whole-line writer with admission The pointer line wrote the feed's event type, kind and action as the API sent them. JSON escapes C0 controls but passes C1 controls such as U+009B (CSI) through as raw UTF-8, so a crafted value could drive the terminal of anyone watching the stream. Those strings now go through richtext.SanitizeTerminal, the rule admission's lines already apply. The line itself is now written through internal/connector/ndjson, one writer for both stdout protocols: a line is written whole or reported failed, and concurrent writers never interleave. It is admission's existing write loop, moved rather than copied, and admission now uses it too; its short-write test passes unchanged. --- internal/connector/admission/run.go | 29 +++------- internal/connector/intake.go | 38 +++++++------- internal/connector/ndjson/ndjson.go | 54 +++++++++++++++++++ internal/connector/ndjson/ndjson_test.go | 37 +++++++++++++ internal/connector/pointer_test.go | 67 ++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 41 deletions(-) create mode 100644 internal/connector/ndjson/ndjson.go create mode 100644 internal/connector/ndjson/ndjson_test.go create mode 100644 internal/connector/pointer_test.go diff --git a/internal/connector/admission/run.go b/internal/connector/admission/run.go index df5700932..1d57728f5 100644 --- a/internal/connector/admission/run.go +++ b/internal/connector/admission/run.go @@ -2,13 +2,13 @@ package admission import ( "context" - "encoding/json" "errors" "fmt" "io" "log/slog" "sync" + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" "github.com/basecamp/basecamp-cli/internal/richtext" ) @@ -179,32 +179,19 @@ func LineFor(v Verdict) Line { } type lineWriter struct { - mu sync.Mutex - w io.Writer + w io.Writer + + once sync.Once + out *ndjson.Writer } func (l *lineWriter) write(v Verdict) error { if l.w == nil { return nil } - b, err := json.Marshal(LineFor(v)) - if err != nil { - return fmt.Errorf("admission: encode line: %w", err) - } - l.mu.Lock() - defer l.mu.Unlock() - // One line, whole: a writer that takes part of it is written the rest, - // and one that takes none without an error has failed. A torn line is - // worse than no line to whoever parses the stream. - for rest := append(b, '\n'); len(rest) > 0; { - n, err := l.w.Write(rest) - if err != nil { - return fmt.Errorf("admission: write line: %w", err) - } - if n <= 0 { - return fmt.Errorf("admission: write line: %w", io.ErrShortWrite) - } - rest = rest[n:] + l.once.Do(func() { l.out = ndjson.NewWriter(l.w) }) + if err := l.out.WriteLine(LineFor(v)); err != nil { + return fmt.Errorf("admission: %w", err) } return nil } diff --git a/internal/connector/intake.go b/internal/connector/intake.go index b5e5c50ea..38fb42e88 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -2,7 +2,6 @@ package connector import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -13,6 +12,9 @@ import ( "time" "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" + "github.com/basecamp/basecamp-cli/internal/richtext" ) // Defaults for the recovery timers. @@ -947,15 +949,13 @@ func (in *Intake) reconnectRequested() bool { } } -// pointerWriter serializes the NDJSON pointer lines. One line is one write: -// two goroutines interleaving inside a line tear it, and the reader on the -// other end has no way to recover a torn line. +// pointerWriter writes the NDJSON pointer lines through the connector's shared +// line writer: whole lines, never interleaved. type pointerWriter struct { - mu sync.Mutex - w io.Writer + out *ndjson.Writer } -func newPointerWriter(w io.Writer) *pointerWriter { return &pointerWriter{w: w} } +func newPointerWriter(w io.Writer) *pointerWriter { return &pointerWriter{out: ndjson.NewWriter(w)} } // Pointer is the line intake writes to stdout for each newly seen event. It // carries what the feed carried and nothing more: no title, no body, no URL, @@ -975,14 +975,17 @@ type Pointer struct { } func (p *pointerWriter) write(event eventfeed.Event, lane Lane) error { - if p.w == nil { - return nil - } - line, err := json.Marshal(Pointer{ + // Every string from Basecamp is stripped of terminal controls. The line is + // a wire, but it is also what a person watching the connector sees: JSON + // escapes C0 controls but passes C1 controls such as U+009B (CSI) through + // as raw UTF-8, which a terminal executes. Admission's lines apply the + // same rule. + clean := richtext.SanitizeTerminal + err := p.out.WriteLine(Pointer{ EventID: event.ID, - EventType: event.EventType, - Kind: event.Kind, - Action: event.Action, + EventType: clean(event.EventType), + Kind: clean(event.Kind), + Action: clean(event.Action), BucketID: event.BucketID, CreatorID: event.CreatorID, PerformedByID: event.PerformedByID, @@ -992,12 +995,7 @@ func (p *pointerWriter) write(event eventfeed.Event, lane Lane) error { State: string(StateSeen), }) if err != nil { - return fmt.Errorf("connector: encode pointer line: %w", err) - } - p.mu.Lock() - defer p.mu.Unlock() - if _, err := p.w.Write(append(line, '\n')); err != nil { - return fmt.Errorf("connector: write pointer line: %w", err) + return fmt.Errorf("connector: pointer line: %w", err) } return nil } diff --git a/internal/connector/ndjson/ndjson.go b/internal/connector/ndjson/ndjson.go new file mode 100644 index 000000000..492f86ec1 --- /dev/null +++ b/internal/connector/ndjson/ndjson.go @@ -0,0 +1,54 @@ +// Package ndjson writes the connector's stdout protocol: one JSON value per +// line, never torn. +// +// Intake's pointer lines and admission's verdict lines are both read by a +// process parsing the stream and watched by a person, so both need the same +// two guarantees. A line is written whole or reported failed, and concurrent +// writers never interleave inside a line. Callers sanitize API-controlled +// strings before encoding (richtext.SanitizeTerminal): JSON escapes C0 controls +// but passes C1 controls such as U+009B through as raw UTF-8. +package ndjson + +import ( + "encoding/json" + "fmt" + "io" + "sync" +) + +// Writer serializes whole lines onto w. The zero value with a nil w discards. +type Writer struct { + mu sync.Mutex + w io.Writer +} + +// NewWriter returns a Writer over w. A nil w discards every line. +func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } + +// WriteLine encodes v and writes it followed by a newline, whole. +// +// A writer that takes part of the line is written the rest, and one that takes +// none without an error has failed (io.ErrShortWrite): a torn line is worse +// than no line to whoever parses the stream. +func (l *Writer) WriteLine(v any) error { + if l == nil || l.w == nil { + return nil + } + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("encode line: %w", err) + } + l.mu.Lock() + defer l.mu.Unlock() + for rest := append(b, '\n'); len(rest) > 0; { + n, err := l.w.Write(rest) + if err != nil { + return fmt.Errorf("write line: %w", err) + } + if n <= 0 { + return fmt.Errorf("write line: %w", io.ErrShortWrite) + } + rest = rest[n:] + } + return nil +} diff --git a/internal/connector/ndjson/ndjson_test.go b/internal/connector/ndjson/ndjson_test.go new file mode 100644 index 000000000..3b5421d18 --- /dev/null +++ b/internal/connector/ndjson/ndjson_test.go @@ -0,0 +1,37 @@ +package ndjson + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type threeBytes struct{ bytes.Buffer } + +func (w *threeBytes) Write(p []byte) (int, error) { + if len(p) > 3 { + p = p[:3] + } + return w.Buffer.Write(p) +} + +type stuck struct{} + +func (stuck) Write([]byte) (int, error) { return 0, nil } + +func TestWriteLineCompletesShortWrites(t *testing.T) { + var w threeBytes + require.NoError(t, NewWriter(&w).WriteLine(map[string]int{"event_id": 42})) + assert.Equal(t, "{\"event_id\":42}\n", w.String()) +} + +func TestWriteLineReportsAWriterThatTakesNothing(t *testing.T) { + assert.ErrorIs(t, NewWriter(stuck{}).WriteLine(1), io.ErrShortWrite) +} + +func TestWriteLineWithNoWriterDiscards(t *testing.T) { + assert.NoError(t, NewWriter(nil).WriteLine(1)) +} diff --git a/internal/connector/pointer_test.go b/internal/connector/pointer_test.go new file mode 100644 index 000000000..c92ac0ecc --- /dev/null +++ b/internal/connector/pointer_test.go @@ -0,0 +1,67 @@ +package connector + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Control characters built from their code points, so the canaries are +// unmistakable in review. +var ( + esc = string(rune(0x1b)) + bel = string(rune(0x07)) + csi = string(rune(0x9b)) // C1 Control Sequence Introducer + osc = string(rune(0x9d)) // C1 Operating System Command + st = string(rune(0x9c)) // C1 String Terminator +) + +// The pointer line is a wire, and it is also what a person watching the +// connector sees. The event type, kind and action come from Basecamp; JSON +// escapes C0 controls but passes C1 controls such as CSI through as raw UTF-8, +// which a terminal executes. +func TestPointerLineCarriesNoTerminalControls(t *testing.T) { + var out bytes.Buffer + intake, _, _ := newTestIntake(t, nil, &out) + event := testEvent(1) + event.EventType = "comment.created" + csi + "31m" + esc + "]0;owned" + bel + event.Kind = "comment_created" + esc + "[2J" + event.Action = "created" + osc + "8;;evil" + st + require.NoError(t, intake.ingest(context.Background(), event, LanePoll)) + + line := out.String() + for name, control := range map[string]string{"ESC": esc, "CSI": csi, "OSC": osc, "ST": st, "BEL": bel} { + assert.NotContains(t, line, control, name) + } + var pointer Pointer + require.NoError(t, json.Unmarshal([]byte(strings.TrimSpace(line)), &pointer)) + assert.True(t, strings.HasPrefix(pointer.EventType, "comment.created")) +} + +// shortWriter accepts at most three bytes per call. +type shortWriter struct{ bytes.Buffer } + +func (w *shortWriter) Write(p []byte) (int, error) { + if len(p) > 3 { + p = p[:3] + } + return w.Buffer.Write(p) +} + +// A writer that takes part of a line is written the rest: a torn line is worse +// than no line to whoever parses the stream. +func TestPointerLineIsWrittenWholeThroughShortWrites(t *testing.T) { + var out shortWriter + intake, _, _ := newTestIntake(t, nil, &out) + require.NoError(t, intake.ingest(context.Background(), testEvent(1), LanePoll)) + + var pointer Pointer + require.NoError(t, json.Unmarshal(bytes.TrimSpace(out.Bytes()), &pointer)) + assert.Equal(t, int64(1), pointer.EventID) + assert.True(t, bytes.HasSuffix(out.Bytes(), []byte("\n"))) +} From b592064257e58b410094539212851c0ad3494279 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 23:45:11 +0200 Subject: [PATCH 19/49] Keep an explicit --since as the entry until a checkpoint supersedes it --since was cleared after the first connection whatever it had achieved. A connection that ended before saving its first page, for instance a reconnect for a newly visible project raised mid-page, left the next connection with no position and no poll-served id, so it entered at the present and skipped the history the operator asked for. The explicit entry now stands until the package reports a saved checkpoint. --- internal/connector/intake.go | 25 +++++++++++-- internal/connector/since_test.go | 60 ++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 internal/connector/since_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 38fb42e88..36a12284f 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -144,6 +144,9 @@ type Intake struct { replaying bool // reentryReplays is whether the pending re-entry is a replay. reentryReplays bool + // checkpointed is whether any connection in this run has saved a + // position. + checkpointed bool // enteredByReentry is whether this connection is itself the safe // re-entry after a refused position. enteredByReentry bool @@ -258,9 +261,14 @@ func (in *Intake) Run(ctx context.Context) error { since := in.opts.SinceEventID for { err := in.runOnce(ctx, since) - // --since is an entry, not a standing instruction: a reconnect - // resumes from what this run stored. - since = 0 + // --since is an entry, not a standing instruction: once a checkpoint + // has been saved, a reconnect resumes from it. Until then the explicit + // entry is the only position this run has, and a connection that ends + // before its first save must not leave the next one entering at the + // present. + if in.checkpointSaved() { + since = 0 + } if ctx.Err() != nil { return ctx.Err() } @@ -582,6 +590,11 @@ func (in *Intake) observer(ctx context.Context) eventfeed.Observer { // nothing, so a quiet feed is never proof of a quiet project. in.log.Info("feed walk reached its head and the buffer drained") }, + Checkpoint: func(string) { + in.mu.Lock() + in.checkpointed = true + in.mu.Unlock() + }, CheckpointSaveFailed: func(err error) { in.log.Error("could not save the feed position", "error", err) }, @@ -843,6 +856,12 @@ func (in *Intake) setReplaying(replaying bool) { in.mu.Unlock() } +func (in *Intake) checkpointSaved() bool { + in.mu.Lock() + defer in.mu.Unlock() + return in.checkpointed +} + // abort ends the current connection and the run with err. func (in *Intake) abort(err error) { in.mu.Lock() diff --git a/internal/connector/since_test.go b/internal/connector/since_test.go new file mode 100644 index 000000000..28edc0523 --- /dev/null +++ b/internal/connector/since_test.go @@ -0,0 +1,60 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// A3: an explicit --since stays the entry until a checkpoint supersedes it. A +// connection that ends before its first page is saved — here, a reconnect for +// a project the live subscription does not hold, raised mid-page — must not +// leave the next connection with no position, entering at the present and +// skipping the history the operator asked for. +func TestAnExplicitSinceSurvivesAReconnectBeforeTheFirstCheckpoint(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + SinceEventID: 17099838000, + Membership: &flakyMembership{buckets: []int64{48699913}}, + MembershipInterval: time.Hour, + }) + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + + unlisted := testEvent(17099838001) + unlisted.BucketID = 777 + // The first event raises the reconnect; the second is then refused + // delivery, so the page is never checkpointed. + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{unlisted, testEvent(17099838002)}, Position: "never-saved"}) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838002)}, Position: "saved"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + return polls.CallCount() >= 2 + }, 5*time.Second, 10*time.Millisecond) + + assert.Equal(t, "17099838000", polls.Calls()[1].Cursor.Since, + "nothing was checkpointed, so the reconnect still enters where the operator asked") + + require.Eventually(t, func() bool { + _, ok, err := ledger.Get(ctx, 17099838002) + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond, "no event after --since is skipped") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} From 00be65c7c6cd2631db4e89c7f9ff06f9b5ecb5f4 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Wed, 16 Sep 2026 23:59:34 +0200 Subject: [PATCH 20/49] Bound repair passes against cycles and endless pages, propagate cancellation, and pin SDK v0.19.0 A repair pass followed next for as long as the server supplied one. A next pointing at itself, or a cycle between two URLs, spun without returning to the repair cadence or checking the window; distinct signed positions forever would have done the same. A pass now ends when a next repeats a URL it already walked, and after a thousand pages, returning the cursor saved on its last page so the next pass resumes there. The repair cadence is the backoff either way. Cancellation during a repair walk now propagates as the caller's error instead of being handled like a transient poll failure, and neither the final pass nor the window check closes a loss once the context is done. Until now that protection held only because the ledger refuses reads on a canceled context; it no longer rests on the driver. The SDK is pinned to the go/v0.19.0 tag (6abe227), one commit past the 4523eac pseudo-version whose only change is version strings. The model files are unchanged; the provenance record names the tag, and the Nix vendorHash is refreshed. --- go.mod | 2 +- go.sum | 4 +- internal/connector/repair.go | 47 +++++++- internal/connector/repair_bounds_test.go | 143 +++++++++++++++++++++++ internal/mcpserver/model/PROVENANCE.json | 4 +- internal/version/sdk-provenance.json | 6 +- nix/package.nix | 2 +- 7 files changed, 198 insertions(+), 10 deletions(-) create mode 100644 internal/connector/repair_bounds_test.go diff --git a/go.mod b/go.mod index d93a02396..14709182e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( charm.land/bubbles/v2 v2.2.1 charm.land/bubbletea/v2 v2.0.9 charm.land/lipgloss/v2 v2.0.6 - github.com/basecamp/basecamp-sdk/go v0.18.1-0.20260916210227-4523eac74cfa + github.com/basecamp/basecamp-sdk/go v0.19.0 github.com/basecamp/cli v0.2.2-0.20260828230226-767413fc712d github.com/basecamp/mcp v0.0.0-20260828100356-2d6f44b51e9d github.com/basecamp/surfguard/go v0.1.0 diff --git a/go.sum b/go.sum index 367115dd3..389256aba 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/basecamp/basecamp-sdk/go v0.18.1-0.20260916210227-4523eac74cfa h1:efZJJSiwKn5lBih05oVfLM6S+ZYN6kB0vcH1DQmd8a8= -github.com/basecamp/basecamp-sdk/go v0.18.1-0.20260916210227-4523eac74cfa/go.mod h1:kIBDYwPMMD59PadNGxpH0YTQuI+blFPZ8MelGI0RK5Q= +github.com/basecamp/basecamp-sdk/go v0.19.0 h1:byygVVbJnWCZsyBNeAlztlUAV23ytLEhPx98WakNy+c= +github.com/basecamp/basecamp-sdk/go v0.19.0/go.mod h1:kIBDYwPMMD59PadNGxpH0YTQuI+blFPZ8MelGI0RK5Q= github.com/basecamp/cli v0.2.2-0.20260828230226-767413fc712d h1:jAzDrCCzDpIwhbFT1xVVs0z2xpXoDEkomHfKB2bUUp8= github.com/basecamp/cli v0.2.2-0.20260828230226-767413fc712d/go.mod h1:iTBTaWvsPEFIcZfkxQHEfISyJ6sZ7036K6bNx0RY3EE= github.com/basecamp/mcp v0.0.0-20260828100356-2d6f44b51e9d h1:zEQVGq1x1nhKMZ2TudFAcSJ32CHT8richI1vQakIKz4= diff --git a/internal/connector/repair.go b/internal/connector/repair.go index e780bb5c1..a4be68bfe 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -24,6 +24,8 @@ import ( type repairWalker struct { ledger *Ledger polls eventfeed.PollSource + // maxPages caps the pages one pass walks; zero means maxRepairPagesPerPass. + maxPages int // origin is the API origin every URL the walk follows must stay on. origin string filters eventfeed.Filters @@ -103,6 +105,11 @@ func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { if _, err := w.walk(ctx, loss); err != nil && !errors.Is(err, errReconciliationEnded) { return err } + if err := ctx.Err(); err != nil { + // The final attempt did not run to its end; closing now would condemn + // ids on a pass that never happened. + return err + } if done, err := w.settled(ctx, *loss); err != nil || done { return err } @@ -113,6 +120,9 @@ func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { // A walk that ends in a failure no retry fixes leaves the loss open for the // next start — but not forever. func (w *repairWalker) closeIfExpired(ctx context.Context, loss Loss) error { + if err := ctx.Err(); err != nil { + return err + } if w.now().Before(loss.DeadlineAt) { return nil } @@ -166,7 +176,25 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { last := loss.RepairCursor pass := &repairPass{followed: map[string]bool{}} - for { + walked := map[string]bool{} + maxPages := w.maxPages + if maxPages <= 0 { + maxPages = maxRepairPagesPerPass + } + for pages := 0; ; { + if err := ctx.Err(); err != nil { + // Cancellation is a delay, never a verdict: the loss stays open + // on disk for the next start. + return last, err + } + if pages >= maxPages { + // Distinct positions forever evade cycle detection. The pass ends + // at the cap, and the next one resumes from the cursor saved on + // the last page. + w.log.Warn("a repair pass reached its page cap; resuming on the repair cadence", "loss_id", loss.ID, "pages", pages) + return last, nil + } + pages++ page, err := w.polls.Poll(ctx, cursor, w.filters) if err != nil { next, err := w.pollFailure(ctx, loss, cursor, err, pass) @@ -205,6 +233,13 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { } // An empty page with a `next` is ordinary — the walk crossed rows the // filters excluded — so the loop never stops on len(Events) == 0. + if walked[page.Next] || page.Next == cursor.PageURL { + // A next already walked this pass — itself, or a cycle — would + // spin. The pass ends and the repair cadence is the backoff. + w.log.Warn("a repair page's next repeats a URL this pass already walked; ending the pass", "loss_id", loss.ID) + return last, nil + } + walked[page.Next] = true if err := sameOrigin(w.origin, page.Next); err != nil { w.log.Error("a repair page's next URL leaves the API origin; the loss stays open for the next start", "loss_id", loss.ID) return last, errReconciliationEnded @@ -219,6 +254,11 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { // full recovery. var errReconciliationEnded = errors.New("connector: reconciliation ended inside the repair walk") +// maxRepairPagesPerPass bounds the pages one repair pass walks. A thousand +// pages crosses up to a million ledger rows; past that the pass yields to the +// repair cadence and resumes from its saved cursor. +const maxRepairPagesPerPass = 1000 + // maxResumesPerPass bounds the 410 resumes one pass follows. Keyed by URL // alone, a server that signs or nonces its resume URLs would make every answer // look new; the bound does not depend on the server choosing stable URLs. @@ -253,6 +293,11 @@ func failureKind(err error) string { // to continue this pass at, nil with no error to end the pass and wait for the // next repair poll, or an error. func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor eventfeed.Cursor, err error, pass *repairPass) (*eventfeed.Cursor, error) { + if ctxErr := ctx.Err(); ctxErr != nil { + // The caller's cancellation, not the walk failing. It propagates, so + // nothing downstream mistakes a shutdown for a pass that ran. + return nil, ctxErr + } var pollErr *eventfeed.PollError if !errors.As(err, &pollErr) { w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go new file mode 100644 index 000000000..9ad2dbe1f --- /dev/null +++ b/internal/connector/repair_bounds_test.go @@ -0,0 +1,143 @@ +package connector + +import ( + "context" + "strconv" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// nextPolls serves empty pages whose next URL is chosen by pick, and gives up +// after limit calls so a walk that never stops fails the test instead of +// hanging it. +type nextPolls struct { + mu sync.Mutex + calls int + limit int + pick func(call int) string +} + +func (n *nextPolls) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + n.mu.Lock() + defer n.mu.Unlock() + n.calls++ + if n.calls > n.limit { + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollUnrecoverable} + } + return eventfeed.PollPage{Position: "pos-" + strconv.Itoa(n.calls), Next: n.pick(n.calls)}, nil +} + +const walkBase = "https://3.basecampapi.com/2914079/events.json?position=" + +func onePassWalker(t *testing.T, polls eventfeed.PollSource) (*repairWalker, *Ledger, Loss) { + t.Helper() + ledger := newTestLedger(t) + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at, 10*time.Minute) + require.NoError(t, err) + walker, _ := newTestWalker(t, ledger, polls, clock) + return walker, ledger, loss +} + +// C3: a next that points at itself ends the pass instead of spinning. +func TestARepairWalkStopsOnASelfReferentialNext(t *testing.T) { + polls := &nextPolls{limit: 300, pick: func(int) string { return walkBase + "same" }} + walker, _, loss := onePassWalker(t, polls) + + _, err := walker.walk(context.Background(), &loss) + require.NoError(t, err) + assert.LessOrEqual(t, polls.calls, 3, "a next already walked this pass is a cycle") +} + +// C3: an A→B→A cycle ends the pass too. +func TestARepairWalkStopsOnACycleOfNextURLs(t *testing.T) { + polls := &nextPolls{limit: 300, pick: func(call int) string { + if call%2 == 1 { + return walkBase + "A" + } + return walkBase + "B" + }} + walker, _, loss := onePassWalker(t, polls) + + _, err := walker.walk(context.Background(), &loss) + require.NoError(t, err) + assert.LessOrEqual(t, polls.calls, 4) +} + +// C3: distinct positions forever evade cycle detection, so a pass is also +// capped in pages, and returns the saved cursor so the next pass resumes. +func TestARepairPassIsCappedInPages(t *testing.T) { + polls := &nextPolls{limit: 300, pick: func(call int) string { return walkBase + "distinct-" + strconv.Itoa(call) }} + walker, ledger, loss := onePassWalker(t, polls) + walker.maxPages = 20 + + cursor, err := walker.walk(context.Background(), &loss) + require.NoError(t, err) + assert.Equal(t, 20, polls.calls) + assert.Equal(t, "pos-20", cursor, "the next pass resumes from the last saved page") + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + require.Len(t, open, 1) + assert.Equal(t, "pos-20", open[0].RepairCursor) +} + +// cancelingPolls cancels the walk's context on its first call, as a shutdown +// arriving mid-walk would, and answers with the context's error. +type cancelingPolls struct { + cancel context.CancelFunc + calls int +} + +func (c *cancelingPolls) Poll(ctx context.Context, _ eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { + c.calls++ + c.cancel() + return eventfeed.PollPage{}, ctx.Err() +} + +// C4: cancellation is not the walk failing. A shutdown mid-walk leaves the +// loss open, with nothing condemned, and the next start completes it. +func TestACanceledRepairWalkLeavesTheLossOpenForTheNextStart(t *testing.T) { + for name, age := range map[string]time.Duration{ + "inside the window": 0, + "on the final pass": 24 * time.Hour, + } { + t.Run(name, func(t *testing.T) { + ledger := newTestLedger(t) + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at.Add(-age), 10*time.Minute) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + walker, _ := newTestWalker(t, ledger, &cancelingPolls{cancel: cancel}, clock) + err = walker.reconcile(ctx, loss) + assert.ErrorIs(t, err, context.Canceled) + + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + require.Len(t, open, 1, "a shutdown mid-walk is a delay, not a verdict") + unrecovered, err := ledger.UnrecoveredIDs(context.Background()) + require.NoError(t, err) + assert.Empty(t, unrecovered) + + // The next start completes it. + restart, _ := newTestWalker(t, ledger, &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "after"}, + }}, clock) + require.NoError(t, restart.reconcile(context.Background(), open[0])) + open, err = ledger.OpenLosses(context.Background()) + require.NoError(t, err) + assert.Empty(t, open) + recovered, err := ledger.MissingIDs(context.Background(), loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, recovered) + }) + } +} diff --git a/internal/mcpserver/model/PROVENANCE.json b/internal/mcpserver/model/PROVENANCE.json index 25c72af63..41423b4a7 100644 --- a/internal/mcpserver/model/PROVENANCE.json +++ b/internal/mcpserver/model/PROVENANCE.json @@ -1,7 +1,7 @@ { "source": "github.com/basecamp/basecamp-sdk", - "commit": "4523eac74cfa6452ec85ea5f8f633baa4dc487a5", - "ref": "go/v0.18.0-63-g4523eac7", + "commit": "6abe227204a465ced72969f10e32d3a06e77bcc1", + "ref": "go/v0.19.0", "files": ["behavior-model.json", "openapi.json"], "synced_by": "scripts/sync-mcp-model.sh", "patches": "binary-upload operations dropped because the toolkit refuses their non-JSON bodies (EXCLUDED_OPERATIONS) and the stream-ticket mint dropped by policy (POLICY_EXCLUDED_OPERATIONS); no tag patches applied (PATCHED_TAGS is empty — the export tags every operation) — see the sync script" diff --git a/internal/version/sdk-provenance.json b/internal/version/sdk-provenance.json index 3ff436f25..d25aac38f 100644 --- a/internal/version/sdk-provenance.json +++ b/internal/version/sdk-provenance.json @@ -1,9 +1,9 @@ { "sdk": { "module": "github.com/basecamp/basecamp-sdk/go", - "version": "v0.18.1-0.20260916210227-4523eac74cfa", - "revision": "4523eac74cfa", - "updated_at": "2026-09-16T21:02:27Z" + "version": "v0.19.0", + "revision": "6abe227204a4", + "updated_at": "2026-09-16T21:15:28Z" }, "api": { "repo": "basecamp/bc3", diff --git a/nix/package.nix b/nix/package.nix index 8a67a0b58..fb7068ab7 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -8,7 +8,7 @@ buildGoModule.override { go = go_1_26; } (finalAttrs: { src = lib.cleanSource ./..; # To update: set to lib.fakeHash, run `nix build`, use the hash from the error. - vendorHash = "sha256-wRIfLiw4cOs8y9NxSAuPB390ZV1q8YALuQb6EbEVGsI="; + vendorHash = "sha256-75G+NnWwmBZ0dHaTWOd9dQLuyOaV6kpx8B+h8SERU8k="; subPackages = [ "cmd/basecamp" ]; From 3d93b57339e010b9eb5ca586c7475cfd5c05dc0a Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 00:13:30 +0200 Subject: [PATCH 21/49] Re-apply this branch's requirements on main's dependency bump The rebase onto #719 took go.mod from main and re-added the SDK v0.19.0 pin and modernc.org/sqlite v1.59.0; go.sum is tidied and the Nix vendorHash recomputed. --- nix/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/package.nix b/nix/package.nix index fb7068ab7..82e2a2522 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -8,7 +8,7 @@ buildGoModule.override { go = go_1_26; } (finalAttrs: { src = lib.cleanSource ./..; # To update: set to lib.fakeHash, run `nix build`, use the hash from the error. - vendorHash = "sha256-75G+NnWwmBZ0dHaTWOd9dQLuyOaV6kpx8B+h8SERU8k="; + vendorHash = "sha256-fbSMybSFUlSHIE8/aqLH/QCu+Q3cvJV/qpBAKz7VAZI="; subPackages = [ "cmd/basecamp" ]; From cd3c9b8fb2900f7140eda2d355836ff0d1e29eda Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 00:15:16 +0200 Subject: [PATCH 22/49] One NDJSON writer per sink, and an unknown membership snapshot that says so Intake and admission each built a writer for the stdout they were handed, so two locks guarded one sink and a pointer line could tear a verdict line. A sink now has one writer, and one lock, for the life of the process, whoever asks for it. A membership read that failed at subscribe left the snapshot nil, and an event from a project the live subscription may not hold was treated as already known, so the stale subscription could stay stale. An unknown snapshot is now unknown: the first such event reconnects once, and the failed read is retried on a backoff that doubles up to the membership interval rather than waiting a full one. --- internal/connector/intake.go | 63 ++++++++++---- internal/connector/membership_test.go | 100 +++++++++++++++++++++++ internal/connector/ndjson/ndjson.go | 35 +++++++- internal/connector/ndjson/shared_test.go | 64 +++++++++++++++ 4 files changed, 243 insertions(+), 19 deletions(-) create mode 100644 internal/connector/membership_test.go create mode 100644 internal/connector/ndjson/shared_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 36a12284f..e780fd13b 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -125,6 +125,8 @@ type Intake struct { // served is the current connection's wrapped poll source. served *servedPolls snapshot map[int64]bool + // membershipRetry is the first backoff after a failed membership read. + membershipRetry time.Duration // learned holds buckets events proved visible that the lister did not // name. learned map[int64]bool @@ -774,7 +776,10 @@ func (in *Intake) takeSnapshot(ctx context.Context) { // the new project immediately; the live lane cannot until it re-subscribes. func (in *Intake) noteBucket(bucketID int64) { in.mu.Lock() - known := in.snapshot == nil || in.snapshot[bucketID] + // With no snapshot — the read at subscribe failed — the live + // subscription's buckets are unknown, not "everything". An event from a + // bucket not yet learned asks for one reconnect. + known := (in.snapshot == nil && in.opts.Membership == nil) || in.snapshot[bucketID] in.mu.Unlock() if known { return @@ -788,7 +793,9 @@ func (in *Intake) noteBucket(bucketID int64) { in.learned = make(map[int64]bool) } in.learned[bucketID] = true - in.snapshot[bucketID] = true + if in.snapshot != nil { + in.snapshot[bucketID] = true + } in.mu.Unlock() in.requestReconnect() } @@ -885,31 +892,53 @@ func (in *Intake) watchMembership(ctx context.Context) func() { done := make(chan struct{}) go func() { defer close(done) - ticker := time.NewTicker(in.opts.MembershipInterval) - defer ticker.Stop() + retry := in.membershipRetry + if retry <= 0 { + retry = defaultMembershipRetry + } for { + // A snapshot the subscribe-time read never produced is retried on + // a backoff rather than left for a full interval: until one exists + // there is no baseline to notice a change against. + wait := in.opts.MembershipInterval + if !in.hasSnapshot() { + wait = min(retry, in.opts.MembershipInterval) + retry = min(retry*2, in.opts.MembershipInterval) + } + timer := time.NewTimer(wait) select { case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + buckets, err := in.opts.Membership.Buckets(ctx) + if err != nil { + // A failed read leaves the snapshot alone. Treating it as a + // change would reconnect the feed every time the API + // hiccupped. + in.log.Warn("could not refresh the agent's projects", "error", err) + continue + } + if in.membershipChanged(buckets) { + in.requestReconnect() return - case <-ticker.C: - buckets, err := in.opts.Membership.Buckets(ctx) - if err != nil { - // A failed read leaves the snapshot alone. Treating it as - // a change would reconnect the feed every time the API - // hiccupped. - in.log.Warn("could not refresh the agent's projects", "error", err) - continue - } - if in.membershipChanged(buckets) { - in.requestReconnect() - return - } } } }() return func() { <-done } } +// defaultMembershipRetry is the first wait before re-reading a membership +// listing that failed; it doubles up to the membership interval. +const defaultMembershipRetry = 5 * time.Second + +func (in *Intake) hasSnapshot() bool { + in.mu.Lock() + defer in.mu.Unlock() + return in.snapshot != nil +} + // membershipChanged compares a fresh read with the snapshot. With no snapshot // — the read at subscribe failed — the fresh read becomes the baseline; // otherwise a failed first read would disable change detection for good. diff --git a/internal/connector/membership_test.go b/internal/connector/membership_test.go new file mode 100644 index 000000000..2324b0690 --- /dev/null +++ b/internal/connector/membership_test.go @@ -0,0 +1,100 @@ +package connector + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// failingFirstMembership fails its first read and serves buckets afterwards. +type failingFirstMembership struct { + mu sync.Mutex + calls atomic.Int32 + buckets []int64 +} + +func (m *failingFirstMembership) Buckets(context.Context) ([]int64, error) { + if m.calls.Add(1) == 1 { + return nil, errors.New("projects listing unavailable") + } + m.mu.Lock() + defer m.mu.Unlock() + return m.buckets, nil +} + +// E3: with the read at subscribe failed, the live subscription's buckets are +// unknown, not "everything". An event from a project the poll lane serves must +// still reconnect the live lane. +func TestAFailedFirstMembershipReadStillReconnectsForANewProject(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: &failingFirstMembership{buckets: []int64{48699913}}, + MembershipInterval: time.Hour, + }) + intake.membershipRetry = time.Hour + minter.ScriptTicket(ticket()) + minter.ScriptTicket(ticket()) + granted := testEvent(17099838600) + granted.BucketID = 777 + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{granted}, Position: "p1"}) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + return len(transport.Dials()) >= 2 + }, 5*time.Second, 10*time.Millisecond, "a project the subscription may not hold reconnects the live lane") + + // Learned once, it costs one reconnect, not one per event. + time.Sleep(100 * time.Millisecond) + assert.LessOrEqual(t, len(transport.Dials()), 2) + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// E3: a failed read is retried on a backoff, not left for a full membership +// interval with no baseline to compare against. +func TestAFailedFirstMembershipReadIsRetriedSoon(t *testing.T) { + ledger := newTestLedger(t) + membership := &failingFirstMembership{buckets: []int64{48699913}} + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: membership, + MembershipInterval: time.Hour, + }) + intake.membershipRetry = 20 * time.Millisecond + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return membership.calls.Load() >= 2 }, 2*time.Second, 10*time.Millisecond, + "the read is retried within its backoff, not after an hour") + require.Eventually(t, func() bool { + intake.mu.Lock() + defer intake.mu.Unlock() + return intake.snapshot != nil && intake.snapshot[48699913] + }, 2*time.Second, 10*time.Millisecond, "the first successful read becomes the baseline") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} diff --git a/internal/connector/ndjson/ndjson.go b/internal/connector/ndjson/ndjson.go index 492f86ec1..c45b1f429 100644 --- a/internal/connector/ndjson/ndjson.go +++ b/internal/connector/ndjson/ndjson.go @@ -13,6 +13,7 @@ import ( "encoding/json" "fmt" "io" + "reflect" "sync" ) @@ -22,8 +23,38 @@ type Writer struct { w io.Writer } -// NewWriter returns a Writer over w. A nil w discards every line. -func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } +// NewWriter returns the Writer for w. A nil w discards every line. +// +// There is one Writer, and so one lock, per sink for the life of the process. +// Intake and admission each build their writer from the stdout they are +// handed; if those were two Writers their locks would not coordinate, and a +// pointer line and a verdict line could tear each other inside one write. +// Keying on the sink makes that impossible whoever calls this, without every +// caller having to agree to pass one instance around. +// +// A sink whose type is not comparable cannot be the same sink twice (it is +// always a copy), so it gets a Writer of its own. +func NewWriter(w io.Writer) *Writer { + if w == nil { + return &Writer{} + } + if !reflect.TypeOf(w).Comparable() { + return &Writer{w: w} + } + sinksMu.Lock() + defer sinksMu.Unlock() + if existing, ok := sinks[w]; ok { + return existing + } + writer := &Writer{w: w} + sinks[w] = writer + return writer +} + +var ( + sinksMu sync.Mutex + sinks = map[io.Writer]*Writer{} +) // WriteLine encodes v and writes it followed by a newline, whole. // diff --git a/internal/connector/ndjson/shared_test.go b/internal/connector/ndjson/shared_test.go new file mode 100644 index 000000000..07773a6b3 --- /dev/null +++ b/internal/connector/ndjson/shared_test.go @@ -0,0 +1,64 @@ +package ndjson + +import ( + "bufio" + "bytes" + "encoding/json" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// lockedShortWriter is a sink that is safe for concurrent use but takes at +// most three bytes per call, so two writers that do not share a lock can +// interleave inside a line. +type lockedShortWriter struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (w *lockedShortWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + if len(p) > 3 { + p = p[:3] + } + return w.buf.Write(p) +} + +// One process stdout has one lock. Intake and admission each build their writer +// for the same sink; those writers must be one writer, or their lines tear +// each other. +func TestWritersForOneSinkShareOneLock(t *testing.T) { + sink := &lockedShortWriter{} + intake, admission := NewWriter(sink), NewWriter(sink) + + var wg sync.WaitGroup + for _, w := range []*Writer{intake, admission} { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 300 { + require.NoError(t, w.WriteLine(map[string]any{"line": i, "padding": "0123456789abcdef"})) + } + }() + } + wg.Wait() + + scanner := bufio.NewScanner(bytes.NewReader(sink.buf.Bytes())) + lines := 0 + for scanner.Scan() { + var v map[string]any + require.NoError(t, json.Unmarshal(scanner.Bytes(), &v), "line %d is not whole JSON: %q", lines, scanner.Text()) + lines++ + } + assert.Equal(t, 600, lines) +} + +func TestWritersForDifferentSinksAreIndependent(t *testing.T) { + var a, b bytes.Buffer + assert.NotSame(t, NewWriter(&a), NewWriter(&b)) + assert.Same(t, NewWriter(&a), NewWriter(&a)) +} From 0af12903dfc779710ea537b3b80de8934770b56c Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 00:30:08 +0200 Subject: [PATCH 23/49] Repair a loss under the filters it was recorded with, and keep learned projects while the lister is down A loss is repaired by walking the feed, and the walk was made with the connector's current filters. Restarted under a different filter set, a repair would walk a lane that never carried the loss's events and then condemn them. Migration 2 stores the filter set on the loss, and the walk uses it; migrations only append, so an existing ledger takes it and its open losses read as "the connector's own". While the project lister is failing there is no snapshot, and the last head treated every arriving event's project as unknown, so each event reconnected. The buckets already learned from arriving events are now consulted too: the first event from a project still reconnects once, and later ones do not. Only reference-like sinks are keyed in the line-writer registry. A comparable sink type holding an uncomparable field panics when hashed, and a value sink is a copy rather than the same sink. TestIntakeSurvivesARestartWithoutDuplicating asserted its counts the moment the ledger row appeared, which is before the pointer line and the hand-off; it waits for them now. That is the Race Detection failure on the last head. --- internal/connector/intake.go | 6 +-- internal/connector/intake_feed_test.go | 8 +++- internal/connector/intake_test.go | 6 ++- internal/connector/invariants_test.go | 4 +- internal/connector/ledger.go | 7 ++++ internal/connector/ledger_recovery.go | 30 +++++++++++--- internal/connector/loss_filters_test.go | 48 +++++++++++++++++++++++ internal/connector/membership_test.go | 50 ++++++++++++++++++++++++ internal/connector/ndjson/shared_test.go | 17 ++++++++ internal/connector/repair.go | 11 ++++++ internal/connector/repair_bounds_test.go | 4 +- internal/connector/repair_test.go | 18 ++++----- internal/connector/review_fixes_test.go | 18 ++++----- internal/connector/round4_test.go | 2 +- internal/connector/round6_test.go | 2 +- internal/connector/round8_test.go | 2 +- 16 files changed, 196 insertions(+), 37 deletions(-) create mode 100644 internal/connector/loss_filters_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index e780fd13b..64791b563 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -658,7 +658,7 @@ func (in *Intake) handleSignal(signal eventfeed.Signal) eventfeed.Disposition { return eventfeed.Accept case eventfeed.BufferOverflow: - loss, err := in.ledger.RecordLoss(ctx, s.DroppedIDs, in.now(), in.opts.RepairWindow) + loss, err := in.ledger.RecordLoss(ctx, s.DroppedIDs, in.now(), in.opts.RepairWindow, in.opts.Filters) if err != nil { // Accept means owning the incompleteness. Owning it begins with // it being on disk: accepting after a failed write would leave a @@ -737,7 +737,7 @@ func (in *Intake) startRepair(ctx context.Context, loss Loss) { log: in.log, sleep: in.repairSleep, } - switch err := walker.reconcile(ctx, loss); { + switch err := walker.reconcileLoss(ctx, loss); { case err == nil: case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): // A shutdown mid-walk is a delay: the loss is still open on disk @@ -779,7 +779,7 @@ func (in *Intake) noteBucket(bucketID int64) { // With no snapshot — the read at subscribe failed — the live // subscription's buckets are unknown, not "everything". An event from a // bucket not yet learned asks for one reconnect. - known := (in.snapshot == nil && in.opts.Membership == nil) || in.snapshot[bucketID] + known := (in.snapshot == nil && in.opts.Membership == nil) || in.snapshot[bucketID] || in.learned[bucketID] in.mu.Unlock() if known { return diff --git a/internal/connector/intake_feed_test.go b/internal/connector/intake_feed_test.go index a340afdd2..80b4fc086 100644 --- a/internal/connector/intake_feed_test.go +++ b/internal/connector/intake_feed_test.go @@ -254,8 +254,12 @@ func TestIntakeSurvivesARestartWithoutDuplicating(t *testing.T) { assert.Equal(t, "position-from-the-previous-run", calls[0].Cursor.Position, "a restart resumes from the stored position, never at the head") - assert.Equal(t, 1, countLines(pointers.String()), + // The ledger row is written before the pointer line and the hand-off, so + // the counts are asserted on their own terms. + require.Eventually(t, func() bool { return countLines(pointers.String()) == 1 && queue.Depth() == 1 }, + 5*time.Second, 5*time.Millisecond, "the event the previous run already saw is not a second unit of work") + assert.Equal(t, 1, countLines(pointers.String())) assert.Equal(t, 1, queue.Depth()) cancel() @@ -326,7 +330,7 @@ func TestShutdownDoesNotWaitOutAnOpenRepairWalk(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err = ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Hour) + _, err = ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Hour, eventfeed.Filters{}) require.NoError(t, err) transport := feedtest.NewTransport() diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go index 7879ccac0..700fae141 100644 --- a/internal/connector/intake_test.go +++ b/internal/connector/intake_test.go @@ -26,11 +26,13 @@ type scriptedPolls struct { pages []eventfeed.PollPage errs []error cursors []eventfeed.Cursor + filters []eventfeed.Filters calls int } -func (s *scriptedPolls) Poll(_ context.Context, cursor eventfeed.Cursor, _ eventfeed.Filters) (eventfeed.PollPage, error) { +func (s *scriptedPolls) Poll(_ context.Context, cursor eventfeed.Cursor, filters eventfeed.Filters) (eventfeed.PollPage, error) { s.cursors = append(s.cursors, cursor) + s.filters = append(s.filters, filters) i := s.calls s.calls++ if i < len(s.errs) && s.errs[i] != nil { @@ -284,7 +286,7 @@ func TestReconciliationResumesOnStart(t *testing.T) { intake, ledger, _ := newTestIntake(t, polls, nil) ctx := context.Background() - _, err := ledger.RecordLoss(ctx, []int64{17099838501}, time.Now(), time.Minute) + _, err := ledger.RecordLoss(ctx, []int64{17099838501}, time.Now(), time.Minute, eventfeed.Filters{}) require.NoError(t, err) require.NoError(t, intake.resumeReconciliation(ctx)) diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go index 3927c326f..bbc6648b4 100644 --- a/internal/connector/invariants_test.go +++ b/internal/connector/invariants_test.go @@ -26,7 +26,7 @@ func TestInvariantH1RepairLogsRenderNoFailureText(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Minute, eventfeed.Filters{}) require.NoError(t, err) var logs bytes.Buffer @@ -95,7 +95,7 @@ func TestInvariantC2A410OnAStoredPositionIsFollowedEvenAtTheFence(t *testing.T) ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) _, err = ledger.MarkUnrecoveredThrough(ctx, loss.ID, 150) require.NoError(t, err) diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index 149687442..a340ce504 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -265,6 +265,13 @@ CREATE TABLE gaps ( note TEXT NOT NULL DEFAULT '' ); `, + // Migration 2. A loss is repaired under the filter set it was recorded + // with: a connector restarted with different filters must not walk the + // wrong lane for it and condemn events the original filters would have + // served. Migrations only ever append, so an existing ledger takes this + // one and its open losses carry an empty set, which reads as "the + // connector's own". + `ALTER TABLE losses ADD COLUMN filters TEXT NOT NULL DEFAULT ''`, } func (l *Ledger) migrate(ctx context.Context) error { diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go index ee107db7f..e15c7001b 100644 --- a/internal/connector/ledger_recovery.go +++ b/internal/connector/ledger_recovery.go @@ -3,9 +3,12 @@ package connector import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "time" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" ) // LossState is what became of one id the live buffer dropped. @@ -38,6 +41,11 @@ type Loss struct { RepairCursor string DeadlineAt time.Time ResolvedAt *time.Time + // Filters is the filter set this loss was recorded under. The repair walk + // uses it, whatever the connector's current filters are: a loss recorded + // under one set and walked under another is repaired against a lane that + // never carried its events. + Filters eventfeed.Filters } // GapClass distinguishes the feed's two 410s. They mean different things and @@ -96,7 +104,7 @@ type Gap struct { // Dropped ids the ledger already holds were lost from the buffer, not from the // connector; they are recorded as never_lost so the repair walk does not go // looking for events it already has. -func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Time, window time.Duration) (Loss, error) { +func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Time, window time.Duration, filters eventfeed.Filters) (Loss, error) { if len(droppedIDs) == 0 { return Loss{}, errors.New("connector: buffer overflow with no dropped ids") } @@ -128,7 +136,13 @@ func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Ti } } + encodedFilters, err := json.Marshal(filters) + if err != nil { + return Loss{}, fmt.Errorf("connector: encode loss filters: %w", err) + } + loss := Loss{ + Filters: filters, DetectedAt: now.UTC(), DroppedCount: len(states), // One below the lowest missing id, so the walk's first page can serve @@ -145,8 +159,8 @@ func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Ti } res, err := tx.ExecContext(ctx, - `INSERT INTO losses (detected_at, dropped_count, repair_since, deadline_at, resolved_at) VALUES (?, ?, ?, ?, ?)`, - stamp(loss.DetectedAt), loss.DroppedCount, loss.RepairSince, stamp(loss.DeadlineAt), nullableStamp(loss.ResolvedAt)) + `INSERT INTO losses (detected_at, dropped_count, repair_since, deadline_at, resolved_at, filters) VALUES (?, ?, ?, ?, ?, ?)`, + stamp(loss.DetectedAt), loss.DroppedCount, loss.RepairSince, stamp(loss.DeadlineAt), nullableStamp(loss.ResolvedAt), string(encodedFilters)) if err != nil { return Loss{}, fmt.Errorf("connector: insert loss: %w", err) } @@ -172,7 +186,7 @@ func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Ti // first. Reconciliation resumes from this on every start. func (l *Ledger) OpenLosses(ctx context.Context) ([]Loss, error) { rows, err := l.db.QueryContext(ctx, ` -SELECT id, detected_at, dropped_count, repair_since, repair_cursor, deadline_at, resolved_at +SELECT id, detected_at, dropped_count, repair_since, repair_cursor, deadline_at, resolved_at, filters FROM losses WHERE resolved_at IS NULL ORDER BY id`) if err != nil { return nil, fmt.Errorf("connector: list open losses: %w", err) @@ -186,10 +200,16 @@ FROM losses WHERE resolved_at IS NULL ORDER BY id`) detected, deadline string resolved sql.NullString ) + var encodedFilters string if err := rows.Scan(&loss.ID, &detected, &loss.DroppedCount, &loss.RepairSince, - &loss.RepairCursor, &deadline, &resolved); err != nil { + &loss.RepairCursor, &deadline, &resolved, &encodedFilters); err != nil { return nil, fmt.Errorf("connector: scan loss: %w", err) } + if encodedFilters != "" { + if err := json.Unmarshal([]byte(encodedFilters), &loss.Filters); err != nil { + return nil, fmt.Errorf("connector: decode loss filters: %w", err) + } + } var err error if loss.DetectedAt, err = parseStamp(detected); err != nil { return nil, err diff --git a/internal/connector/loss_filters_test.go b/internal/connector/loss_filters_test.go new file mode 100644 index 000000000..ca96b1f4f --- /dev/null +++ b/internal/connector/loss_filters_test.go @@ -0,0 +1,48 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// C3: a loss is repaired under the filter set it was recorded with. A +// connector restarted with different filters would otherwise walk the wrong +// lane and condemn events the original filters would have served. +func TestALossIsRepairedUnderTheFilterSetItWasRecordedWith(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + recorded := eventfeed.Filters{Types: []string{"comment.created"}, Buckets: []int64{48699913}} + + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), 10*time.Minute, recorded) + require.NoError(t, err) + assert.Equal(t, recorded.Types, loss.Filters.Types) + + // A later start under a different filter set. + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + require.Equal(t, recorded.Types, open[0].Filters.Types) + require.Equal(t, recorded.Buckets, open[0].Filters.Buckets) + + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.filters = eventfeed.Filters{Types: []string{"card.created"}} // the connector's current set + require.NoError(t, walker.reconcileLoss(ctx, open[0])) + + require.NotEmpty(t, polls.filters) + assert.Equal(t, recorded.Types, polls.filters[0].Types, + "the walk uses the loss's own filters, not the connector's current ones") + + recoveredIDs, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, recoveredIDs) +} diff --git a/internal/connector/membership_test.go b/internal/connector/membership_test.go index 2324b0690..57e2cced9 100644 --- a/internal/connector/membership_test.go +++ b/internal/connector/membership_test.go @@ -98,3 +98,53 @@ func TestAFailedFirstMembershipReadIsRetriedSoon(t *testing.T) { cancel() awaitReturn(t, done, "Run should return on shutdown") } + +// E2/E3: while the project lister is failing, an unlisted project still costs +// one reconnect, not one per event. The buckets learned from arriving events +// are what the connector knows when it has no listing at all. +func TestAFailingListerStillCostsOneReconnectPerProject(t *testing.T) { + ledger := newTestLedger(t) + always := &alwaysFailingMembership{} + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{ + Membership: always, + MembershipInterval: time.Hour, + }) + intake.membershipRetry = time.Hour + for range 10 { + minter.ScriptTicket(ticket()) + } + var events []eventfeed.Event + for id := int64(600); id < 605; id++ { + e := testEvent(id) + e.BucketID = 777 + events = append(events, e) + } + for range 10 { + polls.ScriptPage(eventfeed.PollPage{Events: events, Position: "p"}) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + answered := 1 + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + time.Sleep(10 * time.Millisecond) + } + assert.LessOrEqual(t, len(transport.Dials()), 2, "five events from one project are one reconnect, lister or no lister") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +type alwaysFailingMembership struct{} + +func (alwaysFailingMembership) Buckets(context.Context) ([]int64, error) { + return nil, errors.New("projects listing unavailable") +} diff --git a/internal/connector/ndjson/shared_test.go b/internal/connector/ndjson/shared_test.go index 07773a6b3..880cf4cfc 100644 --- a/internal/connector/ndjson/shared_test.go +++ b/internal/connector/ndjson/shared_test.go @@ -62,3 +62,20 @@ func TestWritersForDifferentSinksAreIndependent(t *testing.T) { assert.NotSame(t, NewWriter(&a), NewWriter(&b)) assert.Same(t, NewWriter(&a), NewWriter(&a)) } + +// A sink whose type is comparable but whose fields are not panics when hashed. +// Only reference-like sinks are keyed, so such a sink never reaches the map. +type wrapperSink struct { + inner any + buf *bytes.Buffer +} + +func (w wrapperSink) Write(p []byte) (int, error) { return w.buf.Write(p) } + +func TestAnUnhashableSinkIsNotKeyed(t *testing.T) { + sink := wrapperSink{inner: []int{1, 2, 3}, buf: &bytes.Buffer{}} + require.NotPanics(t, func() { + require.NoError(t, NewWriter(sink).WriteLine(map[string]int{"line": 1})) + }) + assert.Equal(t, "{\"line\":1}\n", sink.buf.String()) +} diff --git a/internal/connector/repair.go b/internal/connector/repair.go index a4be68bfe..633495a67 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -40,6 +40,17 @@ type repairWalker struct { sleep func(ctx context.Context, d time.Duration) error } +// reconcileLoss walks a loss under its own recorded filter set. +func (w *repairWalker) reconcileLoss(ctx context.Context, loss Loss) error { + walker := *w + if len(loss.Filters.Types) > 0 || len(loss.Filters.Buckets) > 0 || len(loss.Filters.Creators) > 0 || + len(loss.Filters.Performers) > 0 || len(loss.Filters.ExcludePerformers) > 0 || + len(loss.Filters.ActorTypes) > 0 || len(loss.Filters.Reasons) > 0 { + walker.filters = loss.Filters + } + return walker.reconcile(ctx, loss) +} + func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { for { if !w.now().Before(loss.DeadlineAt) { diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go index 9ad2dbe1f..22722e5db 100644 --- a/internal/connector/repair_bounds_test.go +++ b/internal/connector/repair_bounds_test.go @@ -39,7 +39,7 @@ func onePassWalker(t *testing.T, polls eventfeed.PollSource) (*repairWalker, *Le t.Helper() ledger := newTestLedger(t) clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) walker, _ := newTestWalker(t, ledger, polls, clock) return walker, ledger, loss @@ -111,7 +111,7 @@ func TestACanceledRepairWalkLeavesTheLossOpenForTheNextStart(t *testing.T) { t.Run(name, func(t *testing.T) { ledger := newTestLedger(t) clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at.Add(-age), 10*time.Minute) + loss, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, clock.at.Add(-age), 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/connector/repair_test.go b/internal/connector/repair_test.go index 948101e27..b530eb4f4 100644 --- a/internal/connector/repair_test.go +++ b/internal/connector/repair_test.go @@ -47,7 +47,7 @@ func TestRepairWalkEntersOneBelowTheLowestMissingID(t *testing.T) { ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838505, 17099838502}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838505, 17099838502}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{pages: []eventfeed.PollPage{{ @@ -75,7 +75,7 @@ func TestRepairWalkFollowsNextThroughAnEmptyPage(t *testing.T) { ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{pages: []eventfeed.PollPage{ @@ -97,7 +97,7 @@ func TestRepairWalkRepeatsAfterAMissingNext(t *testing.T) { ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{pages: []eventfeed.PollPage{ @@ -123,7 +123,7 @@ func TestRepairCursorNeverReachesTheFeedCheckpoint(t *testing.T) { key := testKey() require.NoError(t, ledger.Save(ctx, key, "feed-position-before-the-overflow")) - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{pages: []eventfeed.PollPage{{ @@ -145,7 +145,7 @@ func TestRepairWalkSavesItsOwnCursorOnTheLoss(t *testing.T) { ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{pages: []eventfeed.PollPage{{Events: nil, Position: "repair-walk-position"}}} @@ -171,7 +171,7 @@ func TestIdsStillMissingWhenTheWindowClosesAreUnrecovered(t *testing.T) { ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509, 17099838510}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509, 17099838510}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) // The straggler arrives on the fourth repair poll; the other never does. @@ -202,7 +202,7 @@ func TestAnUnrecoveredIDIsStillIngestedNormallyLater(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute, eventfeed.Filters{}) require.NoError(t, err) _, err = ledger.CloseLoss(ctx, loss.ID, time.Now()) require.NoError(t, err) @@ -222,7 +222,7 @@ func TestRepairWalkBelowTheEpochEndsWithAGap(t *testing.T) { ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{100, 101}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{100, 101}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{errs: []error{&eventfeed.PollError{ @@ -250,7 +250,7 @@ func TestATransientRepairPollIsRetriedNotGivenUpOn(t *testing.T) { ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{ diff --git a/internal/connector/review_fixes_test.go b/internal/connector/review_fixes_test.go index b18bcecf4..a3b89bcf3 100644 --- a/internal/connector/review_fixes_test.go +++ b/internal/connector/review_fixes_test.go @@ -23,7 +23,7 @@ func TestALateArrivalClearsAnUnrecoveredID(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), time.Minute, eventfeed.Filters{}) require.NoError(t, err) _, err = ledger.CloseLoss(ctx, loss.ID, time.Now()) require.NoError(t, err) @@ -125,7 +125,7 @@ func TestATerminalFeedIsNotHeldOpenByTheMembershipWatcher(t *testing.T) { func TestATerminalFeedIsNotHeldOpenByARepairWalk(t *testing.T) { ledger := newTestLedger(t) - _, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, time.Now(), time.Hour) + _, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, time.Now(), time.Hour, eventfeed.Filters{}) require.NoError(t, err) intake, _, minter, _ := newFeedIntake(t, ledger, Options{RepairInterval: time.Hour}) @@ -265,7 +265,7 @@ func TestAnInboxShaped410InTheRepairWalkIsNotTheEpochsPath(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) var hits atomic.Int32 @@ -293,7 +293,7 @@ func TestTheRepairWalkDoesNotFollowAForeignURL(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{pages: []eventfeed.PollPage{ @@ -303,7 +303,7 @@ func TestTheRepairWalkDoesNotFollowAForeignURL(t *testing.T) { require.NoError(t, walker.reconcile(ctx, loss)) assert.Equal(t, 1, polls.calls, "the foreign next is not followed") - loss2, err := ledger.RecordLoss(ctx, []int64{17099838600}, clock.at, 10*time.Minute) + loss2, err := ledger.RecordLoss(ctx, []int64{17099838600}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls2 := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 17099838550, ResumeURL: "http://3.basecampapi.com/2914079/events.json?since=17099838550"}}} @@ -326,7 +326,7 @@ func TestAnEpoch410InTheRepairWalkOnlyCondemnsTheIDsBehindIt(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{ @@ -359,7 +359,7 @@ func TestARefusedRepairCursorReentersFromTheExplicitID(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) require.NoError(t, ledger.SaveRepairCursor(ctx, loss.ID, "cursor-under-old-filters")) loss.RepairCursor = "cursor-under-old-filters" @@ -383,7 +383,7 @@ func TestAnUnrecoverableRepairPollLeavesTheLossOpen(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollRedirectRefused}}} @@ -402,7 +402,7 @@ func TestARepeated410OnTheResumeEndsTheWalk(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) gone := &eventfeed.PollError{Kind: eventfeed.PollGone, EpochAfterID: 150, diff --git a/internal/connector/round4_test.go b/internal/connector/round4_test.go index 994dcc904..a96ec82f7 100644 --- a/internal/connector/round4_test.go +++ b/internal/connector/round4_test.go @@ -117,7 +117,7 @@ func TestARepairPassIsBoundedWhenEveryResumeURLDiffers(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &noncePolls{} diff --git a/internal/connector/round6_test.go b/internal/connector/round6_test.go index 16bc273df..849797d3e 100644 --- a/internal/connector/round6_test.go +++ b/internal/connector/round6_test.go @@ -63,7 +63,7 @@ func TestAPartlyUnrecoveredLossIsNotReportedAsFullyReconciled(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{100, 200}, clock.at, 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{ diff --git a/internal/connector/round8_test.go b/internal/connector/round8_test.go index 0167075ce..39dd62684 100644 --- a/internal/connector/round8_test.go +++ b/internal/connector/round8_test.go @@ -81,7 +81,7 @@ func TestALossPastItsWindowClosesEvenWhenItsWalkCannotFinish(t *testing.T) { ledger := newTestLedger(t) ctx := context.Background() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} - loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-24*time.Hour), 10*time.Minute) + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-24*time.Hour), 10*time.Minute, eventfeed.Filters{}) require.NoError(t, err) polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollFilterInvalid, Err: errors.New("x")}}} From c9f54c275c014814c4832c3754ba8afee04958c1 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 00:31:50 +0200 Subject: [PATCH 24/49] Key only reference-like sinks in the line-writer registry A restore during the last commit dropped this: a comparable sink type holding an uncomparable field panics when hashed, so only a pointer, channel or unsafe pointer is keyed, which covers every real sink. The test that catches it was already in that commit and was failing. --- internal/connector/ndjson/ndjson.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/connector/ndjson/ndjson.go b/internal/connector/ndjson/ndjson.go index c45b1f429..b2c24e628 100644 --- a/internal/connector/ndjson/ndjson.go +++ b/internal/connector/ndjson/ndjson.go @@ -32,13 +32,19 @@ type Writer struct { // Keying on the sink makes that impossible whoever calls this, without every // caller having to agree to pass one instance around. // -// A sink whose type is not comparable cannot be the same sink twice (it is -// always a copy), so it gets a Writer of its own. +// Only reference-like sinks are keyed — a pointer, a channel, an unsafe +// pointer — which covers every real one, os.Stdout included. A value-typed +// sink is a copy rather than the same sink, and hashing one can panic when it +// holds an uncomparable field, so it gets a Writer of its own. The map holds +// one entry per distinct sink for the life of the process; a connector has +// one stdout. func NewWriter(w io.Writer) *Writer { if w == nil { return &Writer{} } - if !reflect.TypeOf(w).Comparable() { + switch reflect.TypeOf(w).Kind() { + case reflect.Pointer, reflect.Chan, reflect.UnsafePointer: + default: return &Writer{w: w} } sinksMu.Lock() From 4228fa2e624c102cef0fb5fb2aa2add2b8d0b165 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 00:37:19 +0200 Subject: [PATCH 25/49] Tell a loss with no filters from a loss written before filters were stored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Substituting the loss's filters only when one was set made the two look alike, so a loss recorded on the whole-account feed — the common case — was walked under whatever filters a restart had added, which is the very thing storing them prevents. The ledger already distinguishes them: a row written before migration 2 holds an empty string, a recorded one holds JSON. That is now carried as a flag, which also drops the hand-written field-by-field emptiness test. --- internal/connector/ledger_recovery.go | 7 ++++ internal/connector/loss_filters_test.go | 54 +++++++++++++++++++++++++ internal/connector/repair.go | 7 ++-- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go index e15c7001b..955a7ad46 100644 --- a/internal/connector/ledger_recovery.go +++ b/internal/connector/ledger_recovery.go @@ -46,6 +46,11 @@ type Loss struct { // under one set and walked under another is repaired against a lane that // never carried its events. Filters eventfeed.Filters + // HasFilters distinguishes a loss recorded with no filters — the + // whole-account feed, the common case — from one written before losses + // carried them at all. The first is walked with no filters; the second + // with the connector's own, which is the best that row can say. + HasFilters bool } // GapClass distinguishes the feed's two 410s. They mean different things and @@ -143,6 +148,7 @@ func (l *Ledger) RecordLoss(ctx context.Context, droppedIDs []int64, now time.Ti loss := Loss{ Filters: filters, + HasFilters: true, DetectedAt: now.UTC(), DroppedCount: len(states), // One below the lowest missing id, so the walk's first page can serve @@ -206,6 +212,7 @@ FROM losses WHERE resolved_at IS NULL ORDER BY id`) return nil, fmt.Errorf("connector: scan loss: %w", err) } if encodedFilters != "" { + loss.HasFilters = true if err := json.Unmarshal([]byte(encodedFilters), &loss.Filters); err != nil { return nil, fmt.Errorf("connector: decode loss filters: %w", err) } diff --git a/internal/connector/loss_filters_test.go b/internal/connector/loss_filters_test.go index ca96b1f4f..c07eaa455 100644 --- a/internal/connector/loss_filters_test.go +++ b/internal/connector/loss_filters_test.go @@ -46,3 +46,57 @@ func TestALossIsRepairedUnderTheFilterSetItWasRecordedWith(t *testing.T) { require.NoError(t, err) assert.Equal(t, []int64{17099838509}, recoveredIDs) } + +// The whole-account feed records a loss with no filters at all, which is not +// the same as a loss from before filters were stored. Restarted with filters +// added, the repair must still walk the whole account. +func TestALossRecordedWithNoFiltersIsRepairedWithNoFilters(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.filters = eventfeed.Filters{Types: []string{"comment.created"}} // added since + require.NoError(t, walker.reconcileLoss(ctx, open[0])) + + require.NotEmpty(t, polls.filters) + assert.Empty(t, polls.filters[0].Types, + "the loss was recorded on the whole account; a narrower lane never carried its events") +} + +// A loss written before losses carried their filters has none recorded, and is +// walked under the connector's own. +func TestALegacyLossWalksUnderTheConnectorsFilters(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordLoss(ctx, []int64{17099838509}, time.Now(), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + // What migration 2 leaves on a row written before it. + _, err = ledger.db.ExecContext(ctx, `UPDATE losses SET filters = ''`) + require.NoError(t, err) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + assert.False(t, open[0].HasFilters) + + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + polls := &scriptedPolls{pages: []eventfeed.PollPage{ + {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.filters = eventfeed.Filters{Types: []string{"comment.created"}} + require.NoError(t, walker.reconcileLoss(ctx, open[0])) + + require.NotEmpty(t, polls.filters) + assert.Equal(t, []string{"comment.created"}, polls.filters[0].Types) +} diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 633495a67..a0ec725b9 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -43,9 +43,10 @@ type repairWalker struct { // reconcileLoss walks a loss under its own recorded filter set. func (w *repairWalker) reconcileLoss(ctx context.Context, loss Loss) error { walker := *w - if len(loss.Filters.Types) > 0 || len(loss.Filters.Buckets) > 0 || len(loss.Filters.Creators) > 0 || - len(loss.Filters.Performers) > 0 || len(loss.Filters.ExcludePerformers) > 0 || - len(loss.Filters.ActorTypes) > 0 || len(loss.Filters.Reasons) > 0 { + if loss.HasFilters { + // Recorded with the loss, empty set included: the whole-account feed + // is a filter set, and walking it narrowly would miss the very events + // the loss is about. walker.filters = loss.Filters } return walker.reconcile(ctx, loss) From ce1ffa8c1ef0255bdf69b1b92d4d9af070aa8c67 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 01:07:01 +0200 Subject: [PATCH 26/49] Own the filter set, bound repair fan-out, refuse an in-memory ledger, sanitize peer log text Five findings, each with a test that failed before its fix. The filter set is the checkpoint's identity and what every subscription, recorded loss and repair walk runs under. Its slices were the caller's, so a later mutation would change all of those while the key stayed frozen on what it was built from; they are copied at construction, as the SDK copies its own. Repairs ran a goroutine and a poll source per loss. An overloaded feed raises an overflow per dropped event, so that answers a struggling API with a storm of walks. Repairs now run through two workers on a bounded queue; a loss that finds it full stays open for the next start. The ledger refuses SQLite's in-memory database. It would accept every write and lose it on close, which is the one thing the ledger exists not to do. A re-entry that saved a checkpoint has made progress even when the page carried no event, so a later refusal is an ordinary refusal rather than the re-entry being refused; without this the run aborted. A peer's disconnect reason is text the other end chose, and a log is read in a terminal like the pointer lines are. --- internal/connector/intake.go | 111 +++++++++++++++----- internal/connector/intake_test.go | 10 +- internal/connector/ledger.go | 12 +++ internal/connector/round9_test.go | 162 ++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 30 deletions(-) create mode 100644 internal/connector/round9_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 64791b563..784f53998 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "net/url" + "slices" "strconv" "sync" "time" @@ -157,7 +158,9 @@ type Intake struct { // the feed somewhere unsafe. abortErr error - repairs sync.WaitGroup + repairs sync.WaitGroup + repairOnce sync.Once + repairQueue chan Loss // lifetime is Run's context. Repair walks are bound to it rather than to // a connection, so a reconnect does not abandon a walk and a shutdown does // not strand Run waiting on one — an unfinished walk simply resumes on the @@ -195,6 +198,20 @@ func New(opts Options) (*Intake, error) { if err := opts.Filters.Validate(); err != nil { return nil, fmt.Errorf("connector: intake filters: %w", err) } + // The filter set is the checkpoint's identity, and it is also what every + // subscription, recorded loss and repair walk runs under. A caller that + // kept its slices could change all of those while the key stays frozen on + // what it was built from, so they are copied here as the SDK's WithFilters + // copies its own. + opts.Filters = eventfeed.Filters{ + Types: slices.Clone(opts.Filters.Types), + Buckets: slices.Clone(opts.Filters.Buckets), + Creators: slices.Clone(opts.Filters.Creators), + Performers: slices.Clone(opts.Filters.Performers), + ExcludePerformers: slices.Clone(opts.Filters.ExcludePerformers), + ActorTypes: slices.Clone(opts.Filters.ActorTypes), + Reasons: slices.Clone(opts.Filters.Reasons), + } if opts.Clock == nil { opts.Clock = time.Now @@ -574,7 +591,9 @@ func (in *Intake) observer(ctx context.Context) eventfeed.Observer { Connected: func() { in.log.Info("feed socket connected") }, Confirmed: func() { in.log.Info("feed subscription confirmed") }, Disconnected: func(reason string, err error) { - in.log.Warn("feed socket disconnected", "reason", reason, "error", err) + // The reason is the peer's own text, and a log is read in a + // terminal like everything else the connector writes. + in.log.Warn("feed socket disconnected", "reason", richtext.SanitizeSingleLine(reason), "error", err) }, CatchUpStarted: func(eventfeed.Cursor) { in.log.Info("feed catch-up walk started") }, PageDelivered: func(_ int, position string) { @@ -595,6 +614,10 @@ func (in *Intake) observer(ctx context.Context) eventfeed.Observer { Checkpoint: func(string) { in.mu.Lock() in.checkpointed = true + // A saved position is progress, whether or not its page carried + // an event: this connection is no longer a re-entry waiting to + // prove itself, and a later refusal is an ordinary one. + in.enteredByReentry = false in.mu.Unlock() }, CheckpointSaveFailed: func(err error) { @@ -718,35 +741,66 @@ func (in *Intake) repairContext() context.Context { return context.Background() } +// startRepair hands a loss to the repair workers. +// +// Repairs are bounded: an overloaded feed can raise an overflow per dropped +// event, and a goroutine and a poll source per loss would answer an API that +// is already struggling with a storm of walks. A loss that finds the queue +// full is left open on disk, which the next start resumes. func (in *Intake) startRepair(ctx context.Context, loss Loss) { if loss.ResolvedAt != nil { return } - in.repairs.Add(1) - go func() { - defer in.repairs.Done() - // Off the delivery path: nothing about live intake waits for this. - walker := &repairWalker{ - ledger: in.ledger, - polls: in.opts.PollsFor(), - origin: in.key.Origin, - filters: in.opts.Filters, - ingest: in.ingest, - now: in.now, - interval: in.opts.RepairInterval, - log: in.log, - sleep: in.repairSleep, + in.repairOnce.Do(func() { + in.repairQueue = make(chan Loss, repairQueueDepth) + for range maxConcurrentRepairs { + go in.repairWorker(ctx) } - switch err := walker.reconcileLoss(ctx, loss); { - case err == nil: - case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): - // A shutdown mid-walk is a delay: the loss is still open on disk - // and the next start picks it up where this one left off. - in.log.Info("reconciliation paused by shutdown; it resumes on the next start", "loss_id", loss.ID) - default: - in.log.Error("reconciliation of a loss ended early", "loss_id", loss.ID, "error", err) + }) + select { + case in.repairQueue <- loss: + default: + in.log.Warn("the repair queue is full; this loss stays open for the next start", "loss_id", loss.ID) + } +} + +// repairWorker walks one loss at a time until ctx ends. The wait group counts +// walks in flight rather than idle workers, so a shutdown waits for the walk +// it interrupts and not for a worker that is waiting for work. +func (in *Intake) repairWorker(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case loss := <-in.repairQueue: + in.runRepair(ctx, loss) } - }() + } +} + +func (in *Intake) runRepair(ctx context.Context, loss Loss) { + in.repairs.Add(1) + defer in.repairs.Done() + walker := &repairWalker{ + ledger: in.ledger, + polls: in.opts.PollsFor(), + origin: in.key.Origin, + filters: in.opts.Filters, + ingest: in.ingest, + now: in.now, + interval: in.opts.RepairInterval, + log: in.log, + sleep: in.repairSleep, + } + switch err := walker.reconcileLoss(ctx, loss); { + case err == nil: + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + // A shutdown mid-walk is a delay: the loss is still open on disk + // and the next start picks it up where this one left off. + in.log.Info("reconciliation paused by shutdown; it resumes on the next start", "loss_id", loss.ID) + default: + in.log.Error("reconciliation of a loss ended early", "loss_id", loss.ID, "error", err) + } } // takeSnapshot records the buckets the agent can see at the moment this @@ -929,6 +983,13 @@ func (in *Intake) watchMembership(ctx context.Context) func() { return func() { <-done } } +// maxConcurrentRepairs bounds repair walks in flight, and repairQueueDepth +// the losses waiting for one. +const ( + maxConcurrentRepairs = 2 + repairQueueDepth = 1024 +) + // defaultMembershipRetry is the first wait before re-reading a membership // listing that failed; it doubles up to the membership interval. const defaultMembershipRetry = 5 * time.Second diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go index 700fae141..680e3f20e 100644 --- a/internal/connector/intake_test.go +++ b/internal/connector/intake_test.go @@ -290,11 +290,11 @@ func TestReconciliationResumesOnStart(t *testing.T) { require.NoError(t, err) require.NoError(t, intake.resumeReconciliation(ctx)) - intake.repairs.Wait() - - open, err := ledger.OpenLosses(ctx) - require.NoError(t, err) - assert.Empty(t, open, "a crash between the overflow and its repair is a delay, not a lost record") + require.Eventually(t, func() bool { + open, err := ledger.OpenLosses(ctx) + return err == nil && len(open) == 0 + }, 5*time.Second, 5*time.Millisecond, + "a crash between the overflow and its repair is a delay, not a lost record") _, ok, err := ledger.Get(ctx, 17099838501) require.NoError(t, err) diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index a340ce504..fa38bfa09 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -15,6 +15,13 @@ import ( sqlite3 "modernc.org/sqlite/lib" ) +// isInMemory reports a path SQLite would read as its in-memory database +// rather than a file. +func isInMemory(path string) bool { + trimmed := strings.TrimPrefix(path, "file:") + return trimmed == ":memory:" || strings.HasPrefix(trimmed, ":memory:?") || trimmed == "" +} + // RecordState is where an event sits in the ledger's lifecycle. // // Intake only ever writes StateSeen. The rest of the vocabulary is declared @@ -72,6 +79,11 @@ func OpenLedger(path string) (*Ledger, error) { if path == "" { return nil, errors.New("connector: ledger path is required") } + if isInMemory(path) { + // SQLite's in-memory URI accepts every write and loses it on close. + // The ledger's whole promise is that a crash is a delay. + return nil, fmt.Errorf("connector: ledger path %q names SQLite's in-memory database, which is not durable", path) + } if strings.ContainsAny(path, "?#%") { // The driver reads the path as a URI; these would be taken as its // query, fragment or an escape, and open some other file. diff --git a/internal/connector/round9_test.go b/internal/connector/round9_test.go new file mode 100644 index 000000000..7edf2f5e5 --- /dev/null +++ b/internal/connector/round9_test.go @@ -0,0 +1,162 @@ +package connector + +import ( + "bytes" + "context" + "log/slog" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// The filter set is the checkpoint's identity. A caller that keeps its slice +// and mutates it would change the filters intake subscribes, records and +// walks with, while the checkpoint key stays frozen on what it was built from. +func TestTheFilterSetIsCopiedAtConstruction(t *testing.T) { + ledger := newTestLedger(t) + queue, err := NewQueue(1, 2) + require.NoError(t, err) + types := []string{"comment.created"} + buckets := []int64{48699913} + intake, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Filters: eventfeed.Filters{Types: types, Buckets: buckets}, + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + }) + require.NoError(t, err) + + types[0] = "card.created" + buckets[0] = 999 + + assert.Equal(t, []string{"comment.created"}, intake.opts.Filters.Types) + assert.Equal(t, []int64{48699913}, intake.opts.Filters.Buckets) + assert.Equal(t, eventfeed.Filters{Types: []string{"comment.created"}, Buckets: []int64{48699913}}.FilterKey(), + intake.CheckpointKey().FilterKey) +} + +// A peer's disconnect reason is text the other end chose. It reaches an +// operator's terminal through the log. +func TestALoggedDisconnectReasonCarriesNoTerminalControls(t *testing.T) { + var logs bytes.Buffer + intake, _, _ := newTestIntake(t, nil, nil) + intake.log = slog.New(slog.NewJSONHandler(&logs, nil)) + + observer := intake.observer(context.Background()) + require.NotNil(t, observer.Disconnected) + observer.Disconnected("stale"+csi+"31m"+esc+"[2J", nil) + + assert.NotContains(t, logs.String(), csi) + assert.NotContains(t, logs.String(), esc) + assert.Contains(t, logs.String(), "stale") +} + +// A re-entry that saved a checkpoint has made progress, even if the page it +// saved carried no event. A later refusal of that saved position is an +// ordinary refusal, not the re-entry being refused. +func TestAReentryThatCheckpointedAnEmptyPageIsNotStillPending(t *testing.T) { + ledger := newTestLedger(t) + // A short repair cadence so the position saved from the empty page is + // re-polled, and refused, while this connection is streaming. + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{RepairInterval: 50 * time.Millisecond}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, ledger.Save(ctx, intake.CheckpointKey(), "refused")) + require.NoError(t, ledger.NotePollServed(ctx, intake.CheckpointKey(), 1000)) + + for range 6 { + minter.ScriptTicket(ticket()) + } + // The stored position is refused; the re-entry's first page is empty but + // checkpoints; that saved position is then refused in turn. + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Position: "saved-from-an-empty-page"}) + polls.ScriptError(&eventfeed.PollError{Kind: eventfeed.PollPositionInvalid}) + polls.ScriptPage(eventfeed.PollPage{Events: []eventfeed.Event{testEvent(17099838600)}, Position: "p3"}) + for range 4 { + polls.ScriptPage(eventfeed.PollPage{Position: "p4"}) + } + + done := runInBackground(ctx, t, intake) + answered := 0 + require.Eventually(t, func() bool { + if conns := transport.Conns(); len(conns) > answered { + answerSubscription(conns[len(conns)-1]) + answered = len(conns) + } + _, ok, err := ledger.Get(ctx, 17099838600) + return err == nil && ok + }, 5*time.Second, 10*time.Millisecond, "the run aborted instead of re-entering after a checkpointed empty page") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} + +// countingPolls reports the highest number of polls in flight at once. +type countingPolls struct { + inFlight atomic.Int32 + peak atomic.Int32 + calls atomic.Int32 +} + +func (c *countingPolls) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + n := c.inFlight.Add(1) + for { + peak := c.peak.Load() + if n <= peak || c.peak.CompareAndSwap(peak, n) { + break + } + } + defer c.inFlight.Add(-1) + c.calls.Add(1) + time.Sleep(5 * time.Millisecond) + return eventfeed.PollPage{Position: "p"}, nil +} + +// C3: many open losses are repaired within a bound. One goroutine and one poll +// source per loss would turn an overloaded feed into an API storm. +func TestRepairsRunWithinABound(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for i := range 25 { + // A window already closed, so each loss is one final pass and done. + _, err := ledger.RecordLoss(ctx, []int64{int64(1000 + i)}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + } + require.NoError(t, intake.resumeReconciliation(ctx)) + + require.Eventually(t, func() bool { return polls.calls.Load() >= 10 }, 5*time.Second, 5*time.Millisecond) + assert.LessOrEqual(t, int(polls.peak.Load()), maxConcurrentRepairs, "repairs are bounded") + + cancel() + intake.repairs.Wait() +} + +// The ledger is a file. SQLite's in-memory URI would accept every write and +// lose it on close, which is the one thing the ledger exists not to do. +func TestAnInMemoryLedgerIsRefused(t *testing.T) { + for _, path := range []string{":memory:", "file::memory:"} { + _, err := OpenLedger(path) + require.Error(t, err, path) + assert.Contains(t, strings.ToLower(err.Error()), "memory") + } + // A real path containing the word is still a file. + ledger, err := OpenLedger(filepath.Join(t.TempDir(), "state", "memory.db")) + require.NoError(t, err) + require.NoError(t, ledger.Close()) +} From 4bcd8ef66d9ad7c97063e3d43cd133afab83765e Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 01:22:42 +0200 Subject: [PATCH 27/49] Fix the repair pool's shutdown, and make a learned project revocable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repair workers added themselves to the wait group as they picked work up, which Go refuses once a shutdown is already waiting: "WaitGroup is reused before previous Wait has returned". It panicked in the package's own test, 2 runs in 6 under the race detector. The pool is now a fixed size started once, every Add on the goroutine that later waits and before anything can wait, each worker's Done when it exits. A loss offered with no pool running stays open for the next start. A bucket learned from an arriving event was permanent, so a project the lister stopped naming stayed trusted for the life of the process. The lister is the authority: its listing is held separately, and a listing that omits a learned bucket revokes it. Relearning one is rate-limited to once per membership interval, so revocation cannot turn back into a reconnect per event — which is the storm an earlier round fixed. --- internal/connector/intake.go | 194 ++++++++++++++------------ internal/connector/invariants_test.go | 25 ++-- internal/connector/membership_test.go | 45 +++++- 3 files changed, 166 insertions(+), 98 deletions(-) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 784f53998..94580c17e 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -124,13 +124,20 @@ type Intake struct { mu sync.Mutex // served is the current connection's wrapped poll source. - served *servedPolls - snapshot map[int64]bool + served *servedPolls + // listed is what the project lister last said, and it is authoritative: a + // bucket it stops naming is revoked. + listed map[int64]bool // membershipRetry is the first backoff after a failed membership read. membershipRetry time.Duration - // learned holds buckets events proved visible that the lister did not - // name. - learned map[int64]bool + // learned holds buckets an arriving event proved visible that the lister + // has not named. It is provisional — the next listing that omits one takes + // it back — so a bucket learned at runtime never becomes permanent truth. + learned map[int64]bool + // relearned is when each bucket was last learned. A revoked bucket is + // learned again at most once per membership interval, so revocation + // cannot become a reconnect per event. + relearned map[int64]time.Time reconnect chan struct{} // cancelRun ends the current connection; nil between connections. cancelRun context.CancelFunc @@ -692,7 +699,7 @@ func (in *Intake) handleSignal(signal eventfeed.Signal) eventfeed.Disposition { } in.log.Warn("live buffer overflowed; reconciling", "dropped", s.DroppedCount, "loss_id", loss.ID, "repair_since", loss.RepairSince) - in.startRepair(in.repairContext(), loss) + in.startRepair(loss) return eventfeed.Accept } return eventfeed.Terminate @@ -721,66 +728,81 @@ func entryClassOf(resumeURL string) EntryClass { // crash between the overflow and its repair is a delay, not a loss of the // record. func (in *Intake) resumeReconciliation(ctx context.Context) error { + in.startRepairWorkers(ctx) losses, err := in.ledger.OpenLosses(ctx) if err != nil { return err } for _, loss := range losses { in.log.Info("resuming reconciliation of an open loss", "loss_id", loss.ID, "repair_since", loss.RepairSince) - in.startRepair(ctx, loss) + in.startRepair(loss) } return nil } -// repairContext is the lifetime a repair walk runs under. handleSignal is -// invoked by the feed with no context of its own, so the walk takes Run's. -func (in *Intake) repairContext() context.Context { - if in.lifetime != nil { - return in.lifetime - } - return context.Background() -} - // startRepair hands a loss to the repair workers. // // Repairs are bounded: an overloaded feed can raise an overflow per dropped // event, and a goroutine and a poll source per loss would answer an API that // is already struggling with a storm of walks. A loss that finds the queue // full is left open on disk, which the next start resumes. -func (in *Intake) startRepair(ctx context.Context, loss Loss) { +func (in *Intake) startRepair(loss Loss) { if loss.ResolvedAt != nil { return } - in.repairOnce.Do(func() { - in.repairQueue = make(chan Loss, repairQueueDepth) - for range maxConcurrentRepairs { - go in.repairWorker(ctx) - } - }) + in.mu.Lock() + queue := in.repairQueue + in.mu.Unlock() + if queue == nil { + // No pool: nobody would walk this loss. It stays open on disk, and + // the next start resumes it. + in.log.Warn("no repair workers are running; this loss stays open for the next start", "loss_id", loss.ID) + return + } select { - case in.repairQueue <- loss: + case queue <- loss: default: in.log.Warn("the repair queue is full; this loss stays open for the next start", "loss_id", loss.ID) } } -// repairWorker walks one loss at a time until ctx ends. The wait group counts -// walks in flight rather than idle workers, so a shutdown waits for the walk -// it interrupts and not for a worker that is waiting for work. -func (in *Intake) repairWorker(ctx context.Context) { +// startRepairWorkers starts the fixed repair pool, once. +// +// Every Add happens here, on the goroutine that later waits, before anything +// can wait: a worker that added itself as it picked work up would be adding to +// a group a shutdown may already be waiting on, which Go refuses outright. +func (in *Intake) startRepairWorkers(ctx context.Context) { + in.repairOnce.Do(func() { + queue := make(chan Loss, repairQueueDepth) + in.mu.Lock() + in.repairQueue = queue + in.mu.Unlock() + for range maxConcurrentRepairs { + in.repairs.Add(1) + go in.repairWorker(ctx, queue) + } + }) +} + +// repairWorker walks one loss at a time until ctx ends. +// +// A walk holds its worker for as long as it takes, the waits between its +// passes included, so a loss can sit in the queue past its own window and get +// the single catch-up pass a closed window allows. Recovery still happens; it +// happens later. +func (in *Intake) repairWorker(ctx context.Context, queue chan Loss) { + defer in.repairs.Done() for { select { case <-ctx.Done(): return - case loss := <-in.repairQueue: + case loss := <-queue: in.runRepair(ctx, loss) } } } func (in *Intake) runRepair(ctx context.Context, loss Loss) { - in.repairs.Add(1) - defer in.repairs.Done() walker := &repairWalker{ ledger: in.ledger, polls: in.opts.PollsFor(), @@ -803,7 +825,7 @@ func (in *Intake) runRepair(ctx context.Context, loss Loss) { } } -// takeSnapshot records the buckets the agent can see at the moment this +// takeSnapshot records the buckets the lister names at the moment this // connection subscribes — the same set the cable snapshots. func (in *Intake) takeSnapshot(ctx context.Context) { if in.opts.Membership == nil { @@ -811,46 +833,74 @@ func (in *Intake) takeSnapshot(ctx context.Context) { } buckets, err := in.opts.Membership.Buckets(ctx) if err != nil { - in.log.Warn("could not read the agent's projects; keeping the previous snapshot", "error", err) + in.log.Warn("could not read the agent's projects; keeping the previous listing", "error", err) return } + in.adoptListing(buckets) +} + +// adoptListing replaces the authoritative set and revokes every learned bucket +// the listing does not name. It reports whether the listing changed; the first +// listing is a baseline, not a change. +func (in *Intake) adoptListing(buckets []int64) bool { in.mu.Lock() defer in.mu.Unlock() - in.snapshot = make(map[int64]bool, len(buckets)+len(in.learned)) + listed := make(map[int64]bool, len(buckets)) for _, id := range buckets { - in.snapshot[id] = true + listed[id] = true } + // A learned bucket is provisional, and the lister is the trust boundary: + // one it no longer names is revoked, however recently it was learned, and + // the next event from it is unknown again. for id := range in.learned { - in.snapshot[id] = true + if !listed[id] { + delete(in.learned, id) + in.log.Info("a project the lister no longer names is no longer held", "bucket_id", id) + } } + changed := in.listed != nil && !sameBuckets(in.listed, listed) + in.listed = listed + return changed +} + +func sameBuckets(a, b map[int64]bool) bool { + if len(a) != len(b) { + return false + } + for id := range a { + if !b[id] { + return false + } + } + return true } // noteBucket asks for a reconnect when an event arrives from a bucket the live -// snapshot did not hold. The poll lane authorizes at read time, so it covers -// the new project immediately; the live lane cannot until it re-subscribes. +// subscription may not hold. The poll lane authorizes at read time, so it +// covers the project immediately; the live lane cannot until it re-subscribes. func (in *Intake) noteBucket(bucketID int64) { in.mu.Lock() - // With no snapshot — the read at subscribe failed — the live - // subscription's buckets are unknown, not "everything". An event from a - // bucket not yet learned asks for one reconnect. - known := (in.snapshot == nil && in.opts.Membership == nil) || in.snapshot[bucketID] || in.learned[bucketID] + known := in.opts.Membership == nil || in.listed[bucketID] || in.learned[bucketID] + if !known { + // A bucket revoked by a listing is unknown again, but relearning it + // is rate-limited: without that, a project the lister never names — + // archived, or past its page — would cost a reconnect per event. + if since, ok := in.relearned[bucketID]; ok && in.now().Sub(since) < in.opts.MembershipInterval { + known = true + } else { + if in.learned == nil { + in.learned = make(map[int64]bool) + in.relearned = make(map[int64]time.Time) + } + in.learned[bucketID] = true + in.relearned[bucketID] = in.now() + } + } in.mu.Unlock() if known { return } in.log.Info("an event arrived from a project the live subscription does not hold", "bucket_id", bucketID) - // Learned for the life of the process, and merged into every later - // snapshot. A project the membership list never names — archived, or past - // the lister's page — would otherwise cost a reconnect per event, forever. - in.mu.Lock() - if in.learned == nil { - in.learned = make(map[int64]bool) - } - in.learned[bucketID] = true - if in.snapshot != nil { - in.snapshot[bucketID] = true - } - in.mu.Unlock() in.requestReconnect() } @@ -997,41 +1047,13 @@ const defaultMembershipRetry = 5 * time.Second func (in *Intake) hasSnapshot() bool { in.mu.Lock() defer in.mu.Unlock() - return in.snapshot != nil + return in.listed != nil } -// membershipChanged compares a fresh read with the snapshot. With no snapshot -// — the read at subscribe failed — the fresh read becomes the baseline; -// otherwise a failed first read would disable change detection for good. -// -// Learned buckets (proved visible by an event, never named by the list) are -// ignored on the way out, so the list omitting them is not a change. Once the -// list names one, it is listed like any other, so its later revocation is. +// membershipChanged adopts a fresh listing and reports whether the +// authoritative set changed. func (in *Intake) membershipChanged(buckets []int64) bool { - in.mu.Lock() - defer in.mu.Unlock() - if in.snapshot == nil { - in.snapshot = make(map[int64]bool, len(buckets)) - for _, id := range buckets { - in.snapshot[id] = true - } - return false - } - for _, id := range buckets { - if !in.snapshot[id] { - return true - } - } - for _, id := range buckets { - delete(in.learned, id) - } - listed := 0 - for id := range in.snapshot { - if !in.learned[id] { - listed++ - } - } - return listed != len(buckets) + return in.adoptListing(buckets) } // requestReconnect marks a reconnect due and ends the current connection now, diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go index bbc6648b4..e1ac6575e 100644 --- a/internal/connector/invariants_test.go +++ b/internal/connector/invariants_test.go @@ -208,21 +208,24 @@ func TestInvariantE2AStaleReconnectDoesNotSwallowATerminalError(t *testing.T) { "a latch carried into the connection turns its terminal error into a reconnect") } -// E3: membership detection catches every change to the listed set, ignores -// learned buckets, and notices a learned bucket's revocation once the list -// has named it. +// E3: the listing the lister gives is what decides a change. A bucket learned +// from an arriving event is provisional and never counts as one, and it is +// revoked when a listing omits it. func TestInvariantE3MembershipComparison(t *testing.T) { intake, _, _ := newTestIntake(t, nil, nil) - intake.snapshot = map[int64]bool{1: true, 2: true, 9: true} - intake.learned = map[int64]bool{9: true} + intake.opts.Membership = &flakyMembership{buckets: []int64{1, 2}} - assert.False(t, intake.membershipChanged([]int64{1, 2}), "a learned bucket the list omits is not a change") - assert.True(t, intake.membershipChanged([]int64{1}), "a listed bucket dropping off is a change") + require.False(t, intake.adoptListing([]int64{1, 2}), "the first listing is a baseline") + intake.noteBucket(9) + assert.False(t, intake.adoptListing([]int64{1, 2}), "a learned bucket the listing omits is not a change") - intake.snapshot = map[int64]bool{1: true, 2: true, 9: true} - intake.learned = map[int64]bool{9: true} - assert.False(t, intake.membershipChanged([]int64{1, 2, 9}), "the list catching up with a learned bucket is not a change") - assert.True(t, intake.membershipChanged([]int64{1, 2}), "and its later revocation is") + intake.mu.Lock() + held := intake.learned[9] + intake.mu.Unlock() + assert.False(t, held, "and that listing revoked it") + + assert.True(t, intake.adoptListing([]int64{1}), "a listed bucket dropping off is a change") + assert.True(t, intake.adoptListing([]int64{1, 2}), "and its return is a change") } // H2 availability: two processes opening one fresh ledger both get it. diff --git a/internal/connector/membership_test.go b/internal/connector/membership_test.go index 57e2cced9..b5294eb29 100644 --- a/internal/connector/membership_test.go +++ b/internal/connector/membership_test.go @@ -92,7 +92,7 @@ func TestAFailedFirstMembershipReadIsRetriedSoon(t *testing.T) { require.Eventually(t, func() bool { intake.mu.Lock() defer intake.mu.Unlock() - return intake.snapshot != nil && intake.snapshot[48699913] + return intake.listed != nil && intake.listed[48699913] }, 2*time.Second, 10*time.Millisecond, "the first successful read becomes the baseline") cancel() @@ -148,3 +148,46 @@ type alwaysFailingMembership struct{} func (alwaysFailingMembership) Buckets(context.Context) ([]int64, error) { return nil, errors.New("projects listing unavailable") } + +// A bucket learned from an arriving event is provisional. The lister is the +// trust boundary: once it lists projects and does not name that one, the +// connector stops holding it, and the next event from it is unknown again. +func TestALearnedProjectIsRevokedWhenTheListerStopsNamingIt(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.opts.Membership = &flakyMembership{buckets: []int64{1}} + + require.False(t, intake.adoptListing([]int64{1}), "the first listing is a baseline") + intake.noteBucket(777) + intake.mu.Lock() + learned := intake.learned[777] + intake.mu.Unlock() + require.True(t, learned, "an event proved the project visible") + + // A later listing that does not name it takes it back. + intake.adoptListing([]int64{1}) + intake.mu.Lock() + stillHeld := intake.learned[777] || intake.listed[777] + intake.mu.Unlock() + assert.False(t, stillHeld, "the lister no longer names it, so the connector no longer holds it") + + // And its next event is unknown again, not silently trusted. Relearning + // is rate-limited per bucket, so the clock has to move past the + // membership interval for the next event to count as new. + intake.reconnectRequested() + require.Empty(t, intake.reconnect) + intake.opts.MembershipInterval = 0 + intake.noteBucket(777) + assert.Len(t, intake.reconnect, 1, "a revoked project's next event is unknown") +} + +// The listing the lister does name is what changes trigger a reconnect. +func TestOnlyTheListedSetDecidesAMembershipChange(t *testing.T) { + intake, _, _ := newTestIntake(t, nil, nil) + intake.opts.Membership = &flakyMembership{buckets: []int64{1, 2}} + + assert.False(t, intake.adoptListing([]int64{1, 2})) + intake.noteBucket(9) // learned, never listed + assert.False(t, intake.adoptListing([]int64{1, 2}), "a learned bucket the listing omits is not a change") + assert.True(t, intake.adoptListing([]int64{1}), "a listed bucket dropping off is a change") + assert.True(t, intake.adoptListing([]int64{1, 2}), "and its return is a change") +} From b2aad2f7c3a9dcc212700145945e936d16a585f4 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 01:27:45 +0200 Subject: [PATCH 28/49] Give each Run its own repair pool The pool was started once per intake and its queue outlived the run that started it, so a second Run enqueued losses to a pool whose workers had stopped with the first run's context: nothing walked them, and nothing said so. The pool is now forgotten once its workers have stopped, which Run defers first so it happens last. --- internal/connector/intake.go | 33 +++++++++++++++++++++---------- internal/connector/round9_test.go | 30 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 94580c17e..ec78b2f46 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -166,7 +166,6 @@ type Intake struct { abortErr error repairs sync.WaitGroup - repairOnce sync.Once repairQueue chan Loss // lifetime is Run's context. Repair walks are bound to it rather than to // a connection, so a reconnect does not abandon a walk and a shutdown does @@ -276,6 +275,9 @@ func (in *Intake) Run(ctx context.Context) error { // out its sixty-second cadence, and an unfinished repair resumes on the // next start from the loss record. repairCtx, stopRepairs := context.WithCancel(ctx) + // Deferred first, so it runs last: the pool is forgotten only once its + // workers have stopped, and the next Run starts its own. + defer in.releaseRepairWorkers() defer in.repairs.Wait() defer stopRepairs() in.lifetime = repairCtx @@ -772,16 +774,27 @@ func (in *Intake) startRepair(loss Loss) { // can wait: a worker that added itself as it picked work up would be adding to // a group a shutdown may already be waiting on, which Go refuses outright. func (in *Intake) startRepairWorkers(ctx context.Context) { - in.repairOnce.Do(func() { - queue := make(chan Loss, repairQueueDepth) - in.mu.Lock() - in.repairQueue = queue + in.mu.Lock() + if in.repairQueue != nil { in.mu.Unlock() - for range maxConcurrentRepairs { - in.repairs.Add(1) - go in.repairWorker(ctx, queue) - } - }) + return + } + queue := make(chan Loss, repairQueueDepth) + in.repairQueue = queue + in.mu.Unlock() + for range maxConcurrentRepairs { + in.repairs.Add(1) + go in.repairWorker(ctx, queue) + } +} + +// releaseRepairWorkers forgets a pool whose workers have stopped, so the next +// Run starts its own. A queue left in place would take losses nobody walks, +// and say nothing. +func (in *Intake) releaseRepairWorkers() { + in.mu.Lock() + defer in.mu.Unlock() + in.repairQueue = nil } // repairWorker walks one loss at a time until ctx ends. diff --git a/internal/connector/round9_test.go b/internal/connector/round9_test.go index 7edf2f5e5..f5c442d71 100644 --- a/internal/connector/round9_test.go +++ b/internal/connector/round9_test.go @@ -160,3 +160,33 @@ func TestAnInMemoryLedgerIsRefused(t *testing.T) { require.NoError(t, err) require.NoError(t, ledger.Close()) } + +// Each Run gets its own repair pool. A pool left over from a finished run has +// no workers, so a second Run would queue losses nobody walks — silently. +func TestASecondRunRepairsToo(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + intake.opts.Minter = stubMinter{} + + record := func() { + _, err := ledger.RecordLoss(context.Background(), []int64{17099838509}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + } + + first, cancelFirst := context.WithCancel(context.Background()) + record() + require.NoError(t, intake.resumeReconciliation(first)) + require.Eventually(t, func() bool { return polls.calls.Load() >= 1 }, 5*time.Second, 5*time.Millisecond) + cancelFirst() + intake.repairs.Wait() + intake.releaseRepairWorkers() + after := polls.calls.Load() + + second, cancelSecond := context.WithCancel(context.Background()) + defer cancelSecond() + record() + require.NoError(t, intake.resumeReconciliation(second)) + require.Eventually(t, func() bool { return polls.calls.Load() > after }, 5*time.Second, 5*time.Millisecond, + "the second run's repairs are queued to a pool with no workers") +} From 38f439f305f81c31ee0090a7b2694bce4c80d187 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 01:41:55 +0200 Subject: [PATCH 29/49] Hand over every record still unjudged on a start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ledger row is written before the pointer line and before the queue, so that a crash can never lose the pointer. The cost is the other way round: a failure after the commit — stdout gone, a shutdown mid-hand-off — leaves an event the ledger's own dedupe suppresses on every retry, and nothing would have offered it again. A start now offers every record still in seen, in id order, in batches. Offering one twice costs nothing, since the queue carries ids and admission moves a record out of seen before it acts, and a shutdown mid-requeue leaves the rest for the next start. The restart test said the event a previous run had seen was "not a second unit of work". Half of that stands and is now stated properly: it is not written out again, and it is handed over exactly once, because it was left unjudged. --- internal/connector/intake.go | 36 ++++++++++ internal/connector/intake_feed_test.go | 16 ++++- internal/connector/intake_test.go | 13 +++- internal/connector/ledger_events.go | 11 +++ internal/connector/requeue_test.go | 94 ++++++++++++++++++++++++++ 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 internal/connector/requeue_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index ec78b2f46..c2c5418d6 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -285,6 +285,9 @@ func (in *Intake) Run(ctx context.Context) error { if err := in.resumeReconciliation(repairCtx); err != nil { return err } + if err := in.requeueSeen(ctx); err != nil { + return err + } since := in.opts.SinceEventID for { @@ -726,6 +729,39 @@ func entryClassOf(resumeURL string) EntryClass { } } +// requeueSeen hands every record still in seen to the queue. +// +// The ledger row is written before the pointer line and before the hand-off, +// so that a crash can never lose the pointer — but that ordering means a +// failure after the commit leaves an event the ledger's own dedupe will +// suppress on every retry. Nothing may be left in that state, so a start +// offers every record nothing has judged yet. Offering one twice costs +// nothing: the queue carries ids, and admission moves a record out of seen +// before it acts. +func (in *Intake) requeueSeen(ctx context.Context) error { + var after int64 + for { + records, err := in.ledger.RecordsInStateAfter(ctx, StateSeen, after, requeueBatch) + if err != nil { + return err + } + for _, record := range records { + if err := in.queue.Offer(ctx, record.ID); err != nil { + // A shutdown mid-requeue leaves the rest recorded and still + // seen, which the next start offers again. + return err + } + after = record.ID + } + if len(records) < requeueBatch { + return nil + } + } +} + +// requeueBatch is how many seen records a start reads at a time. +const requeueBatch = 500 + // resumeReconciliation restarts every open loss's repair walk on start. A // crash between the overflow and its repair is a delay, not a loss of the // record. diff --git a/internal/connector/intake_feed_test.go b/internal/connector/intake_feed_test.go index 80b4fc086..d3b7dfc25 100644 --- a/internal/connector/intake_feed_test.go +++ b/internal/connector/intake_feed_test.go @@ -256,11 +256,21 @@ func TestIntakeSurvivesARestartWithoutDuplicating(t *testing.T) { // The ledger row is written before the pointer line and the hand-off, so // the counts are asserted on their own terms. - require.Eventually(t, func() bool { return countLines(pointers.String()) == 1 && queue.Depth() == 1 }, + // + // One pointer line: the event the previous run already saw is not written + // out again. Two ids in the queue: that event was left unjudged in the + // ledger, so this start hands it over too, exactly once. + require.Eventually(t, func() bool { return countLines(pointers.String()) == 1 && queue.Depth() == 2 }, 5*time.Second, 5*time.Millisecond, - "the event the previous run already saw is not a second unit of work") + "the event the previous run already saw is not written out again, and the unjudged one is handed over") assert.Equal(t, 1, countLines(pointers.String())) - assert.Equal(t, 1, queue.Depth()) + queued := map[int64]int{} + for range 2 { + id, err := queue.Take(ctx) + require.NoError(t, err) + queued[id]++ + } + assert.Equal(t, map[int64]int{17099838500: 1, 17099838501: 1}, queued) cancel() select { diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go index 680e3f20e..64a73c585 100644 --- a/internal/connector/intake_test.go +++ b/internal/connector/intake_test.go @@ -50,7 +50,18 @@ func (c *fixedClock) now() time.Time { return c.at } func newTestIntake(t *testing.T, polls eventfeed.PollSource, pointers io.Writer) (*Intake, *Ledger, *Queue) { t.Helper() - ledger := newTestLedger(t) + return newTestIntakeWith(t, newTestLedger(t), polls, pointers) +} + +// newTestIntakeOn builds an intake over an existing ledger, as a later start +// would. +func newTestIntakeOn(t *testing.T, ledger *Ledger, pointers io.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() + return newTestIntakeWith(t, ledger, nil, pointers) +} + +func newTestIntakeWith(t *testing.T, ledger *Ledger, polls eventfeed.PollSource, pointers io.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) require.NoError(t, err) if polls == nil { diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go index 3ef86b43f..77ac597ae 100644 --- a/internal/connector/ledger_events.go +++ b/internal/connector/ledger_events.go @@ -121,6 +121,17 @@ func (l *Ledger) Get(ctx context.Context, id int64) (Record, bool, error) { return records[0], true, nil } +// RecordsInStateAfter returns up to limit records in state whose id is above +// afterID, oldest first — the paging form of RecordsInState. +func (l *Ledger) RecordsInStateAfter(ctx context.Context, state RecordState, afterID int64, limit int) ([]Record, error) { + rows, err := l.db.QueryContext(ctx, + selectRecords+` WHERE state = ? AND id > ? ORDER BY id LIMIT ?`, string(state), afterID, limit) + if err != nil { + return nil, fmt.Errorf("connector: list %s records: %w", state, err) + } + return scanRecords(rows) +} + // RecordsInState returns up to limit records in state, oldest event first. // Every non-terminal state is re-run on start, and this is how they are found. func (l *Ledger) RecordsInState(ctx context.Context, state RecordState, limit int) ([]Record, error) { diff --git a/internal/connector/requeue_test.go b/internal/connector/requeue_test.go new file mode 100644 index 000000000..c3e91d7eb --- /dev/null +++ b/internal/connector/requeue_test.go @@ -0,0 +1,94 @@ +package connector + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("stdout is gone") } + +// B2: the ledger row is written before the pointer line and the hand-off, so +// that a crash cannot lose the pointer. The cost is that a failure after the +// commit leaves an event the ledger's dedupe will suppress on every retry. +// Nothing may be left in that state: a record still seen is queued again on +// the next start. +func TestAnEventWhoseHandOffFailedIsQueuedOnTheNextStart(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + + // A first run whose stdout is gone: the row commits, the pointer line + // fails, and the id never reaches the queue. + first, _, firstQueue := newTestIntakeOn(t, ledger, failingWriter{}) + require.Error(t, first.ingest(ctx, testEvent(17099838500), LanePoll)) + require.Zero(t, firstQueue.Depth()) + + record, ok, err := ledger.Get(ctx, 17099838500) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, StateSeen, record.State) + + // The next start hands it over. + second, _, secondQueue := newTestIntakeOn(t, ledger, nil) + require.NoError(t, second.requeueSeen(ctx)) + assert.Equal(t, 1, secondQueue.Depth()) + id, err := secondQueue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(17099838500), id) +} + +// Records a later stage has already taken are not queued again. +func TestOnlyRecordsStillSeenAreQueuedOnAStart(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + for _, id := range []int64{1, 2, 3} { + _, err := ledger.RecordSeen(ctx, testEvent(id), LanePoll) + require.NoError(t, err) + } + require.NoError(t, ledger.SetState(ctx, 2, StateAdmitted, "")) + require.NoError(t, ledger.SetState(ctx, 3, StateDiscarded, "untrusted_author")) + + intake, _, queue := newTestIntakeOn(t, ledger, nil) + require.NoError(t, intake.requeueSeen(ctx)) + assert.Equal(t, 1, queue.Depth()) + id, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), id) +} + +// The hand-off waits for room rather than dropping, and a shutdown mid-requeue +// leaves the rest for the next start. +func TestRequeueingStopsOnShutdownAndLeavesTheRest(t *testing.T) { + ledger := newTestLedger(t) + ctx, cancel := context.WithCancel(context.Background()) + for _, id := range []int64{1, 2, 3} { + _, err := ledger.RecordSeen(ctx, testEvent(id), LanePoll) + require.NoError(t, err) + } + intake, _, queue := newTestIntakeOn(t, ledger, nil) + small, err := NewQueue(1, 1) + require.NoError(t, err) + intake.queue = small + _ = queue + + done := make(chan error, 1) + go func() { done <- intake.requeueSeen(ctx) }() + require.Eventually(t, small.Paused, 2*time.Second, time.Millisecond) + cancel() + select { + case err := <-done: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("requeueing should stop on shutdown") + } + + remaining, err := ledger.RecordsInState(context.Background(), StateSeen, 10) + require.NoError(t, err) + assert.Len(t, remaining, 3, "nothing is consumed by being queued; the next start sees them all") +} From f4c4c8a6e332911e36fdbb33c632a62f4931bf24 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 01:47:49 +0200 Subject: [PATCH 30/49] Say that a re-offered record's pointer line is not re-emitted A record whose pointer write failed is handed over again on the next start but gets no line then. The work happens through the queue; only the stdout stream keeps the gap, and re-emitting would put duplicate lines in front of every watcher for the ordinary case. --- internal/connector/intake.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index c2c5418d6..3210c7fac 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -738,6 +738,12 @@ func entryClassOf(resumeURL string) EntryClass { // offers every record nothing has judged yet. Offering one twice costs // nothing: the queue carries ids, and admission moves a record out of seen // before it acts. +// +// The pointer line is NOT written again here. A record that got no line when +// it was recorded — the write that failed — never gets one, and the work still +// happens through the queue. Re-emitting would put duplicate lines in front of +// every watcher for the ordinary case, to close a gap in the stream rather +// than in the work. func (in *Intake) requeueSeen(ctx context.Context) error { var after int64 for { From 0670c704406898e4c1f834fb8eb0f04fda4792a2 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 02:06:15 +0200 Subject: [PATCH 31/49] Derive a queue crossing from the operation that caused it, not a later read The warning transition read the channel's length after the operation, so an offer and a concurrent take could both observe the depth AFTER the take: the queue really crossed the threshold and no warning fired at all. The depth is now a counter the queue owns, and each operation's delta is applied under the lock that decides the crossing. An offer counts its id before sending it, so a take can only ever decrement something an increment already covered, and the two can no longer invert. An id waiting for room is backlog too, and the pause callback now says so. TestACrossingIsNotLostToAConcurrentTake forces exactly that interleaving through a test seam and fails with the old shape: zero warnings for a queue that crossed and came back. --- internal/connector/queue.go | 66 ++++++++++++++++++++++++-------- internal/connector/queue_test.go | 47 ++++++++++++++++++++++- 2 files changed, 96 insertions(+), 17 deletions(-) diff --git a/internal/connector/queue.go b/internal/connector/queue.go index afef468e8..f41a950ca 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -36,6 +36,11 @@ type Queue struct { // warning that no later event will clear. edges sync.Mutex warned bool + // depth is the queue's length as the operations themselves report it. + depth int + // afterChannelOp runs between a channel operation and the transition it + // produces. A test seam: it is where the queue was losing a crossing. + afterChannelOp func() // pending holds edges decided but not yet delivered, in the order they // were decided; delivering says a goroutine is already draining them. pending []queueEdge @@ -73,9 +78,15 @@ func NewQueue(warnAt, pauseAt int) (*Queue, error) { // at the pause threshold. Waiting here is the pause: the caller is the feed's // delivery path, and it is not reading the feed while it waits. func (q *Queue) Offer(ctx context.Context, id int64) error { + // Counted before it is sent. An id on its way into the queue is backlog + // either way, and counting it first is what keeps the depth honest: a + // take can only receive what a send has already put in, so its decrement + // can never be applied before the increment it belongs to, and a crossing + // can never be lost to the order two operations happen to take the lock in. + q.applyDelta(1) select { case q.ids <- id: - q.noteDepth() + q.afterOp() return nil default: } @@ -91,9 +102,11 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { select { case q.ids <- id: - q.noteDepth() + q.afterOp() return nil case <-ctx.Done(): + // It never went in, so it is not backlog. + q.applyDelta(-1) return ctx.Err() } } @@ -106,7 +119,8 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { func (q *Queue) Take(ctx context.Context) (int64, error) { select { case id := <-q.ids: - q.noteDepth() + q.afterOp() + q.applyDelta(-1) return id, nil case <-ctx.Done(): return 0, ctx.Err() @@ -114,25 +128,45 @@ func (q *Queue) Take(ctx context.Context) (int64, error) { } // Depth is the number of ids waiting. -func (q *Queue) Depth() int { return len(q.ids) } +func (q *Queue) Depth() int { + q.edges.Lock() + defer q.edges.Unlock() + return q.depth +} // Paused reports whether an offer is currently waiting for room — which is to // say whether the feed is being consumed. func (q *Queue) Paused() bool { return q.waiting.Load() > 0 } -// noteDepth fires the warning edges. It is edge-triggered, not level: a -// backlog that sits above the threshold for an hour is one warning, and the -// recovery is the other half of the pair, so a warning is never left standing -// after the thing it warned about went away. -func (q *Queue) noteDepth() { - // The transition is decided under the lock, and appended to one ordered - // list of edges. Exactly one goroutine at a time delivers that list, in - // order, with the lock released around each callback: an operator sees the - // edges in the order the state took them, and a callback may use the queue - // — its own edge is queued behind, and delivered by, the drain already - // running. +// afterOp is a test seam: it runs between a channel operation and whatever +// follows it, which is where a crossing used to be lost. +func (q *Queue) afterOp() { + if q.afterChannelOp != nil { + q.afterChannelOp() + } +} + +// applyDelta records what one operation does to the depth and fires the +// warning edges it crosses. +// +// The depth is a counter this method owns, not a later read of the channel's +// length. With a length read, an offer and a concurrent take could both +// observe the depth AFTER the take, and a queue that really crossed the +// threshold raised no warning at all. Every delta is applied under one lock +// and the crossing is derived from the depth that operation produced. +// +// The edges are edge-triggered, not level: a backlog that sits above the +// threshold for an hour is one warning, and the recovery is the other half of +// the pair, so a warning is never left standing after the thing it warned +// about went away. +func (q *Queue) applyDelta(delta int) { + // The transition is decided under the lock; the callback runs after it, so + // a callback may observe or use the queue without deadlocking against the + // operation that raised it. Callbacks can therefore arrive out of order + // across goroutines, but the state they report on never is. q.edges.Lock() - depth := q.Depth() + q.depth += delta + depth := q.depth switch { case depth >= q.warnAt && !q.warned: q.warned = true diff --git a/internal/connector/queue_test.go b/internal/connector/queue_test.go index 2010d74a0..697e6fb61 100644 --- a/internal/connector/queue_test.go +++ b/internal/connector/queue_test.go @@ -2,6 +2,8 @@ package connector import ( "context" + "sync" + "sync/atomic" "testing" "time" @@ -53,7 +55,9 @@ func TestQueuePausesTheCallerAtTheThreshold(t *testing.T) { select { case depth := <-paused: - assert.Equal(t, 2, depth) + // Three: the two in the queue and the one waiting to go in, which is + // backlog too. + assert.Equal(t, 3, depth) case <-time.After(2 * time.Second): t.Fatal("the third offer should have waited for room") } @@ -103,3 +107,44 @@ func TestQueueRefusesNonsenseThresholds(t *testing.T) { _, err = NewQueue(5, 0) assert.Error(t, err) } + +// F2: a crossing is never lost. An offer and a concurrent take could both read +// the depth after the take, so a queue that really crossed the threshold +// raised no warning at all. +func TestACrossingIsNotLostToAConcurrentTake(t *testing.T) { + queue, err := NewQueue(1, 4) + require.NoError(t, err) + var warns, recovers int + var mu sync.Mutex + queue.OnWarn = func(int) { mu.Lock(); warns++; mu.Unlock() } + queue.OnRecover = func(int) { mu.Lock(); recovers++; mu.Unlock() } + + ctx := context.Background() + taken := make(chan int64, 1) + // The take lands between the offer's send and the transition it produces. + // A flag, not a Once: the take runs this hook too, and must not wait on + // the offer that is waiting for it. + var interleaved atomic.Bool + queue.afterChannelOp = func() { + if !interleaved.CompareAndSwap(false, true) { + return + } + done := make(chan struct{}) + go func() { + defer close(done) + id, err := queue.Take(ctx) + assert.NoError(t, err) + taken <- id + }() + <-done + } + + require.NoError(t, queue.Offer(ctx, 42)) + assert.Equal(t, int64(42), <-taken) + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, 1, warns, "the queue crossed the threshold, so it warned") + assert.Equal(t, 1, recovers, "and the take brought it back") + assert.Zero(t, queue.Depth()) +} From b76da1617c503f9e46a53994d0a0e3d600ef2af7 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 02:28:57 +0200 Subject: [PATCH 32/49] Wait as long as a throttled repair poll was told to A throttled poll was handled as an ordinary transient, so the server's Retry-After was discarded and the walk polled again on its own cadence: a fifteen-minute wait answered every minute. The seam's contract is that the value is honored exactly and is exempt from the cadence's cap, so the pass now carries it out and the next pass waits the longer of the two. TestARepairWalkHonoursRetryAfter records what the walk waits and fails when a throttle is folded back in with the other transients. --- internal/connector/repair.go | 26 +++++++++++++++++++--- internal/connector/repair_bounds_test.go | 28 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/internal/connector/repair.go b/internal/connector/repair.go index a0ec725b9..c137fc669 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -26,6 +26,9 @@ type repairWalker struct { polls eventfeed.PollSource // maxPages caps the pages one pass walks; zero means maxRepairPagesPerPass. maxPages int + // retryAfter is a server-directed wait the last pass was given. It is + // honored exactly and is exempt from the repair cadence's own cap. + retryAfter time.Duration // origin is the API origin every URL the walk follows must stay on. origin string filters eventfeed.Filters @@ -101,8 +104,14 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { // A missing id the walk did not serve is NOT yet a gap: `next` can end // while an event is still inside the poll lane's safety delay. The - // walk repeats on the repair cadence until the window closes. - if err := w.wait(ctx, w.interval); err != nil { + // walk repeats on the repair cadence until the window closes — or + // after a server-directed wait, whichever is longer. + wait := w.interval + if w.retryAfter > wait { + wait = w.retryAfter + } + w.retryAfter = 0 + if err := w.wait(ctx, wait); err != nil { return err } } @@ -384,7 +393,18 @@ func (w *repairWalker) pollFailure(ctx context.Context, loss *Loss, cursor event } return &eventfeed.Cursor{Since: strconv.FormatInt(loss.RepairSince, 10)}, nil - case eventfeed.PollTransient, eventfeed.PollThrottled, eventfeed.PollUnauthorized: + case eventfeed.PollThrottled: + // The server named a wait. It is a directive, not a hint: polling + // again on the repair cadence would answer a fifteen-minute + // Retry-After every minute. + if pollErr.RetryAfter > w.retryAfter { + w.retryAfter = pollErr.RetryAfter + } + w.log.Warn("a repair poll was throttled; waiting as the server asked", + "loss_id", loss.ID, "retry_after", pollErr.RetryAfter) + return nil, nil + + case eventfeed.PollTransient, eventfeed.PollUnauthorized: // A reason to try again on the next repair poll, not a reason to call // the ids unrecovered. A slow or throttled walk delays nothing else. w.log.Warn("a repair poll failed; retrying on the repair cadence", "loss_id", loss.ID, "failure", failureKind(err)) diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go index 22722e5db..88b13b5be 100644 --- a/internal/connector/repair_bounds_test.go +++ b/internal/connector/repair_bounds_test.go @@ -141,3 +141,31 @@ func TestACanceledRepairWalkLeavesTheLossOpenForTheNextStart(t *testing.T) { }) } } + +// C3: a throttled repair poll waits at least as long as the server said. The +// seam's RetryAfter is a directive, not a hint: polling again on the repair +// cadence would answer a fifteen-minute wait every minute. +func TestARepairWalkHonoursRetryAfter(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + var waits []time.Duration + walker.sleep = func(_ context.Context, d time.Duration) error { + waits = append(waits, d) + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + require.NotEmpty(t, waits) + assert.GreaterOrEqual(t, waits[0], 15*time.Minute, + "the server asked for fifteen minutes; the repair cadence does not overrule it") +} From 251aa9e87bb2cd1a82456cd506181267d24fc24e Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 02:41:50 +0200 Subject: [PATCH 33/49] A throttle is not a verdict, and a new Run starts clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A final pass whose only attempt was refused with Retry-After closed the loss and called its ids unrecovered, though nothing had been learned about them — most likely for a loss that waited in the queue until after its own window. Such a pass now leaves the loss open, and the next start tries again. Run is reusable, and each one builds its own repair pool, but its other per-run state was never reset: a checkpoint an earlier run saved would clear this run's --since before it had saved anything of its own, and an earlier abort would end it before it began. Run now clears that state on entry. What survives is knowledge about the account rather than the run — the listed and learned projects. --- internal/connector/intake.go | 26 +++++++++++++++++ internal/connector/repair.go | 8 ++++++ internal/connector/repair_bounds_test.go | 22 +++++++++++++++ internal/connector/round9_test.go | 36 ++++++++++++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 3210c7fac..fdaecc0eb 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -270,6 +270,7 @@ func (in *Intake) CheckpointKey() eventfeed.CheckpointKey { return in.key } // position refused before this run had a safe re-entry of its own. Everything // else the feed can recover from, it recovers from inside the package. func (in *Intake) Run(ctx context.Context) error { + in.resetRunState() // Repairs get a child lifetime that ends when Run does, whatever the // reason. A repair is off the delivery path: a terminal feed must not wait // out its sixty-second cadence, and an unfinished repair resumes on the @@ -312,6 +313,31 @@ func (in *Intake) Run(ctx context.Context) error { } } +// resetRunState clears everything scoped to one Run. +// +// Run is reusable — each one builds its own repair pool — so nothing a +// previous one decided may leak into the next: a checkpoint it saved would +// clear this run's --since before this run has saved anything, and an abort it +// suffered would end this one before it began. What survives is knowledge +// about the account rather than the run: the listed and learned projects. +func (in *Intake) resetRunState() { + in.mu.Lock() + defer in.mu.Unlock() + in.checkpointed = false + in.abortErr = nil + in.hasReentry = false + in.reentry = eventfeed.Start{} + in.reentryLog = "" + in.reentryReplays = false + in.replaying = false + in.enteredByReentry = false + in.promotedThisRun = false + select { + case <-in.reconnect: + default: + } +} + // errReconnect asks the supervisor for a fresh connection. It is not a // failure. var errReconnect = errors.New("connector: reconnect the feed") diff --git a/internal/connector/repair.go b/internal/connector/repair.go index c137fc669..161045823 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -126,6 +126,14 @@ func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { if _, err := w.walk(ctx, loss); err != nil && !errors.Is(err, errReconciliationEnded) { return err } + if w.retryAfter > 0 { + // The server refused the only attempt this pass had and named a wait. + // Nothing was learned about the missing ids, and a throttle is not a + // verdict: the loss stays open and the next start tries again. + w.log.Warn("the last repair attempt was throttled; the loss stays open for the next start", + "loss_id", loss.ID, "retry_after", w.retryAfter) + return nil + } if err := ctx.Err(); err != nil { // The final attempt did not run to its end; closing now would condemn // ids on a pass that never happened. diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go index 88b13b5be..2079353c0 100644 --- a/internal/connector/repair_bounds_test.go +++ b/internal/connector/repair_bounds_test.go @@ -169,3 +169,25 @@ func TestARepairWalkHonoursRetryAfter(t *testing.T) { assert.GreaterOrEqual(t, waits[0], 15*time.Minute, "the server asked for fifteen minutes; the repair cadence does not overrule it") } + +// C2: a throttle is not a verdict. A final pass whose only poll was refused +// with Retry-After has learned nothing about the missing ids, so the loss +// stays open for the next start rather than closing them as unrecovered. +func TestAThrottledFinalPassLeavesTheLossOpen(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + require.NoError(t, walker.reconcile(ctx, loss)) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Len(t, open, 1, "the window closed, but nothing was learned about the ids") + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered, "a throttle is not a verdict") +} diff --git a/internal/connector/round9_test.go b/internal/connector/round9_test.go index f5c442d71..018eb8a75 100644 --- a/internal/connector/round9_test.go +++ b/internal/connector/round9_test.go @@ -3,6 +3,7 @@ package connector import ( "bytes" "context" + "errors" "log/slog" "path/filepath" "strings" @@ -190,3 +191,38 @@ func TestASecondRunRepairsToo(t *testing.T) { require.Eventually(t, func() bool { return polls.calls.Load() > after }, 5*time.Second, 5*time.Millisecond, "the second run's repairs are queued to a pool with no workers") } + +// Run is reusable, so its per-run state starts clean. A checkpoint saved by an +// earlier run would otherwise clear this run's --since before it has saved +// anything of its own, and an earlier abort would end it before it began. +func TestASecondRunStartsWithCleanRunState(t *testing.T) { + ledger := newTestLedger(t) + intake, transport, minter, polls := newFeedIntake(t, ledger, Options{SinceEventID: 17099838000}) + intake.checkpointed = true + intake.abortErr = errors.New("an earlier run's fatal") + intake.hasReentry = true + intake.replaying = true + intake.enteredByReentry = true + + minter.ScriptTicket(ticket()) + polls.ScriptPage(eventfeed.PollPage{Position: "p1"}) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := runInBackground(ctx, t, intake) + subscribedConn(t, transport) + + require.Eventually(t, func() bool { return polls.CallCount() > 0 }, 5*time.Second, 10*time.Millisecond, + "an earlier run's abort must not end this one") + assert.Equal(t, "17099838000", polls.Calls()[0].Cursor.Since) + + // checkpointed is legitimately true again by now, this run having saved + // its own page; what must not come back is the rest. + intake.mu.Lock() + clean := intake.abortErr == nil && !intake.hasReentry && !intake.replaying + intake.mu.Unlock() + assert.True(t, clean, "per-run state starts clean") + + cancel() + awaitReturn(t, done, "Run should return on shutdown") +} From 886ec7d83fc96685cc4d31ef82e42bc0fed411b9 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 02:52:45 +0200 Subject: [PATCH 34/49] Never strand a loss, and never let a throttle spend its window Two rules, each in one place. A throttle is the server asking for patience. It is not a failure and not a verdict, and the time it costs belongs to the server: the loss's window moves out by exactly the wait, wherever the throttle arrives, so a slow server can neither run a loss out of attempts nor turn one into unrecovered ids. Only a window that expires without a throttle closes a loss. A loss is always somewhere: being walked, or waiting to be. The repair queue is bounded and a walk can end early, so a sweep re-offers every open loss on a cadence, and a loss already in flight is not offered twice. Waiting for a restart was not an answer for a connector that runs for weeks. --- internal/connector/intake.go | 72 ++++++++++++++++++++++- internal/connector/ledger_recovery.go | 11 ++++ internal/connector/loss_schedule_test.go | 74 ++++++++++++++++++++++++ internal/connector/repair.go | 66 ++++++++++++--------- internal/connector/repair_bounds_test.go | 39 +++++++++---- 5 files changed, 223 insertions(+), 39 deletions(-) create mode 100644 internal/connector/loss_schedule_test.go diff --git a/internal/connector/intake.go b/internal/connector/intake.go index fdaecc0eb..49f5662ce 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -167,6 +167,12 @@ type Intake struct { repairs sync.WaitGroup repairQueue chan Loss + // repairQueueSize and repairSweep override the pool's defaults in tests. + repairQueueSize int + repairSweep time.Duration + // inFlight is the losses a worker is walking or the queue is holding, so + // the sweeper does not offer one twice. + inFlight map[int64]bool // lifetime is Run's context. Repair walks are bound to it rather than to // a connection, so a reconnect does not abandon a walk and a shutdown does // not strand Run waiting on one — an unfinished walk simply resumes on the @@ -829,10 +835,26 @@ func (in *Intake) startRepair(loss Loss) { in.log.Warn("no repair workers are running; this loss stays open for the next start", "loss_id", loss.ID) return } + in.mu.Lock() + if in.inFlight == nil { + in.inFlight = make(map[int64]bool) + } + if in.inFlight[loss.ID] { + in.mu.Unlock() + return + } + in.inFlight[loss.ID] = true + in.mu.Unlock() + select { case queue <- loss: default: - in.log.Warn("the repair queue is full; this loss stays open for the next start", "loss_id", loss.ID) + // No room now. The sweeper offers it again, so a loss is never left + // with nothing that will attempt it. + in.mu.Lock() + delete(in.inFlight, loss.ID) + in.mu.Unlock() + in.log.Warn("the repair queue is full; this loss is left for the sweep", "loss_id", loss.ID) } } @@ -847,15 +869,56 @@ func (in *Intake) startRepairWorkers(ctx context.Context) { in.mu.Unlock() return } - queue := make(chan Loss, repairQueueDepth) + size := in.repairQueueSize + if size <= 0 { + size = repairQueueDepth + } + queue := make(chan Loss, size) in.repairQueue = queue in.mu.Unlock() for range maxConcurrentRepairs { in.repairs.Add(1) go in.repairWorker(ctx, queue) } + in.repairs.Add(1) + go in.sweepLosses(ctx) +} + +// sweepLosses offers every open loss nothing is already walking. +// +// The queue is bounded, so an overloaded connector can turn one away; and a +// walk can end early, leaving its loss open. Neither may leave a loss with +// nothing that will attempt it again, and waiting for a restart is not an +// answer for a connector that runs for weeks. The sweep is what closes that: +// every open loss is either being walked, or is offered again here. +func (in *Intake) sweepLosses(ctx context.Context) { + defer in.repairs.Done() + every := in.repairSweep + if every <= 0 { + every = defaultRepairSweep + } + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + losses, err := in.ledger.OpenLosses(ctx) + if err != nil { + in.log.Warn("could not read the open losses", "error", err) + continue + } + for _, loss := range losses { + in.startRepair(loss) + } + } + } } +// defaultRepairSweep is how often the open losses are re-offered. +const defaultRepairSweep = time.Minute + // releaseRepairWorkers forgets a pool whose workers have stopped, so the next // Run starts its own. A queue left in place would take losses nobody walks, // and say nothing. @@ -884,6 +947,11 @@ func (in *Intake) repairWorker(ctx context.Context, queue chan Loss) { } func (in *Intake) runRepair(ctx context.Context, loss Loss) { + defer func() { + in.mu.Lock() + delete(in.inFlight, loss.ID) + in.mu.Unlock() + }() walker := &repairWalker{ ledger: in.ledger, polls: in.opts.PollsFor(), diff --git a/internal/connector/ledger_recovery.go b/internal/connector/ledger_recovery.go index 955a7ad46..8e82192fd 100644 --- a/internal/connector/ledger_recovery.go +++ b/internal/connector/ledger_recovery.go @@ -257,6 +257,17 @@ func (l *Ledger) MissingIDs(ctx context.Context, lossID int64, state LossState) return ids, rows.Err() } +// ExtendLossDeadline pushes a loss's window out, for time the walk spent +// waiting on the server rather than working. +func (l *Ledger) ExtendLossDeadline(ctx context.Context, lossID int64, deadline time.Time) error { + _, err := l.db.ExecContext(ctx, + `UPDATE losses SET deadline_at = ? WHERE id = ?`, stamp(deadline.UTC()), lossID) + if err != nil { + return fmt.Errorf("connector: extend loss deadline: %w", err) + } + return nil +} + // SaveRepairCursor records where the repair walk has reached. // // This is the walk's own cursor and it is stored on the loss, never on the diff --git a/internal/connector/loss_schedule_test.go b/internal/connector/loss_schedule_test.go new file mode 100644 index 000000000..ffc3c19db --- /dev/null +++ b/internal/connector/loss_schedule_test.go @@ -0,0 +1,74 @@ +package connector + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" +) + +// C2: a loss is always somewhere — repaired, scheduled for another attempt, or +// closed with a reason. Waiting on a server's own delay is not spending the +// window, so a throttle can neither condemn the ids nor run the clock out. +func TestAThrottleDoesNotSpendTheLossWindow(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + // A window with a minute left, and a server asking for fifteen. + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-9*time.Minute), 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + before := loss.DeadlineAt + + // Two refusals, each longer than what is left of the window, then the + // page. Only a window that does not spend on the server's own delays + // still has attempts left by then. + throttle := &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute} + polls := &scriptedPolls{ + errs: []error{throttle, throttle}, + pages: []eventfeed.PollPage{{}, {}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.sleep = func(_ context.Context, d time.Duration) error { + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered, "the server refused the attempt; it says nothing about the ids") + recovered, err := ledger.MissingIDs(ctx, loss.ID, LossRecovered) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, recovered, "the retry after the wait found it") + _ = before +} + +// C2: a loss the repair queue had no room for is swept back in, without +// waiting for a restart. +func TestALossSkippedByAFullQueueIsSweptBackIn(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + intake.repairQueueSize = 1 + intake.repairSweep = 20 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for i := range 6 { + _, err := ledger.RecordLoss(ctx, []int64{int64(2000 + i)}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + } + require.NoError(t, intake.resumeReconciliation(ctx)) + + require.Eventually(t, func() bool { + open, err := ledger.OpenLosses(context.Background()) + return err == nil && len(open) == 0 + }, 10*time.Second, 20*time.Millisecond, "every open loss is eventually attempted, queue or no queue") + + cancel() + intake.repairs.Wait() +} diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 161045823..5d8cdb8b6 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -60,9 +60,9 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { if !w.now().Before(loss.DeadlineAt) { // Past the window already — perhaps across restarts whose walks // each ended in a failure no retry fixes. One last pass is still - // worth trying; after it, the loss closes either way. - err := w.finalPass(ctx, &loss) - return err + // worth trying; after it the loss closes, unless the server + // refused that attempt too. + return w.finalPass(ctx, &loss) } done, err := w.settled(ctx, loss) @@ -86,22 +86,6 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { return err } - if !w.now().Before(loss.DeadlineAt) { - // The window is closed. What is still missing is a late - // straggler, a recording deleted before it became poll-visible, - // or history behind the epoch. It is recorded as unrecovered and - // shown by status — the poll lane may still serve it later, and - // intake resolves it then like any other event. - unrecovered, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()) - if err != nil { - return err - } - if unrecovered > 0 { - w.log.Error("a buffer overflow left events unrecovered", "loss_id", loss.ID, "unrecovered", unrecovered) - } - return nil - } - // A missing id the walk did not serve is NOT yet a gap: `next` can end // while an event is still inside the poll lane's safety delay. The // walk repeats on the repair cadence until the window closes — or @@ -110,13 +94,32 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { if w.retryAfter > wait { wait = w.retryAfter } - w.retryAfter = 0 + if w.retryAfter > 0 { + if err := w.postpone(ctx, &loss); err != nil { + return err + } + } if err := w.wait(ctx, wait); err != nil { return err } } } +// postpone moves a loss's window out by the wait the server asked for. +// +// This is the whole rule for a throttle, in one place: it is not a failure and +// not a verdict, and the time it costs belongs to the server, not to the loss. +// The window therefore never runs out while a server is asking for patience, +// and a throttled attempt never condemns an id. +func (w *repairWalker) postpone(ctx context.Context, loss *Loss) error { + extended := loss.DeadlineAt.Add(w.retryAfter) + if err := w.ledger.ExtendLossDeadline(ctx, loss.ID, extended); err != nil { + return err + } + loss.DeadlineAt = extended + return nil +} + // finalPass runs one walk for a loss past its window and then closes it, // whatever the walk managed: its missing ids become unrecovered. func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { @@ -126,13 +129,24 @@ func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { if _, err := w.walk(ctx, loss); err != nil && !errors.Is(err, errReconciliationEnded) { return err } + if done, err := w.settled(ctx, *loss); err != nil || done { + return err + } if w.retryAfter > 0 { - // The server refused the only attempt this pass had and named a wait. - // Nothing was learned about the missing ids, and a throttle is not a - // verdict: the loss stays open and the next start tries again. - w.log.Warn("the last repair attempt was throttled; the loss stays open for the next start", - "loss_id", loss.ID, "retry_after", w.retryAfter) - return nil + // The server refused this attempt and named a wait, so there is + // nothing to conclude about the ids. The window moves out by the wait + // and the loss is walked again. + wait := w.retryAfter + if err := w.postpone(ctx, loss); err != nil { + return err + } + w.retryAfter = 0 + w.log.Warn("the last repair attempt was throttled; waiting as the server asked and trying again", + "loss_id", loss.ID, "retry_after", wait) + if err := w.wait(ctx, wait); err != nil { + return err + } + return w.reconcile(ctx, *loss) } if err := ctx.Err(); err != nil { // The final attempt did not run to its end; closing now would condemn diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go index 2079353c0..8a753172d 100644 --- a/internal/connector/repair_bounds_test.go +++ b/internal/connector/repair_bounds_test.go @@ -170,24 +170,41 @@ func TestARepairWalkHonoursRetryAfter(t *testing.T) { "the server asked for fifteen minutes; the repair cadence does not overrule it") } -// C2: a throttle is not a verdict. A final pass whose only poll was refused -// with Retry-After has learned nothing about the missing ids, so the loss -// stays open for the next start rather than closing them as unrecovered. -func TestAThrottledFinalPassLeavesTheLossOpen(t *testing.T) { +// C2: a throttle is not a verdict. A server that only ever asks for patience +// can never turn a loss into unrecovered ids, however long its window has been +// over: the wait is the server's time, not the loss's. +func TestAPermanentlyThrottlingServerNeverCondemnsALoss(t *testing.T) { ledger := newTestLedger(t) - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-time.Hour), time.Minute, eventfeed.Filters{}) require.NoError(t, err) - polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}} + polls := &alwaysThrottling{} walker, _ := newTestWalker(t, ledger, polls, clock) - require.NoError(t, walker.reconcile(ctx, loss)) + waits := 0 + walker.sleep = func(_ context.Context, d time.Duration) error { + clock.at = clock.at.Add(d) + if waits++; waits >= 5 { + cancel() + return ctx.Err() + } + return nil + } + require.ErrorIs(t, walker.reconcile(ctx, loss), context.Canceled) - open, err := ledger.OpenLosses(ctx) + open, err := ledger.OpenLosses(context.Background()) require.NoError(t, err) - assert.Len(t, open, 1, "the window closed, but nothing was learned about the ids") - unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.Len(t, open, 1, "still scheduled, never condemned") + assert.True(t, open[0].DeadlineAt.After(loss.DeadlineAt), "the window moved out by what the server asked for") + unrecovered, err := ledger.UnrecoveredIDs(context.Background()) require.NoError(t, err) - assert.Empty(t, unrecovered, "a throttle is not a verdict") + assert.Empty(t, unrecovered) +} + +type alwaysThrottling struct{} + +func (alwaysThrottling) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { + return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute} } From 329be359ef5b636a5e066fbb2f4754af8418e46f Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 02:55:16 +0200 Subject: [PATCH 35/49] Pin the in-window postpone on its own Removing it was masked by the final pass doing the same thing, so the rule was only covered end to end. The new test reads the stored deadline at the moment the walk waits. --- internal/connector/loss_schedule_test.go | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/internal/connector/loss_schedule_test.go b/internal/connector/loss_schedule_test.go index ffc3c19db..2820da574 100644 --- a/internal/connector/loss_schedule_test.go +++ b/internal/connector/loss_schedule_test.go @@ -72,3 +72,35 @@ func TestALossSkippedByAFullQueueIsSweptBackIn(t *testing.T) { cancel() intake.repairs.Wait() } + +// The same rule inside the window: the deadline moves out the moment the +// server asks for a wait, not only on the last attempt. +func TestAThrottleInsideTheWindowPostponesItImmediately(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{ + errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}, + pages: []eventfeed.PollPage{{}, {Events: []eventfeed.Event{testEvent(17099838509)}, Position: "p"}}, + } + walker, _ := newTestWalker(t, ledger, polls, clock) + var atWait time.Time + walker.sleep = func(_ context.Context, d time.Duration) error { + if atWait.IsZero() { + open, err := ledger.OpenLosses(context.Background()) + require.NoError(t, err) + require.Len(t, open, 1) + atWait = open[0].DeadlineAt + } + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + require.False(t, atWait.IsZero(), "the walk waited") + assert.Equal(t, loss.DeadlineAt.Add(15*time.Minute).UTC(), atWait.UTC(), + "the window moved out by exactly what the server asked for") +} From 458c06411997f2d27d293cc63915dcb4ec76ab0a Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 02:55:40 +0200 Subject: [PATCH 36/49] Use the walk's own context in the postpone test --- internal/connector/loss_schedule_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/connector/loss_schedule_test.go b/internal/connector/loss_schedule_test.go index 2820da574..f4f0a3aeb 100644 --- a/internal/connector/loss_schedule_test.go +++ b/internal/connector/loss_schedule_test.go @@ -88,9 +88,9 @@ func TestAThrottleInsideTheWindowPostponesItImmediately(t *testing.T) { } walker, _ := newTestWalker(t, ledger, polls, clock) var atWait time.Time - walker.sleep = func(_ context.Context, d time.Duration) error { + walker.sleep = func(sleepCtx context.Context, d time.Duration) error { if atWait.IsZero() { - open, err := ledger.OpenLosses(context.Background()) + open, err := ledger.OpenLosses(sleepCtx) require.NoError(t, err) require.Len(t, open, 1) atWait = open[0].DeadlineAt From 753affa7d69a8156b64d896028fcb6c2cf6e6d92 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 03:06:14 +0200 Subject: [PATCH 37/49] Close the throttle, the pool's bookkeeping, and the stdout stream's last split Four findings, each with a test that failed before it. A queue callback was delivered while an offer still had its id in hand: a callback that took from the queue waited for an id the offer could not send until the callback returned. The crossing is still decided under the lock with the operation that caused it, but the callbacks wait for the channel work to finish. One throttle governed every later pass. The wait is the server's answer to the attempt it refused and says nothing about the next one, so it is cleared once used. A loss cannot live forever either. A throttle postpones its window, and a server that keeps asking for patience would postpone it without end, so a loss closes at an absolute deadline a day after it was recorded, whatever the retry state, with its ids reported. The marks that say a worker holds a loss belong to the pool, and go with it: a loss still queued when a run ended was skipped by every later run and sweep. And the stdout stream: two value-typed sinks can wrap one writer and compare unequal, so the registry cannot pair them. Intake and admission now both accept the writer itself, and a caller putting them on one stdout hands them one. --- internal/connector/admission/run.go | 18 +++-- internal/connector/intake.go | 18 ++++- internal/connector/intake_test.go | 11 ++++ internal/connector/loss_schedule_test.go | 59 +++++++++++++++++ internal/connector/ndjson/ndjson.go | 6 ++ internal/connector/queue.go | 37 +++++++---- internal/connector/repair.go | 20 ++++++ internal/connector/repair_bounds_test.go | 4 +- internal/connector/round4_test.go | 8 ++- internal/connector/round9_test.go | 27 ++++++++ internal/connector/shared_lines_test.go | 83 ++++++++++++++++++++++++ 11 files changed, 266 insertions(+), 25 deletions(-) create mode 100644 internal/connector/shared_lines_test.go diff --git a/internal/connector/admission/run.go b/internal/connector/admission/run.go index 1d57728f5..1d1f9f8b0 100644 --- a/internal/connector/admission/run.go +++ b/internal/connector/admission/run.go @@ -42,8 +42,12 @@ type RunOptions struct { // Workers is the fetcher pool size; DefaultWorkers when zero. Workers int // Lines receives one NDJSON line per committed verdict. - Lines io.Writer - Logger *slog.Logger + Lines io.Writer + // LineWriter, when set, is the writer the lines go through, and Lines is + // ignored: one sink has one writer and one lock, and a caller wiring + // admission beside intake passes the same writer to both. + LineWriter *ndjson.Writer + Logger *slog.Logger } // Run takes ids until ctx ends, deciding and committing each. It returns nil @@ -68,7 +72,7 @@ func Run(ctx context.Context, opts RunOptions) error { if log == nil { log = slog.New(slog.DiscardHandler) } - lines := &lineWriter{w: opts.Lines} + lines := &lineWriter{w: opts.Lines, out: opts.LineWriter} ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -186,10 +190,14 @@ type lineWriter struct { } func (l *lineWriter) write(v Verdict) error { - if l.w == nil { + if l.w == nil && l.out == nil { return nil } - l.once.Do(func() { l.out = ndjson.NewWriter(l.w) }) + l.once.Do(func() { + if l.out == nil { + l.out = ndjson.NewWriter(l.w) + } + }) if err := l.out.WriteLine(LineFor(v)); err != nil { return fmt.Errorf("admission: %w", err) } diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 49f5662ce..191990ff7 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -72,6 +72,11 @@ type Options struct { // serialized: an interleaved write tears a line and breaks the watcher // reading it. Pointers io.Writer + // Lines, when set, is the writer the pointer lines go through, and + // Pointers is ignored. One process stdout has one writer and one lock, so + // a caller wiring intake and admission onto the same stdout passes the + // same writer to both rather than relying on them to find each other. + Lines *ndjson.Writer // Logger receives everything else. Pointer lines are the protocol; // logging is not. Logger *slog.Logger @@ -247,7 +252,7 @@ func New(opts Options) (*Intake, error) { queue: opts.Queue, log: opts.Logger, now: opts.Clock, - pointer: newPointerWriter(opts.Pointers), + pointer: newPointerWriter(opts.Pointers, opts.Lines), reconnect: make(chan struct{}, 1), key: eventfeed.CheckpointKey{ Origin: origin, @@ -926,6 +931,10 @@ func (in *Intake) releaseRepairWorkers() { in.mu.Lock() defer in.mu.Unlock() in.repairQueue = nil + // The marks say "a worker has this", and the workers are gone. A loss + // still sitting in the dropped queue would otherwise be skipped by every + // later run and every sweep. + in.inFlight = nil } // repairWorker walks one loss at a time until ctx ends. @@ -1235,7 +1244,12 @@ type pointerWriter struct { out *ndjson.Writer } -func newPointerWriter(w io.Writer) *pointerWriter { return &pointerWriter{out: ndjson.NewWriter(w)} } +func newPointerWriter(w io.Writer, shared *ndjson.Writer) *pointerWriter { + if shared != nil { + return &pointerWriter{out: shared} + } + return &pointerWriter{out: ndjson.NewWriter(w)} +} // Pointer is the line intake writes to stdout for each newly seen event. It // carries what the feed carried and nothing more: no title, no body, no URL, diff --git a/internal/connector/intake_test.go b/internal/connector/intake_test.go index 64a73c585..3ff36c074 100644 --- a/internal/connector/intake_test.go +++ b/internal/connector/intake_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/require" "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" ) type stubMinter struct{} @@ -53,6 +55,15 @@ func newTestIntake(t *testing.T, polls eventfeed.PollSource, pointers io.Writer) return newTestIntakeWith(t, newTestLedger(t), polls, pointers) } +// newTestIntakeLines builds an intake whose pointer lines go through an +// injected writer. +func newTestIntakeLines(t *testing.T, lines *ndjson.Writer) (*Intake, *Ledger, *Queue) { + t.Helper() + intake, ledger, queue := newTestIntakeWith(t, newTestLedger(t), nil, nil) + intake.pointer = newPointerWriter(linesSink(lines), lines) + return intake, ledger, queue +} + // newTestIntakeOn builds an intake over an existing ledger, as a later start // would. func newTestIntakeOn(t *testing.T, ledger *Ledger, pointers io.Writer) (*Intake, *Ledger, *Queue) { diff --git a/internal/connector/loss_schedule_test.go b/internal/connector/loss_schedule_test.go index f4f0a3aeb..940f291d0 100644 --- a/internal/connector/loss_schedule_test.go +++ b/internal/connector/loss_schedule_test.go @@ -104,3 +104,62 @@ func TestAThrottleInsideTheWindowPostponesItImmediately(t *testing.T) { assert.Equal(t, loss.DeadlineAt.Add(15*time.Minute).UTC(), atWait.UTC(), "the window moved out by exactly what the server asked for") } + +// One throttle governs one pass. A later pass with a healthy server waits the +// ordinary cadence and does not push the window out again. +func TestOneThrottleDoesNotGovernEveryLaterPass(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + polls := &scriptedPolls{errs: []error{&eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute}}} + walker, _ := newTestWalker(t, ledger, polls, clock) + var waits []time.Duration + walker.sleep = func(sleepCtx context.Context, d time.Duration) error { + waits = append(waits, d) + clock.at = clock.at.Add(d) + if len(waits) >= 3 { + return context.Canceled + } + return nil + } + require.ErrorIs(t, walker.reconcile(ctx, loss), context.Canceled) + + require.Len(t, waits, 3) + assert.Equal(t, 15*time.Minute, waits[0], "the server asked for this one") + assert.Equal(t, walker.interval, waits[1], "and said nothing about the next") + assert.Equal(t, walker.interval, waits[2]) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, open, 1) + assert.Equal(t, loss.DeadlineAt.Add(15*time.Minute).UTC(), open[0].DeadlineAt.UTC(), + "the window moved out once, for the one wait the server asked for") +} + +// C2: a loss cannot live forever. However long a server keeps asking for +// patience, a loss closes at its absolute deadline with its ids reported. +func TestALossClosesAtItsAbsoluteDeadline(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at, 10*time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + + walker, _ := newTestWalker(t, ledger, alwaysThrottling{}, clock) + walker.sleep = func(_ context.Context, d time.Duration) error { + clock.at = clock.at.Add(d) + return nil + } + require.NoError(t, walker.reconcile(ctx, loss)) + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open, "closed exactly once, at its absolute deadline") + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Equal(t, []int64{17099838509}, unrecovered, "and its ids are reported, not forgotten") + assert.False(t, clock.at.Before(loss.DetectedAt.Add(maxLossLifetime)), "not before the absolute deadline") +} diff --git a/internal/connector/ndjson/ndjson.go b/internal/connector/ndjson/ndjson.go index b2c24e628..d09d0dfe8 100644 --- a/internal/connector/ndjson/ndjson.go +++ b/internal/connector/ndjson/ndjson.go @@ -32,6 +32,12 @@ type Writer struct { // Keying on the sink makes that impossible whoever calls this, without every // caller having to agree to pass one instance around. // +// A value-typed sink is the one case this cannot settle by itself: two copies +// can wrap one underlying writer and compare unequal, and hashing one can +// panic, so each gets its own Writer. A composition that puts two producers on +// one such sink passes them one Writer instead — intake's Options.Lines and +// admission's RunOptions.LineWriter are there for exactly that. +// // Only reference-like sinks are keyed — a pointer, a channel, an unsafe // pointer — which covers every real one, os.Stdout included. A value-typed // sink is a copy rather than the same sink, and hashing one can panic when it diff --git a/internal/connector/queue.go b/internal/connector/queue.go index f41a950ca..66887a43a 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -83,10 +83,11 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { // take can only receive what a send has already put in, so its decrement // can never be applied before the increment it belongs to, and a crossing // can never be lost to the order two operations happen to take the lock in. - q.applyDelta(1) + q.stage(1) select { case q.ids <- id: q.afterOp() + q.deliver() return nil default: } @@ -103,10 +104,12 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { select { case q.ids <- id: q.afterOp() + q.deliver() return nil case <-ctx.Done(): // It never went in, so it is not backlog. - q.applyDelta(-1) + q.stage(-1) + q.deliver() return ctx.Err() } } @@ -120,7 +123,8 @@ func (q *Queue) Take(ctx context.Context) (int64, error) { select { case id := <-q.ids: q.afterOp() - q.applyDelta(-1) + q.stage(-1) + q.deliver() return id, nil case <-ctx.Done(): return 0, ctx.Err() @@ -146,8 +150,8 @@ func (q *Queue) afterOp() { } } -// applyDelta records what one operation does to the depth and fires the -// warning edges it crosses. +// stage records what one operation does to the depth and decides the edges it +// crosses, without delivering them. // // The depth is a counter this method owns, not a later read of the channel's // length. With a length read, an offer and a concurrent take could both @@ -155,16 +159,13 @@ func (q *Queue) afterOp() { // threshold raised no warning at all. Every delta is applied under one lock // and the crossing is derived from the depth that operation produced. // -// The edges are edge-triggered, not level: a backlog that sits above the -// threshold for an hour is one warning, and the recovery is the other half of -// the pair, so a warning is never left standing after the thing it warned -// about went away. -func (q *Queue) applyDelta(delta int) { - // The transition is decided under the lock; the callback runs after it, so - // a callback may observe or use the queue without deadlocking against the - // operation that raised it. Callbacks can therefore arrive out of order - // across goroutines, but the state they report on never is. +// Deciding and delivering are separate because an offer counts its id before +// it sends it: a callback delivered there could call Take and wait for an id +// the offer has not sent yet, and neither would ever finish. So the edges wait +// for deliver, which every operation calls once its channel work is done. +func (q *Queue) stage(delta int) { q.edges.Lock() + defer q.edges.Unlock() q.depth += delta depth := q.depth switch { @@ -175,6 +176,14 @@ func (q *Queue) applyDelta(delta int) { q.warned = false q.pending = append(q.pending, queueEdge{fire: q.OnRecover, depth: depth}) } +} + +// deliver hands the staged edges to their callbacks, in the order they were +// decided, one goroutine at a time and with the lock released around each: a +// callback may use the queue, and its own edge is delivered by the drain +// already running. +func (q *Queue) deliver() { + q.edges.Lock() if q.delivering { q.edges.Unlock() return diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 5d8cdb8b6..d3db5c5b3 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -57,6 +57,19 @@ func (w *repairWalker) reconcileLoss(ctx context.Context, loss Loss) error { func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { for { + if !w.now().Before(loss.DetectedAt.Add(maxLossLifetime)) { + // The absolute deadline. A throttle postpones the window, and a + // server that keeps asking for patience could postpone it forever; + // a loss that can never close is its own kind of silence. At a day + // old it closes, whatever the retry state, with its ids reported. + unrecovered, err := w.ledger.CloseLoss(ctx, loss.ID, w.now()) + if err != nil { + return err + } + w.log.Error("a buffer overflow reached its absolute deadline; what is still missing is unrecovered", + "loss_id", loss.ID, "unrecovered", unrecovered, "age", w.now().Sub(loss.DetectedAt)) + return nil + } if !w.now().Before(loss.DeadlineAt) { // Past the window already — perhaps across restarts whose walks // each ended in a failure no retry fixes. One last pass is still @@ -98,6 +111,9 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { if err := w.postpone(ctx, &loss); err != nil { return err } + // Used: this wait was the server's answer to this attempt, and + // says nothing about the next one. + w.retryAfter = 0 } if err := w.wait(ctx, wait); err != nil { return err @@ -297,6 +313,10 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { // full recovery. var errReconciliationEnded = errors.New("connector: reconciliation ended inside the repair walk") +// maxLossLifetime is how long a loss can stay open, however often a server +// asks for patience. A day is far past any horizon the feed itself has. +const maxLossLifetime = 24 * time.Hour + // maxRepairPagesPerPass bounds the pages one repair pass walks. A thousand // pages crosses up to a million ledger rows; past that the pass yields to the // repair cadence and resumes from its saved cursor. diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go index 8a753172d..d6d3736dc 100644 --- a/internal/connector/repair_bounds_test.go +++ b/internal/connector/repair_bounds_test.go @@ -106,7 +106,9 @@ func (c *cancelingPolls) Poll(ctx context.Context, _ eventfeed.Cursor, _ eventfe func TestACanceledRepairWalkLeavesTheLossOpenForTheNextStart(t *testing.T) { for name, age := range map[string]time.Duration{ "inside the window": 0, - "on the final pass": 24 * time.Hour, + // Past its window, but well inside the absolute deadline: this is + // about cancellation, not about a loss that has run out of time. + "on the final pass": time.Hour, } { t.Run(name, func(t *testing.T) { ledger := newTestLedger(t) diff --git a/internal/connector/round4_test.go b/internal/connector/round4_test.go index a96ec82f7..5bca8b681 100644 --- a/internal/connector/round4_test.go +++ b/internal/connector/round4_test.go @@ -153,10 +153,12 @@ func TestQueueCallbacksMayTouchTheQueue(t *testing.T) { queue, err := NewQueue(1, 4) require.NoError(t, err) var depths []int + // No timeout: a callback that takes from the queue must actually get the + // id the offer is delivering, not time out and call that success. queue.OnWarn = func(int) { - ctxTake, stop := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer stop() - _, _ = queue.Take(ctxTake) + id, err := queue.Take(context.Background()) + assert.NoError(t, err) + assert.Equal(t, int64(1), id) depths = append(depths, queue.Depth()) } diff --git a/internal/connector/round9_test.go b/internal/connector/round9_test.go index 018eb8a75..ebcb530d4 100644 --- a/internal/connector/round9_test.go +++ b/internal/connector/round9_test.go @@ -226,3 +226,30 @@ func TestASecondRunStartsWithCleanRunState(t *testing.T) { cancel() awaitReturn(t, done, "Run should return on shutdown") } + +// A loss still in the queue when a run ends is walked by the next one: the +// mark that says "a worker has this" belongs to the pool, and goes with it. +func TestALossQueuedAtShutdownIsWalkedByTheNextRun(t *testing.T) { + polls := &countingPolls{} + intake, ledger, _ := newTestIntake(t, polls, nil) + intake.opts.RepairInterval = time.Hour + intake.repairSweep = time.Hour + + first, cancelFirst := context.WithCancel(context.Background()) + intake.startRepairWorkers(first) + loss, err := ledger.RecordLoss(first, []int64{17099838509}, intake.now().Add(-time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + cancelFirst() + intake.repairs.Wait() + intake.startRepair(loss) // queued, with no worker left to take it + intake.releaseRepairWorkers() + + second, cancelSecond := context.WithCancel(context.Background()) + defer cancelSecond() + intake.startRepairWorkers(second) + intake.startRepair(loss) + require.Eventually(t, func() bool { return polls.calls.Load() > 0 }, 5*time.Second, 5*time.Millisecond, + "the next run walks it") + cancelSecond() + intake.repairs.Wait() +} diff --git a/internal/connector/shared_lines_test.go b/internal/connector/shared_lines_test.go new file mode 100644 index 000000000..efe0dec94 --- /dev/null +++ b/internal/connector/shared_lines_test.go @@ -0,0 +1,83 @@ +package connector + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/connector/admission" + "github.com/basecamp/basecamp-cli/internal/connector/ndjson" +) + +// A sink that is safe for concurrent use but takes three bytes at a time, so +// two producers that do not share a lock tear each other's lines. It is a +// value type, which is the case the writer registry cannot settle by itself. +type valueSink struct { + mu *sync.Mutex + buf *bytes.Buffer +} + +func (s valueSink) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + if len(p) > 3 { + p = p[:3] + } + return s.buf.Write(p) +} + +// Intake's pointer lines and admission's verdict lines share one stdout, so +// the composition hands them one writer. +func TestIntakeAndAdmissionShareOneLineWriter(t *testing.T) { + sink := valueSink{mu: &sync.Mutex{}, buf: &bytes.Buffer{}} + testSink = sink + lines := ndjson.NewWriter(sink) + + intake, _, _ := newTestIntakeLines(t, lines) + verdicts := &lineWriterFor{lines: lines} + + ctx := context.Background() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for id := int64(1); id <= 200; id++ { + require.NoError(t, intake.ingest(ctx, testEvent(id), LanePoll)) + } + }() + go func() { + defer wg.Done() + for id := int64(1); id <= 200; id++ { + require.NoError(t, verdicts.write(id)) + } + }() + wg.Wait() + + scanner := bufio.NewScanner(bytes.NewReader(sink.buf.Bytes())) + lineCount := 0 + for scanner.Scan() { + var v map[string]any + require.NoError(t, json.Unmarshal(scanner.Bytes(), &v), "torn line %d: %q", lineCount, scanner.Text()) + lineCount++ + } + assert.Equal(t, 400, lineCount) +} + +// lineWriterFor stands in for admission's verdict lines over the same writer. +type lineWriterFor struct{ lines *ndjson.Writer } + +func (l *lineWriterFor) write(id int64) error { + return l.lines.WriteLine(admission.Line{Type: "event", EventID: id, EventType: "comment.created"}) +} + +var testSink valueSink + +// linesSink is the sink the test's writer was built for, so a mutation can +// build a second writer over the same one. +func linesSink(*ndjson.Writer) valueSink { return testSink } From e8a0d4f84f2adcde2ca166adbe7843424f30565a Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 03:12:41 +0200 Subject: [PATCH 38/49] Report an absolute-deadline close for what it was, and unlatch the edge drain on a panic A loss the poll lane had already served closed at its absolute deadline with an error line saying its events were unrecovered. It says what happened now, and keeps the error for ids nobody could serve. The queue's edge drain cleared its latch at the end of the loop, so a callback that panicked and was recovered above would have left it latched and every later edge undelivered. Nothing recovers today; a defer costs nothing and does not depend on that staying true. --- internal/connector/loss_schedule_test.go | 25 ++++++++++++++++++++++++ internal/connector/queue.go | 9 +++++++-- internal/connector/repair.go | 9 +++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/internal/connector/loss_schedule_test.go b/internal/connector/loss_schedule_test.go index 940f291d0..9bc2322cb 100644 --- a/internal/connector/loss_schedule_test.go +++ b/internal/connector/loss_schedule_test.go @@ -1,7 +1,9 @@ package connector import ( + "bytes" "context" + "log/slog" "testing" "time" @@ -163,3 +165,26 @@ func TestALossClosesAtItsAbsoluteDeadline(t *testing.T) { assert.Equal(t, []int64{17099838509}, unrecovered, "and its ids are reported, not forgotten") assert.False(t, clock.at.Before(loss.DetectedAt.Add(maxLossLifetime)), "not before the absolute deadline") } + +// A loss that was reconciled before its absolute deadline closes quietly: the +// error line is for ids nobody could serve, not for a tidy close. +func TestAReconciledLossPastItsDeadlineDoesNotReportUnrecovered(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, clock.at.Add(-48*time.Hour), time.Minute, eventfeed.Filters{}) + require.NoError(t, err) + // Served in the meantime, by the ordinary poll lane. + _, err = ledger.RecordSeen(ctx, testEvent(17099838509), LanePoll) + require.NoError(t, err) + + var logs bytes.Buffer + walker, _ := newTestWalker(t, ledger, &scriptedPolls{}, clock) + walker.log = slog.New(slog.NewTextHandler(&logs, nil)) + require.NoError(t, walker.reconcile(ctx, loss)) + + assert.NotContains(t, logs.String(), "unrecovered") + unrecovered, err := ledger.UnrecoveredIDs(ctx) + require.NoError(t, err) + assert.Empty(t, unrecovered) +} diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 66887a43a..18e45a729 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -189,6 +189,13 @@ func (q *Queue) deliver() { return } q.delivering = true + // Cleared with a defer: a callback that panics and is recovered above + // would otherwise leave the drain latched and every later edge + // undelivered. + defer func() { + q.delivering = false + q.edges.Unlock() + }() for len(q.pending) > 0 { edge := q.pending[0] q.pending = q.pending[1:] @@ -198,8 +205,6 @@ func (q *Queue) deliver() { } q.edges.Lock() } - q.delivering = false - q.edges.Unlock() } type queueEdge struct { diff --git a/internal/connector/repair.go b/internal/connector/repair.go index d3db5c5b3..7a82f8d39 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -66,8 +66,13 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { if err != nil { return err } - w.log.Error("a buffer overflow reached its absolute deadline; what is still missing is unrecovered", - "loss_id", loss.ID, "unrecovered", unrecovered, "age", w.now().Sub(loss.DetectedAt)) + if unrecovered > 0 { + w.log.Error("a buffer overflow reached its absolute deadline; what is still missing is unrecovered", + "loss_id", loss.ID, "unrecovered", unrecovered, "age", w.now().Sub(loss.DetectedAt)) + } else { + w.log.Info("a buffer overflow was reconciled before its absolute deadline", + "loss_id", loss.ID, "age", w.now().Sub(loss.DetectedAt)) + } return nil } if !w.now().Before(loss.DeadlineAt) { From 8e1e98c71432a611659223a282f58718a0d4ca14 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 03:23:22 +0200 Subject: [PATCH 39/49] Pair the edge drain's unlock with the lock it takes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cleanup that unlatches the drain runs holding the mutex, but the loop released it around each callback, so a callback that panicked unwound with the lock not held and the deferred Unlock aborted the process — the very panic it was meant to survive, now fatal and with the original panic lost. Retake the lock on the way out of every callback, returned or panicked, so the loop and its cleanup always hold it. --- internal/connector/queue.go | 17 ++++++++++++----- internal/connector/queue_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 18e45a729..1d06e3fc5 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -191,7 +191,8 @@ func (q *Queue) deliver() { q.delivering = true // Cleared with a defer: a callback that panics and is recovered above // would otherwise leave the drain latched and every later edge - // undelivered. + // undelivered. The loop below holds the lock whenever it is not inside a + // callback, so this runs holding it on every path. defer func() { q.delivering = false q.edges.Unlock() @@ -200,10 +201,16 @@ func (q *Queue) deliver() { edge := q.pending[0] q.pending = q.pending[1:] q.edges.Unlock() - if edge.fire != nil { - edge.fire(edge.depth) - } - q.edges.Lock() + // The lock is retaken by a defer, not after the call: a callback that + // panics unwinds through here, and the cleanup above unlocks. Retaking + // it on the way out of every callback — returned or panicked — is what + // makes that unlock the one that pairs with this Lock. + func() { + defer q.edges.Lock() + if edge.fire != nil { + edge.fire(edge.depth) + } + }() } } diff --git a/internal/connector/queue_test.go b/internal/connector/queue_test.go index 697e6fb61..5d45e7d2f 100644 --- a/internal/connector/queue_test.go +++ b/internal/connector/queue_test.go @@ -148,3 +148,35 @@ func TestACrossingIsNotLostToAConcurrentTake(t *testing.T) { assert.Equal(t, 1, recovers, "and the take brought it back") assert.Zero(t, queue.Depth()) } + +func TestAPanickingBacklogCallbackLeavesTheQueueUsable(t *testing.T) { + queue, err := NewQueue(1, 4) + require.NoError(t, err) + + var recoveries int + queue.OnWarn = func(int) { panic("a warning callback panics") } + queue.OnRecover = func(int) { recoveries++ } + + // The panic belongs to the callback. It must reach the caller, who can + // recover it, and not take the process — or the drain — with it. + panicked := func() (caught bool) { + defer func() { caught = recover() != nil }() + require.NoError(t, queue.Offer(context.Background(), 1)) + return false + }() + require.True(t, panicked, "the callback's panic should reach the caller") + + // The drain is unlatched and the queue still works: a second id goes in, + // both come out, and the recovery edge is delivered. + require.NoError(t, queue.Offer(context.Background(), 2)) + assert.Equal(t, 2, queue.Depth()) + + first, err := queue.Take(context.Background()) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + second, err := queue.Take(context.Background()) + require.NoError(t, err) + assert.Equal(t, int64(2), second) + assert.Equal(t, 0, queue.Depth()) + assert.Equal(t, 1, recoveries, "the drain should still be delivering edges") +} From a10a38c2d93f3a520e5ea09aa8b5cf0aeca1e5d1 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 03:30:10 +0200 Subject: [PATCH 40/49] Make the instance lock a lock, and the pause edges ordered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the same review. The instance lock accepted whatever directory it was handed: MkdirAll applies 0700 only when it creates the directory, so an existing group-writable one was used as found, and another local user could swap the lock pathname between two connectors' opens. Each would then hold a different inode and neither would exclude the other — one agent's mentions dispatched twice, which is the whole reason the lock exists. It now goes through the same private-path check card 16 landed for connect.json, exported from that package rather than written a second time: the directory and everything above it must be this user's alone, the lock file is opened without following symlinks and re-checked with the lock held, and a lock that cannot be established is a refusal rather than a connector running without one. Pause and resume were fired by the waiting goroutines themselves, so a resume that began first could finish after a later pause and leave an operator reading "resumed" while the feed was not being read. They are now decided under the same lock as the warning edges and delivered by the same ordered drain. And the package said a busy dispatcher can never stall the socket, which the bounded backlog contradicts on purpose. Say what is true: absorbed up to the pause threshold, and backpressure past it. --- internal/connector/doc.go | 10 ++- internal/connector/intake.go | 6 +- internal/connector/lock.go | 33 ++++---- internal/connector/lock_test.go | 61 ++++++++++++++ internal/connector/queue.go | 57 +++++++++++--- internal/connector/queue_test.go | 69 ++++++++++++++++ internal/connector/setup/private_state.go | 96 +++++++++++++++++++++++ 7 files changed, 301 insertions(+), 31 deletions(-) create mode 100644 internal/connector/setup/private_state.go diff --git a/internal/connector/doc.go b/internal/connector/doc.go index 356fd95f2..127aa1927 100644 --- a/internal/connector/doc.go +++ b/internal/connector/doc.go @@ -7,8 +7,14 @@ // Feed rows are pointers — id, type, bucket, creator, recording — and nothing // else. No title, no body, no URL, no names. Intake writes that pointer to the // ledger if the id is new and hands the id to a queue. That is the whole of -// the work on the feed's delivery path, so a slow admission or a busy -// dispatcher can never stall the socket. +// the work on the feed's delivery path: a slow admission or a busy dispatcher +// is absorbed by the backlog rather than felt by the socket. +// +// Absorbed, not unbounded. The backlog is a bounded queue, and at its pause +// threshold an offer waits for room — which is intake deliberately stopping +// its read of the feed, so the backlog cannot grow until it is the process's +// memory that fails. The decoupling holds up to that depth; past it, +// backpressure is the design, and the pause is reported rather than hidden. // // Deciding whether an event deserves an agent's attention is admission's job // and it costs a read per event it cares about. Intake does none of it. It diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 191990ff7..c983a456a 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -112,8 +112,10 @@ func LiveOptions(live *eventfeed.Live) Options { // Intake is the feed's delivery path: write the pointer, hand over the id. // // Everything else — reading the recording, judging it, dispatching it — is -// downstream of the queue, so a slow admission or a busy dispatcher can never -// stall the socket. +// downstream of the queue, so a slow admission or a busy dispatcher is +// absorbed by the backlog rather than felt by the socket — until the backlog +// reaches its pause threshold, where intake stops reading the feed on purpose +// rather than let the queue grow without bound. type Intake struct { opts Options ledger *Ledger diff --git a/internal/connector/lock.go b/internal/connector/lock.go index 2d417818b..19821d240 100644 --- a/internal/connector/lock.go +++ b/internal/connector/lock.go @@ -9,7 +9,7 @@ import ( "strconv" "time" - "github.com/gofrs/flock" + "github.com/basecamp/basecamp-cli/internal/connector/setup" ) // ErrAlreadyRunning reports a second connector for the same agent. @@ -28,8 +28,8 @@ var ErrAlreadyRunning = errors.New("connector: another connector already holds t // stale-lock reaping to get wrong. The metadata written beside it is // diagnostic only: the lock is the lock. type InstanceLock struct { - flock *flock.Flock - path string + unlock func() error + path string } // instanceHolder is what a running connector writes beside its lock so the @@ -51,18 +51,23 @@ func AcquireInstanceLock(dir, accountID string, agentPersonID int64, now time.Ti // "2914079" are one account and must meet one lock, or the refusal of a // second connector is a matter of how the id was typed. accountID = strconv.FormatUint(account, 10) - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, fmt.Errorf("connector: create state directory: %w", err) - } path := filepath.Join(dir, "instance-"+accountID+"-"+strconv.FormatInt(agentPersonID, 10)+".lock") - lock := flock.New(path) - held, err := lock.TryLock() - if err != nil { - return nil, fmt.Errorf("connector: take the instance lock: %w", err) - } - if !held { + // The lock is only a lock if nobody else can reach it. A directory that + // merely EXISTS at 0700 is not enough — MkdirAll leaves a group-writable + // one exactly as it found it — so the directory, everything above it and + // the lock file itself are vetted here, by the same check the connector's + // trust file gets. Someone who can write the directory can point two + // connectors at two inodes, and then neither excludes the other: one + // agent's mentions dispatched twice, which is the failure this lock + // exists to prevent. + unlock, err := setup.TryLockPrivate(path) + switch { + case errors.Is(err, setup.ErrLockHeld): return nil, fmt.Errorf("%w: %s", ErrAlreadyRunning, describeHolder(path)) + case err != nil: + // No lock means no connector. There is no degraded mode here. + return nil, fmt.Errorf("connector: take the instance lock: %w", err) } holder, err := json.Marshal(instanceHolder{ @@ -77,13 +82,13 @@ func AcquireInstanceLock(dir, accountID string, agentPersonID int64, now time.Ti _ = os.WriteFile(path+".json", append(holder, '\n'), 0o600) } - return &InstanceLock{flock: lock, path: path}, nil + return &InstanceLock{unlock: unlock, path: path}, nil } // Release drops the lock. func (l *InstanceLock) Release() error { _ = os.Remove(l.path + ".json") - return l.flock.Unlock() + return l.unlock() } // Path is the lock file, for diagnostics. diff --git a/internal/connector/lock_test.go b/internal/connector/lock_test.go index a9de38cfd..ff3852b93 100644 --- a/internal/connector/lock_test.go +++ b/internal/connector/lock_test.go @@ -1,11 +1,16 @@ package connector import ( + "errors" + "os" + "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/basecamp/basecamp-cli/internal/connector/setup" ) // Two connectors on one agent identity dispatch every mention twice. The Ruby @@ -62,3 +67,59 @@ func TestLockRefusesAnIncompleteIdentity(t *testing.T) { _, err = AcquireInstanceLock(dir, "2914079", 0, time.Now()) assert.Error(t, err) } + +// A lock in a directory other local users can write is not a lock: they can +// replace the file between two connectors' opens, and each then holds an +// inode the other never sees. +func TestTheLockRefusesADirectoryOthersCanWrite(t *testing.T) { + dir := filepath.Join(t.TempDir(), "state") + require.NoError(t, os.Mkdir(dir, 0o700)) + require.NoError(t, os.Chmod(dir, 0o770)) // after the umask, not through it + + _, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) + assert.NotErrorIs(t, err, ErrAlreadyRunning, "a directory this connector will not use is not a second connector") +} + +// A lock path that is a symlink locks whatever it points at, so two +// connectors pointed at two symlinks exclude nobody. +func TestTheLockRefusesASymlinkedLockPath(t *testing.T) { + dir := t.TempDir() + elsewhere := filepath.Join(t.TempDir(), "elsewhere.lock") + require.NoError(t, os.WriteFile(elsewhere, nil, 0o600)) + require.NoError(t, os.Symlink(elsewhere, filepath.Join(dir, "instance-2914079-52007412.lock"))) + + _, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} + +// The directory is created when it is missing, and created owner-only. +func TestTheLockCreatesItsDirectoryOwnerOnly(t *testing.T) { + dir := filepath.Join(t.TempDir(), "state") + + lock, err := AcquireInstanceLock(dir, "2914079", 52007412, time.Now()) + require.NoError(t, err) + t.Cleanup(func() { _ = lock.Release() }) + + info, err := os.Stat(dir) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o700), info.Mode().Perm()) + file, err := os.Stat(lock.Path()) + require.NoError(t, err) + assert.Zero(t, file.Mode().Perm()&0o077, "the lock file is the owner's alone") +} + +// A lock that cannot be taken is a refusal, never a connector running +// unlocked. +func TestTheLockRefusesWhenItCannotBeEstablished(t *testing.T) { + missing := filepath.Join(t.TempDir(), "absent", "state") + + _, err := AcquireInstanceLock(missing, "2914079", 52007412, time.Now()) + + require.Error(t, err) + assert.True(t, errors.Is(err, setup.ErrNotPrivate) || errors.Is(err, os.ErrNotExist), "got %v", err) +} diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 1d06e3fc5..8cdef4778 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -4,7 +4,6 @@ import ( "context" "errors" "sync" - "sync/atomic" ) // Backlog thresholds. Intake is the only work on the feed's delivery path, so @@ -23,9 +22,11 @@ const ( // Queue is the seam between intake and admission: intake writes a pointer and // hands over an id, admission reads it when it gets there. Two queues with -// visible depth rather than one pipeline, so a busy dispatcher can never stall -// the socket — and so the place where work is piling up is the place the depth -// is showing. +// visible depth rather than one pipeline, so a busy dispatcher is absorbed up +// to the pause threshold instead of being felt on the socket — and so the +// place where work is piling up is the place the depth is showing. At that +// threshold the offer waits: the backlog is bounded, and the pause is the +// backpressure reaching the feed, reported rather than hidden. type Queue struct { ids chan int64 warnAt int @@ -47,8 +48,13 @@ type Queue struct { delivering bool // waiting counts offers blocked for room. Intake and every open repair // walk offer concurrently, so a single flag would be cleared by the first - // waiter to resume while another — perhaps the feed — still waits. - waiting atomic.Int32 + // waiter to resume while another — perhaps the feed — still waits. It is + // held under edges with the depth, because its callbacks go through the + // same ordered drain: told out of order, a resume that started first but + // finished last leaves the operator reading "resumed" while the feed is + // paused. + waiting int + paused bool // OnWarn fires when the depth first crosses the warning threshold, and // OnRecover when it falls back below. Both are optional. @@ -92,13 +98,11 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { default: } - if q.waiting.Add(1) == 1 && q.OnPause != nil { - q.OnPause(q.Depth()) - } + q.stageWait(1) + q.deliver() defer func() { - if q.waiting.Add(-1) == 0 && q.OnResume != nil { - q.OnResume(q.Depth()) - } + q.stageWait(-1) + q.deliver() }() select { @@ -140,7 +144,11 @@ func (q *Queue) Depth() int { // Paused reports whether an offer is currently waiting for room — which is to // say whether the feed is being consumed. -func (q *Queue) Paused() bool { return q.waiting.Load() > 0 } +func (q *Queue) Paused() bool { + q.edges.Lock() + defer q.edges.Unlock() + return q.waiting > 0 +} // afterOp is a test seam: it runs between a channel operation and whatever // follows it, which is where a crossing used to be lost. @@ -178,6 +186,29 @@ func (q *Queue) stage(delta int) { } } +// stageWait records an offer starting or finishing its wait for room and the +// pause edge that crosses, without delivering it. +// +// Pause and resume are transitions of the same state as the warning edges, so +// they are decided under the same lock and delivered by the same drain, in the +// order they were decided. Fired from the waiting goroutines themselves they +// could interleave: a resume delayed in its callback could land after a later +// pause, and an observer would be left believing the feed is being read when +// it is not. +func (q *Queue) stageWait(delta int) { + q.edges.Lock() + defer q.edges.Unlock() + q.waiting += delta + switch { + case q.waiting > 0 && !q.paused: + q.paused = true + q.pending = append(q.pending, queueEdge{fire: q.OnPause, depth: q.depth}) + case q.waiting == 0 && q.paused: + q.paused = false + q.pending = append(q.pending, queueEdge{fire: q.OnResume, depth: q.depth}) + } +} + // deliver hands the staged edges to their callbacks, in the order they were // decided, one goroutine at a time and with the lock released around each: a // callback may use the queue, and its own edge is delivered by the drain diff --git a/internal/connector/queue_test.go b/internal/connector/queue_test.go index 5d45e7d2f..d99c0461f 100644 --- a/internal/connector/queue_test.go +++ b/internal/connector/queue_test.go @@ -180,3 +180,72 @@ func TestAPanickingBacklogCallbackLeavesTheQueueUsable(t *testing.T) { assert.Equal(t, 0, queue.Depth()) assert.Equal(t, 1, recoveries, "the drain should still be delivering edges") } + +// An operator watching the queue must never be told the feed resumed while it +// is paused. The transitions are delivered in the order they happened, even +// when an earlier callback is slower than a later transition. +func TestPauseAndResumeAreObservedInTheOrderTheyHappened(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + + var mu sync.Mutex + var seen []string + record := func(what string) { + mu.Lock() + defer mu.Unlock() + seen = append(seen, what) + } + observed := func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), seen...) + } + resumeStarted, holdResume := make(chan struct{}), make(chan struct{}) + queue.OnPause = func(int) { record("paused") } + var firstResume sync.Once + queue.OnResume = func(int) { + held := false + firstResume.Do(func() { held = true; close(resumeStarted) }) + if held { + <-holdResume + } + record("resumed") + } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + + var waiters sync.WaitGroup + waiters.Add(1) + go func() { + defer waiters.Done() + assert.NoError(t, queue.Offer(ctx, 2)) // waits for room, then resumes + }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + + first, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + <-resumeStarted // the resume has begun and is not finished + + waiters.Add(1) + go func() { + defer waiters.Done() + assert.NoError(t, queue.Offer(ctx, 3)) // pauses again, mid-resume + }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + + close(holdResume) + require.Eventually(t, func() bool { return len(observed()) == 3 }, time.Second, time.Millisecond) + assert.Equal(t, []string{"paused", "resumed", "paused"}, observed(), + "a resume that started first must not be reported after the pause that followed it") + + second, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2), second) + waiters.Wait() + third, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(3), third) + assert.False(t, queue.Paused()) +} diff --git a/internal/connector/setup/private_state.go b/internal/connector/setup/private_state.go new file mode 100644 index 000000000..daa9d995a --- /dev/null +++ b/internal/connector/setup/private_state.go @@ -0,0 +1,96 @@ +package setup + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/gofrs/flock" +) + +// ErrLockHeld reports a lock another process already holds. +var ErrLockHeld = errors.New("the lock is held by another process") + +// EnsurePrivateDir creates dir owner-only when it is missing, and refuses it +// when dir — or any directory above it — could be changed by someone else. +// +// It is the same check connect.json gets, exported for the state the running +// connector keeps beside it: the instance lock and the ledger live under the +// same config root, and a directory another local user can write is a +// directory where a lock file can be swapped for one that excludes nobody. +// +// Only the last component is created. Everything above it must already exist +// and already be trustworthy, which is what makes the refusal meaningful. +func EnsurePrivateDir(dir string) error { + if dir == "" { + return fmt.Errorf("%w: no directory given", ErrNotPrivate) + } + abs, err := filepath.Abs(dir) + if err != nil { + return err + } + if err := checkAncestors(filepath.Dir(abs)); err != nil { + return err + } + switch _, err := os.Lstat(abs); { + case errors.Is(err, os.ErrNotExist): + if err := os.Mkdir(abs, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("create %s: %w", abs, err) + } + case err != nil: + return fmt.Errorf("inspect %s: %w", abs, err) + } + return checkPrivateDir(abs) +} + +// TryLockPrivate takes an exclusive lock on path without waiting, having first +// established that the lock is a lock: the directory holding it is this user's +// alone, and the file itself is this user's own regular file rather than a +// symlink or something another user left there. A lock on a file two processes +// can be pointed at different inodes excludes nobody. +// +// It reports ErrLockHeld when another process holds it, which is an answer +// rather than a failure. Every other error means the lock could not be +// established, and the caller must refuse rather than carry on unlocked. +func TryLockPrivate(path string) (unlock func() error, err error) { + if err := EnsurePrivateDir(filepath.Dir(path)); err != nil { + return nil, err + } + if err := checkPrivateLockFile(path); err != nil { + if errors.Is(err, ErrNotPrivate) { + return nil, err + } + return nil, fmt.Errorf("%w: %s: %w", ErrLockUnavailable, path, err) + } + lock := flock.New(path, flock.SetPermissions(0o600)) + held, err := lock.TryLock() + if err != nil { + return nil, fmt.Errorf("%w: %s: %w", ErrLockUnavailable, path, err) + } + if !held { + return nil, ErrLockHeld + } + // Re-checked with the lock held, and this time the file must be there: + // a lock file that has been unlinked or replaced since the open excludes + // nobody, whatever this process is holding. + if err := checkHeldLockFile(path); err != nil { + _ = lock.Unlock() + return nil, err + } + return lock.Unlock, nil +} + +// checkHeldLockFile is checkPrivateLockFile for a lock already taken, where a +// missing file is itself the tampering it is looking for. +func checkHeldLockFile(path string) error { + f, err := openNoFollow(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%w: %s was removed while it was being locked", ErrNotPrivate, path) + } + return fmt.Errorf("%w: %s: %w", ErrLockUnavailable, path, err) + } + defer f.Close() + return checkPrivateFile(f, path) +} From 20eba386d70ab22c8baa850176f7d12b3025fe52 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 03:46:20 +0200 Subject: [PATCH 41/49] Validate the ledger this process opens, and let no committed event stop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings. The ledger holds feed positions, which resume the account's feed, so it is a credential file — but it was vetted by name: os.Stat on the immediate directory only, following an existing symlink, with nothing said about the ancestors. A group-writable ancestor is enough to rename the checked 0700 directory away and put another in its place between the check and SQLite's open. It now goes through the same private-path check as the instance lock and the trust file: every ancestor owned by this user and unwritable by others, the file opened without following symlinks and inspected through that descriptor, and a file another user can so much as read refused. Its own extra rule stays, because SQLite writes -wal and -shm beside it: the directory must be 0700, not merely unwritable. And a handover that failed after the commit waited for a restart. That is a long wait for a connector that runs for weeks, and the repair walk is where it bites: the commit resolves the loss's missing id, so the reconciliation that follows sees nothing missing and closes, while the event was recorded, never judged and never mentioned again. Every commit already goes through one handover path; now a failure on it is remembered there, and the sweep that re-offers open losses re-offers those ids too. A crash still loses the list, and loses nothing with it: the next start offers every record still in seen, and clears what it has offered. --- internal/connector/intake.go | 68 +++++++++++++++++++- internal/connector/ledger.go | 77 +++++++++-------------- internal/connector/ledger_test.go | 47 ++++++++++++++ internal/connector/requeue_test.go | 60 ++++++++++++++++++ internal/connector/setup/private_state.go | 54 ++++++++++++++++ 5 files changed, 258 insertions(+), 48 deletions(-) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index c983a456a..d07d7d7f5 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -172,6 +172,12 @@ type Intake struct { // the feed somewhere unsafe. abortErr error + // stranded holds ids this process committed to the ledger and then failed + // to hand over. They are the one thing the ledger's own dedupe would hide: + // the row is there, so every retry of the event is suppressed as a + // duplicate, and without this the id would wait for a restart. + stranded map[int64]struct{} + repairs sync.WaitGroup repairQueue chan Loss // repairQueueSize and repairSweep override the pool's defaults in tests. @@ -529,10 +535,15 @@ func (in *Intake) ingest(ctx context.Context, event eventfeed.Event, lane Lane) return nil } + // Past this line the row is committed, so the event is invisible to every + // later delivery of itself: it is this process's to finish handing over, + // or to remember that it did not. Both failures below are that. if err := in.pointer.write(event, lane); err != nil { + in.strand(event.ID) return err } if err := in.queue.Offer(ctx, event.ID); err != nil { + in.strand(event.ID) return err } // Only once the id is handed over: the reconnect cancels the connection @@ -768,6 +779,53 @@ func entryClassOf(resumeURL string) EntryClass { } } +// strand remembers an id the ledger has but the queue does not. +// +// Every commit goes through ingest, and every handover failure after a commit +// goes through here, so there is one place an event can be stranded and one +// place that answers for it. A crash loses the list, and nothing is lost with +// it: the next start offers every record still in seen. +func (in *Intake) strand(id int64) { + in.mu.Lock() + defer in.mu.Unlock() + if in.stranded == nil { + in.stranded = map[int64]struct{}{} + } + in.stranded[id] = struct{}{} +} + +// sweepStranded offers the ids a failed handover left behind. +// +// A restart is not an answer for a connector that runs for weeks, and a +// repair walk is the case that makes it urgent: its failure closes nothing +// downstream, the id's loss row is already resolved by the commit, and the +// reconciliation that follows sees nothing missing. The event would be +// recorded, never judged, and never mentioned again. +// +// It offers and only then forgets: an offer refused by a canceled context +// leaves the id stranded for the next sweep, or for the next start. +func (in *Intake) sweepStranded(ctx context.Context) { + in.mu.Lock() + ids := make([]int64, 0, len(in.stranded)) + for id := range in.stranded { + ids = append(ids, id) + } + in.mu.Unlock() + if len(ids) == 0 { + return + } + slices.Sort(ids) // oldest first, as the feed served them + for _, id := range ids { + if err := in.queue.Offer(ctx, id); err != nil { + in.log.Warn("an event the ledger holds could not be handed over; it stays for the next sweep", "event_id", id, "error", err) + return + } + in.mu.Lock() + delete(in.stranded, id) + in.mu.Unlock() + } +} + // requeueSeen hands every record still in seen to the queue. // // The ledger row is written before the pointer line and before the hand-off, @@ -799,6 +857,11 @@ func (in *Intake) requeueSeen(ctx context.Context) error { after = record.ID } if len(records) < requeueBatch { + // Everything in seen has just been offered, the ids a previous + // run stranded included. + in.mu.Lock() + in.stranded = nil + in.mu.Unlock() return nil } } @@ -891,7 +954,9 @@ func (in *Intake) startRepairWorkers(ctx context.Context) { go in.sweepLosses(ctx) } -// sweepLosses offers every open loss nothing is already walking. +// sweepLosses is the periodic repair of both things that can be left behind: +// an event committed but never handed over, and an open loss nothing is +// walking. // // The queue is bounded, so an overloaded connector can turn one away; and a // walk can end early, leaving its loss open. Neither may leave a loss with @@ -911,6 +976,7 @@ func (in *Intake) sweepLosses(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: + in.sweepStranded(ctx) losses, err := in.ledger.OpenLosses(ctx) if err != nil { in.log.Warn("could not read the open losses", "error", err) diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index fa38bfa09..d5fa1eb18 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -7,12 +7,13 @@ import ( "fmt" "os" "path/filepath" - "runtime" "strings" "time" "modernc.org/sqlite" // database/sql driver "sqlite", pure Go: no cgo on any of the five release targets. sqlite3 "modernc.org/sqlite/lib" + + "github.com/basecamp/basecamp-cli/internal/connector/setup" ) // isInMemory reports a path SQLite would read as its in-memory database @@ -158,60 +159,42 @@ func isBusy(err error) bool { // securePath makes the ledger private or refuses it. // // The ledger holds feed positions — signed tokens that resume the account's -// feed — and every event's metadata. Its directory must be 0700 and the file -// 0600. A directory or file that already exists with looser permissions is -// refused rather than tightened: something else chose those permissions, and -// silently changing them could break it or hide that the ledger was exposed. +// feed — and every event's metadata, so it is a credential file and is +// treated as one. The check is the same one the connector's trust file and +// instance lock get: every directory on the way must be this user's own and +// unwritable by anyone else, and the ledger itself is opened without +// following symlinks and inspected through that descriptor rather than by +// name. Validating the name would validate whatever the name pointed at when +// it was asked, which is not necessarily what SQLite then opens. +// +// A file or directory that already exists with looser permissions is refused +// rather than tightened: something else chose those permissions, and silently +// changing them could break it or hide that the ledger was exposed. +// +// The check cannot be made on a platform without POSIX owners and modes, and +// a ledger whose privacy cannot be established is refused there rather than +// opened — the same way setup refuses to write a trust file it cannot vouch +// for. func securePath(path string) error { + if err := setup.EnsurePrivateFile(path); err != nil { + return fmt.Errorf("connector: secure the ledger: %w", err) + } + // One rule of the ledger's own, beyond what a trust file needs: its + // directory must be 0700, not merely unwritable by others. SQLite writes + // -wal and -shm beside the database, and a directory other users can read + // is one whose entries they can list. The directory is already known not + // to be a symlink, so Lstat here inspects the directory itself. dir := filepath.Dir(path) - switch info, err := os.Stat(dir); { - case os.IsNotExist(err): - if err := os.MkdirAll(dir, 0o700); err != nil { - return fmt.Errorf("connector: create ledger directory: %w", err) - } - if err := os.Chmod(dir, 0o700); err != nil { //nolint:gosec // a directory needs its search bit; 0700 is owner-only - return fmt.Errorf("connector: secure ledger directory: %w", err) - } - case err != nil: + info, err := os.Lstat(dir) + if err != nil { return fmt.Errorf("connector: inspect ledger directory: %w", err) - case !info.IsDir(): - return fmt.Errorf("connector: ledger directory %s is not a directory", dir) - case looserThan(info.Mode(), 0o700): - return fmt.Errorf("connector: ledger directory %s is readable by other users (mode %04o); it must be 0700", dir, info.Mode().Perm()) } - - switch f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600); { - case err == nil: - if err := f.Close(); err != nil { - return fmt.Errorf("connector: create ledger: %w", err) - } - if err := os.Chmod(path, 0o600); err != nil { - return fmt.Errorf("connector: secure ledger: %w", err) - } - return nil - case !os.IsExist(err): - return fmt.Errorf("connector: create ledger: %w", err) - } - // It exists — perhaps created a moment ago by another process opening the - // same fresh ledger. Its permissions decide, not who created it. - switch info, err := os.Stat(path); { - case err != nil: - return fmt.Errorf("connector: inspect ledger: %w", err) - case looserThan(info.Mode(), 0o600): - return fmt.Errorf("connector: ledger %s is readable by other users (mode %04o); it must be 0600", path, info.Mode().Perm()) + if perm := info.Mode().Perm(); perm&0o077 != 0 { + return fmt.Errorf("connector: ledger directory %s is readable by other users (mode %04o); it must be 0700", dir, perm) } return nil } -// looserThan reports permission bits beyond limit. Windows has no POSIX bits -// to speak of; access there is the ACL of the user's profile directory. -func looserThan(mode os.FileMode, limit os.FileMode) bool { - if runtime.GOOS == "windows" { - return false - } - return mode.Perm()&^limit != 0 -} - // Close releases the ledger's handle. func (l *Ledger) Close() error { return l.db.Close() } diff --git a/internal/connector/ledger_test.go b/internal/connector/ledger_test.go index c59203492..0c94ceb8f 100644 --- a/internal/connector/ledger_test.go +++ b/internal/connector/ledger_test.go @@ -3,6 +3,7 @@ package connector import ( "context" "encoding/json" + "os" "path/filepath" "testing" "time" @@ -11,6 +12,8 @@ import ( "github.com/stretchr/testify/require" "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" + + "github.com/basecamp/basecamp-cli/internal/connector/setup" ) func newTestLedger(t *testing.T) *Ledger { @@ -191,3 +194,47 @@ func TestCheckpointsAreKeyedByFilterDigest(t *testing.T) { require.NoError(t, err) assert.False(t, ok, "a filter change re-enters under its own lineage") } + +// The ledger holds feed positions, which resume the account's feed. A path +// that can be redirected is a ledger that can be read, so what is validated +// is the file that gets opened, not the name that was asked for. +func TestLedgerRefusesASymlinkedPath(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Chmod(dir, 0o700)) + elsewhere := filepath.Join(t.TempDir(), "elsewhere.db") + require.NoError(t, os.WriteFile(elsewhere, nil, 0o600)) + path := filepath.Join(dir, "connector.db") + require.NoError(t, os.Symlink(elsewhere, path)) + + _, err := OpenLedger(path) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} + +func TestLedgerRefusesASymlinkedParent(t *testing.T) { + target := filepath.Join(t.TempDir(), "target") + require.NoError(t, os.Mkdir(target, 0o700)) + link := filepath.Join(t.TempDir(), "state") + require.NoError(t, os.Symlink(target, link)) + + _, err := OpenLedger(filepath.Join(link, "connector.db")) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} + +// A 0700 directory under an ancestor anyone can write is not private: the +// ancestor's owner can rename it away and put their own in its place. +func TestLedgerRefusesALooseAncestor(t *testing.T) { + loose := filepath.Join(t.TempDir(), "loose") + require.NoError(t, os.Mkdir(loose, 0o700)) + require.NoError(t, os.Chmod(loose, 0o777)) // after the umask, not through it + dir := filepath.Join(loose, "state") + require.NoError(t, os.Mkdir(dir, 0o700)) + + _, err := OpenLedger(filepath.Join(dir, "connector.db")) + + require.Error(t, err) + assert.ErrorIs(t, err, setup.ErrNotPrivate) +} diff --git a/internal/connector/requeue_test.go b/internal/connector/requeue_test.go index c3e91d7eb..e7486f6b0 100644 --- a/internal/connector/requeue_test.go +++ b/internal/connector/requeue_test.go @@ -92,3 +92,63 @@ func TestRequeueingStopsOnShutdownAndLeavesTheRest(t *testing.T) { require.NoError(t, err) assert.Len(t, remaining, 3, "nothing is consumed by being queued; the next start sees them all") } + +// The restart is not the only answer. A repair walk is the case that makes it +// urgent: the commit resolves the loss's missing id, so the reconciliation +// that follows sees nothing missing and closes, and a connector that runs for +// weeks would never mention the event again. Every commit goes through one +// handover path, and a failure on it is remembered and retried in process. +func TestAnEventWhoseHandOffFailedIsSweptWithoutARestart(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + intake, _, queue := newTestIntakeOn(t, ledger, failingWriter{}) + + require.Error(t, intake.ingest(ctx, testEvent(17099838500), LaneRepair)) + require.Zero(t, queue.Depth(), "the id never reached the queue") + + // The ledger's dedupe now suppresses the event on every later delivery of + // itself, so nothing but the sweep can hand it over. + fresh, err := ledger.RecordSeen(ctx, testEvent(17099838500), LaneRepair) + require.NoError(t, err) + require.False(t, fresh) + + intake.sweepStranded(ctx) + + require.Equal(t, 1, queue.Depth()) + id, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(17099838500), id) + + // Exactly once: a second sweep has nothing left to offer. + intake.sweepStranded(ctx) + assert.Zero(t, queue.Depth()) +} + +// A sweep that cannot hand an id over leaves it for the next one, and for the +// next start after that. +func TestASweepThatCannotHandOverKeepsTheStrandedID(t *testing.T) { + ctx := context.Background() + intake, _, _ := newTestIntakeOn(t, newTestLedger(t), failingWriter{}) + full, err := NewQueue(1, 1) + require.NoError(t, err) + intake.queue = full + + require.Error(t, intake.ingest(ctx, testEvent(42), LaneLive)) + require.NoError(t, full.Offer(ctx, 99)) // no room for anything else + + stopped, cancel := context.WithCancel(ctx) + cancel() + intake.sweepStranded(stopped) + assert.Equal(t, 1, full.Depth(), "the sweep waited for room and gave up, keeping the id") + + waiting, err := full.Take(ctx) + require.NoError(t, err) + require.Equal(t, int64(99), waiting) + + intake.sweepStranded(ctx) + bounded, stop := context.WithTimeout(ctx, 5*time.Second) + defer stop() + id, err := full.Take(bounded) + require.NoError(t, err, "the id the earlier sweep kept should be handed over now") + assert.Equal(t, int64(42), id) +} diff --git a/internal/connector/setup/private_state.go b/internal/connector/setup/private_state.go index daa9d995a..a4436c0ff 100644 --- a/internal/connector/setup/private_state.go +++ b/internal/connector/setup/private_state.go @@ -94,3 +94,57 @@ func checkHeldLockFile(path string) error { defer f.Close() return checkPrivateFile(f, path) } + +// EnsurePrivateFile creates path owner-only when it is missing, and refuses it +// when it is not this user's own regular file reached through directories only +// this user can change. +// +// It is stricter than the check connect.json gets in one way: a file another +// user can merely READ is refused too. It exists for the connector's ledger, +// which holds the account's feed positions — resumable tokens, so credentials +// — and there a readable file is already the leak. +// +// What is validated is the descriptor, not the name: the file is opened +// without following symlinks and inspected through that open file, so the +// thing checked is the thing the caller will go on to use. Creation is +// exclusive for the same reason — O_CREAT|O_EXCL refuses a symlink outright +// rather than following it somewhere else. +func EnsurePrivateFile(path string) error { + if err := EnsurePrivateDir(filepath.Dir(path)); err != nil { + return err + } + f, err := openNoFollow(path) + if errors.Is(err, os.ErrNotExist) { + created, createErr := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + switch { + case createErr == nil: + defer created.Close() + return checkPrivateReadableFile(created, path) + case !errors.Is(createErr, os.ErrExist): + return fmt.Errorf("create %s: %w", path, createErr) + } + // Something appeared between the open and the create — including, + // possibly, a symlink, which is why this reopen still refuses to + // follow one. + f, err = openNoFollow(path) + } + if err != nil { + return err + } + defer f.Close() + return checkPrivateReadableFile(f, path) +} + +func checkPrivateReadableFile(f *os.File, path string) error { + if err := checkPrivateFile(f, path); err != nil { + return err + } + info, err := f.Stat() + if err != nil { + return fmt.Errorf("inspect %s: %w", path, err) + } + if perm := info.Mode().Perm(); perm&0o077 != 0 { + return fmt.Errorf("%w: %s can be read by other users (mode %04o); it must be 0600", ErrNotPrivate, path, perm) + } + return nil +} From f022c1a61802e3ea1da408b7d4e9476646887b44 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 03:56:29 +0200 Subject: [PATCH 42/49] Report the create that fails, and forget only what was offered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A writable handle was closed in a defer, so a close that failed said nothing and the caller went on to treat the file as one that exists. It is closed and reported explicitly now, and since what is durable about an empty file is its directory entry, the directory is synced before the create is called done. The other write in the package already syncs, closes and renames; SQLite answers for the ledger's own durability. And the start's re-queue cleared the whole stranded list, on the assumption that it had just offered everything in it. It had not necessarily: the repair walks start before the re-queue and serve older ids, so one stranded mid-pass — past the page the walk is on — was forgotten without ever being handed over, which is the wait this was meant to remove. It now clears only the ids it carried in. --- internal/connector/intake.go | 35 +++++++++++++++++------ internal/connector/lock.go | 2 +- internal/connector/requeue_test.go | 35 +++++++++++++++++++++++ internal/connector/setup/private_state.go | 16 +++++++++-- 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index d07d7d7f5..1c373c690 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -779,6 +779,19 @@ func entryClassOf(resumeURL string) EntryClass { } } +// strandedIDs is the ids a handover failed on, oldest first, as the feed +// served them. +func (in *Intake) strandedIDs() []int64 { + in.mu.Lock() + defer in.mu.Unlock() + ids := make([]int64, 0, len(in.stranded)) + for id := range in.stranded { + ids = append(ids, id) + } + slices.Sort(ids) + return ids +} + // strand remembers an id the ledger has but the queue does not. // // Every commit goes through ingest, and every handover failure after a commit @@ -805,16 +818,10 @@ func (in *Intake) strand(id int64) { // It offers and only then forgets: an offer refused by a canceled context // leaves the id stranded for the next sweep, or for the next start. func (in *Intake) sweepStranded(ctx context.Context) { - in.mu.Lock() - ids := make([]int64, 0, len(in.stranded)) - for id := range in.stranded { - ids = append(ids, id) - } - in.mu.Unlock() + ids := in.strandedIDs() if len(ids) == 0 { return } - slices.Sort(ids) // oldest first, as the feed served them for _, id := range ids { if err := in.queue.Offer(ctx, id); err != nil { in.log.Warn("an event the ledger holds could not be handed over; it stays for the next sweep", "event_id", id, "error", err) @@ -842,6 +849,11 @@ func (in *Intake) sweepStranded(ctx context.Context) { // every watcher for the ordinary case, to close a gap in the stream rather // than in the work. func (in *Intake) requeueSeen(ctx context.Context) error { + // What is cleared at the end is what was stranded when this began. A + // repair walk is already running by now, and it serves OLD ids, which the + // paging below may already have passed: one stranded mid-pass would be + // forgotten by a clear that assumed it had offered everything. + carried := in.strandedIDs() var after int64 for { records, err := in.ledger.RecordsInStateAfter(ctx, StateSeen, after, requeueBatch) @@ -860,7 +872,9 @@ func (in *Intake) requeueSeen(ctx context.Context) error { // Everything in seen has just been offered, the ids a previous // run stranded included. in.mu.Lock() - in.stranded = nil + for _, id := range carried { + delete(in.stranded, id) + } in.mu.Unlock() return nil } @@ -958,6 +972,11 @@ func (in *Intake) startRepairWorkers(ctx context.Context) { // an event committed but never handed over, and an open loss nothing is // walking. // +// One goroutine does both, so a handover that waits at the pause threshold +// also holds up the re-offering of open losses. That is the right way round: +// the backlog is full, the pipeline is stopped, and starting more repair +// walks would only make the backlog worse. +// // The queue is bounded, so an overloaded connector can turn one away; and a // walk can end early, leaving its loss open. Neither may leave a loss with // nothing that will attempt it again, and waiting for a restart is not an diff --git a/internal/connector/lock.go b/internal/connector/lock.go index 19821d240..4a99c4119 100644 --- a/internal/connector/lock.go +++ b/internal/connector/lock.go @@ -23,7 +23,7 @@ var ErrAlreadyRunning = errors.New("connector: another connector already holds t // mentions being dispatched twice, which is a property of the identity, not of // the file that names it. // -// The kernel drops an flock when the holding descriptor closes, process death +// The kernel drops a flock when the holding descriptor closes, process death // included, so a crashed connector cannot wedge the lock and there is no // stale-lock reaping to get wrong. The metadata written beside it is // diagnostic only: the lock is the lock. diff --git a/internal/connector/requeue_test.go b/internal/connector/requeue_test.go index e7486f6b0..0996b9fb2 100644 --- a/internal/connector/requeue_test.go +++ b/internal/connector/requeue_test.go @@ -3,6 +3,7 @@ package connector import ( "context" "errors" + "sync" "testing" "time" @@ -152,3 +153,37 @@ func TestASweepThatCannotHandOverKeepsTheStrandedID(t *testing.T) { require.NoError(t, err, "the id the earlier sweep kept should be handed over now") assert.Equal(t, int64(42), id) } + +// The start's re-queue offers what is in seen and then forgets the ids it +// carried in — but only those. A repair walk is already running by then, and +// it serves OLD ids, which the paging may already have passed: one stranded +// mid-pass must not be forgotten by a clear that assumed it had offered +// everything. +func TestTheStartOnlyForgetsTheStrandedIDsItCarriedIn(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + + intake, _, _ := newTestIntakeOn(t, ledger, nil) + queue, err := NewQueue(1, 10) + require.NoError(t, err) + intake.queue = queue + var once sync.Once + // Stranded while the re-queue is paging: a repair walk's hand-off failing + // on an id the paging has already gone past. + queue.OnWarn = func(int) { once.Do(func() { intake.strand(999) }) } + + require.NoError(t, intake.requeueSeen(ctx)) + require.Equal(t, 1, queue.Depth()) + + intake.sweepStranded(ctx) + + require.Equal(t, 2, queue.Depth(), "the id stranded mid-pass was never offered") + first, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + second, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(999), second) +} diff --git a/internal/connector/setup/private_state.go b/internal/connector/setup/private_state.go index a4436c0ff..81b55d319 100644 --- a/internal/connector/setup/private_state.go +++ b/internal/connector/setup/private_state.go @@ -118,8 +118,20 @@ func EnsurePrivateFile(path string) error { created, createErr := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) switch { case createErr == nil: - defer created.Close() - return checkPrivateReadableFile(created, path) + // The handle is writable, so its close is reported rather than + // deferred away: a close that fails is a file that may not be + // there, and the caller is about to treat it as one that is. + // Nothing has been written to it, so the durable part is the + // directory entry, which is what is synced. + if err := checkPrivateReadableFile(created, path); err != nil { + _ = created.Close() + return err + } + if err := created.Close(); err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + syncDir(filepath.Dir(path)) + return nil case !errors.Is(createErr, os.ErrExist): return fmt.Errorf("create %s: %w", path, createErr) } From 3963d28048d276e607b24566750997c9aad35053 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 04:06:26 +0200 Subject: [PATCH 43/49] Make terminal mean terminal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit completed and discarded are documented as the end of a record's life, but SetState updated the row whatever it held, so either could be moved back into the working states — an event dispatched a second time, or, once DropContent has taken the payload and left the tombstone, a row picked up as work with nothing in it. The lifecycle is now a table of edges rather than a sentence in a comment, and the refusal is in the UPDATE itself: a state is written only where the record is in a state that may enter it, so a check and a write cannot race. Writing the state a record already has stays allowed — that is a repeat, not a move, so a retry after a crash is not an error. A refused transition is told apart from a missing row. Migration 3 adds the same refusal as a trigger, for the two edges that matter most: whatever ever writes to this file, a terminal record cannot change state. --- internal/connector/invariants_test.go | 97 +++++++++++++++++++++++++++ internal/connector/ledger.go | 15 +++++ internal/connector/ledger_events.go | 77 +++++++++++++++++++-- internal/connector/round7_test.go | 3 + 4 files changed, 188 insertions(+), 4 deletions(-) diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go index e1ac6575e..0ae6a6460 100644 --- a/internal/connector/invariants_test.go +++ b/internal/connector/invariants_test.go @@ -250,3 +250,100 @@ func TestInvariantH2ConcurrentFreshOpensAllSucceed(t *testing.T) { assert.NoError(t, err) } } + +// E4: terminal means terminal. A completed or discarded record that could move +// back into the working states could be dispatched a second time — and once +// DropContent has taken its payload, requeued as work with nothing in it. +func TestInvariantE4TerminalRecordsHaveNoWayBack(t *testing.T) { + active := []RecordState{StateSeen, StateAdmitted, StateQueued, StateBlocked, StateDispatched} + for _, terminal := range []RecordState{StateCompleted, StateDiscarded} { + for _, target := range append(active, terminalPeer(terminal)) { + t.Run(string(terminal)+" to "+string(target), func(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + require.NoError(t, reachTerminal(t, ledger, 1, terminal)) + + reason := "" + if target == StateBlocked || target == StateDiscarded { + reason = "a reason" + } + err := ledger.SetState(ctx, 1, target, reason) + + require.Error(t, err) + assert.ErrorIs(t, err, ErrNotATransition) + record, ok, getErr := ledger.Get(ctx, 1) + require.NoError(t, getErr) + require.True(t, ok) + assert.Equal(t, terminal, record.State, "the record stayed where it was") + }) + } + } +} + +// Writing the state a record already has is a repeat, not a move: a retry +// after a crash is not an error, and the record does not leave its state. +func TestInvariantE4TerminalRecordsTolerateARepeat(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + require.NoError(t, reachTerminal(t, ledger, 1, StateCompleted)) + + require.NoError(t, ledger.SetState(ctx, 1, StateCompleted, "")) + + record, ok, err := ledger.Get(ctx, 1) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateCompleted, record.State) +} + +// The refusal is the database's too, so anything that ever writes to this file +// meets it — not only this package's own SetState. +func TestInvariantE4TheDatabaseRefusesLeavingATerminalState(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + require.NoError(t, reachTerminal(t, ledger, 1, StateDiscarded)) + + _, err := ledger.db.ExecContext(ctx, `UPDATE events SET state = 'seen' WHERE id = 1`) + + require.Error(t, err) + assert.Contains(t, err.Error(), "terminal record") +} + +// A record that does not exist is told apart from a transition that is not +// allowed: one is a missing row, the other a refusal. +func TestInvariantE4AMissingRecordIsNotARefusedTransition(t *testing.T) { + ledger := newTestLedger(t) + + err := ledger.SetState(context.Background(), 404, StateAdmitted, "") + + assert.ErrorIs(t, err, ErrNoSuchRecord) + assert.NotErrorIs(t, err, ErrNotATransition) +} + +// terminalPeer is the other terminal state, so the pairs cover completed to +// discarded and back. +func terminalPeer(state RecordState) RecordState { + if state == StateCompleted { + return StateDiscarded + } + return StateCompleted +} + +// reachTerminal walks a fresh record to a terminal state along the lifecycle's +// own edges. +func reachTerminal(t *testing.T, ledger *Ledger, id int64, terminal RecordState) error { + t.Helper() + ctx := context.Background() + if _, err := ledger.RecordSeen(ctx, testEvent(id), LanePoll); err != nil { + return err + } + if terminal == StateDiscarded { + return ledger.SetState(ctx, id, StateDiscarded, "untrusted_author") + } + if err := ledger.SetState(ctx, id, StateAdmitted, ""); err != nil { + return err + } + if err := ledger.SetState(ctx, id, StateDispatched, ""); err != nil { + return err + } + return ledger.SetState(ctx, id, StateCompleted, "") +} diff --git a/internal/connector/ledger.go b/internal/connector/ledger.go index d5fa1eb18..0a9589bd3 100644 --- a/internal/connector/ledger.go +++ b/internal/connector/ledger.go @@ -267,6 +267,21 @@ CREATE TABLE gaps ( // one and its open losses carry an empty set, which reads as "the // connector's own". `ALTER TABLE losses ADD COLUMN filters TEXT NOT NULL DEFAULT ''`, + // Migration 3. Terminal means terminal, in the database and not only in + // the code that writes to it. A completed or discarded record that could + // be moved back into the working states could be dispatched a second + // time, or — once its payload has been dropped and only the tombstone + // remains — requeued as work with nothing in it. The Go side refuses + // every edge the lifecycle does not have; this refuses the two that + // matter to anything that ever writes to this file. + ` +CREATE TRIGGER events_terminal_is_terminal +BEFORE UPDATE OF state ON events +WHEN OLD.state IN ('completed', 'discarded') AND NEW.state <> OLD.state +BEGIN + SELECT RAISE(ABORT, 'a terminal record cannot change state'); +END; +`, } func (l *Ledger) migrate(ctx context.Context) error { diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go index 77ac597ae..82d1fa0b8 100644 --- a/internal/connector/ledger_events.go +++ b/internal/connector/ledger_events.go @@ -6,6 +6,8 @@ import ( "encoding/json" "errors" "fmt" + "slices" + "strings" "time" "github.com/basecamp/basecamp-sdk/go/pkg/basecamp/eventfeed" @@ -152,9 +154,55 @@ func (l *Ledger) CountInState(ctx context.Context, state RecordState) (int, erro return n, nil } +// lifecycle is the ledger's state machine: for each state, the states a record +// may move to from it. +// +// Intake writes only seen; the rest is written by admission and dispatch. The +// table lives here anyway, because the guarantee it makes is the ledger's: a +// completed or discarded record is finished, and a store that lets one move +// back into the working states is a store where an event can be dispatched +// twice, or where a tombstone whose payload has been dropped is picked up as +// work with nothing in it. +// +// Blocked is not terminal on purpose: it is retained and retried, so it has +// edges back into the working states. Completed and discarded have none. +var lifecycle = map[RecordState][]RecordState{ + StateSeen: {StateAdmitted, StateBlocked, StateDiscarded}, + StateAdmitted: {StateQueued, StateDispatched, StateBlocked, StateDiscarded}, + StateQueued: {StateDispatched, StateBlocked, StateDiscarded}, + StateBlocked: {StateAdmitted, StateQueued, StateDispatched, StateDiscarded}, + StateDispatched: {StateCompleted, StateBlocked}, + StateCompleted: nil, + StateDiscarded: nil, +} + +// enterableFrom is the states a record may be in for a move to target to be +// allowed, target itself included: writing the state a record already has is a +// repeat, not a move, so a retry after a crash is not an error. +func enterableFrom(target RecordState) []string { + froms := []string{string(target)} + for from, tos := range lifecycle { + if slices.Contains(tos, target) { + froms = append(froms, string(from)) + } + } + // Sorted so the statement is the same every time it is built, whatever + // order the map ranges in. + slices.Sort(froms) + return froms +} + +// ErrNotATransition reports a state change the lifecycle does not have. +var ErrNotATransition = errors.New("not a transition the ledger's lifecycle allows") + // SetState moves a record to state with a reason, which must be empty for // every state but blocked and discarded — those two are the only ones a reason // explains. +// +// The move is refused unless the lifecycle has that edge, and it is refused by +// the UPDATE itself rather than by a read before it: a check and a write in +// two statements is a race, and this is the guarantee that a finished record +// stays finished. func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reason string) error { switch state { case StateBlocked, StateDiscarded: @@ -169,9 +217,17 @@ func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reas // A state outside the lifecycle is a row no recovery scan looks for. return fmt.Errorf("connector: set state of %d: %q is not a ledger state", id, state) } - res, err := l.db.ExecContext(ctx, - `UPDATE events SET state = ?, reason = ?, updated_at = ? WHERE id = ?`, - string(state), reason, l.timestamp(), id) + froms := enterableFrom(state) + args := []any{string(state), reason, l.timestamp(), id} + for _, from := range froms { + args = append(args, from) + } + // The only thing concatenated is a list of "?" as long as the lifecycle's + // own edge list. Every value is bound. + //nolint:gosec // G202: placeholders, not values + query := `UPDATE events SET state = ?, reason = ?, updated_at = ? WHERE id = ? AND state IN (` + + strings.TrimSuffix(strings.Repeat("?, ", len(froms)), ", ") + `)` + res, err := l.db.ExecContext(ctx, query, args...) if err != nil { return fmt.Errorf("connector: set state of %d: %w", id, err) } @@ -180,11 +236,24 @@ func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reas return fmt.Errorf("connector: set state of %d: %w", id, err) } if affected == 0 { - return fmt.Errorf("connector: set state of %d: %w", id, ErrNoSuchRecord) + return l.explainRefusal(ctx, id, state) } return nil } +// explainRefusal says why an update changed nothing: there is no such record, +// or the record is somewhere the lifecycle cannot leave for state. +func (l *Ledger) explainRefusal(ctx context.Context, id int64, state RecordState) error { + var current string + switch err := l.db.QueryRowContext(ctx, `SELECT state FROM events WHERE id = ?`, id).Scan(¤t); { + case errors.Is(err, sql.ErrNoRows): + return fmt.Errorf("connector: set state of %d: %w", id, ErrNoSuchRecord) + case err != nil: + return fmt.Errorf("connector: set state of %d: %w", id, err) + } + return fmt.Errorf("connector: set state of %d: %s to %s is %w", id, current, state, ErrNotATransition) +} + // ErrNoSuchRecord reports a state change addressed at an id the ledger does // not hold. var ErrNoSuchRecord = errors.New("no such event record") diff --git a/internal/connector/round7_test.go b/internal/connector/round7_test.go index 2b5e50945..b55e73c6d 100644 --- a/internal/connector/round7_test.go +++ b/internal/connector/round7_test.go @@ -39,6 +39,9 @@ func TestDropContentKeepsARecordUpdatedJustAfterAWholeSecondCutoff(t *testing.T) _, err := ledger.RecordSeen(ctx, testEvent(43), LanePoll) require.NoError(t, err) + // The lifecycle has no shortcut to completed, so the record walks there. + require.NoError(t, ledger.SetState(ctx, 43, StateAdmitted, "")) + require.NoError(t, ledger.SetState(ctx, 43, StateDispatched, "")) require.NoError(t, ledger.SetState(ctx, 43, StateCompleted, "")) dropped, err := ledger.DropContent(ctx, cutoff, cutoff) From 9c89acac85a5975b53f6d8bacd85c36e32bdd47c Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 04:18:10 +0200 Subject: [PATCH 44/49] Let a refused position die with the epoch that refused it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 410 with an epoch says the history below it is gone for good, and the walk clears its cursor on disk when it takes the fence. But the pass kept its own copy of the position in a local, and returned that copy if the resumed poll then failed for any other reason — a connection reset was enough. The caller put it back in memory, and every later pass entered at a position the feed had already refused forever. The walk's memory of the cursor is now the loss's own field, kept equal to what the ledger holds at every point: saved first, remembered second, and whatever a failure wrote is what the pass returns. Two edges the lifecycle was missing, from the same spec the table came from: seen to queued, because admission commits an admitted verdict AS queued when the conversation is already live, and dispatched back to admitted, because a dispatched record whose worker never started has its exposure withdrawn. Without them the next card writes around SetState. And a repeat no longer touches updated_at, which is the retention clock: a re-written completion was silently restarting the window. --- internal/connector/invariants_test.go | 109 ++++++++++++++++++++++++++ internal/connector/ledger_events.go | 33 ++++++-- internal/connector/repair.go | 35 ++++++--- 3 files changed, 156 insertions(+), 21 deletions(-) diff --git a/internal/connector/invariants_test.go b/internal/connector/invariants_test.go index 0ae6a6460..596262e02 100644 --- a/internal/connector/invariants_test.go +++ b/internal/connector/invariants_test.go @@ -347,3 +347,112 @@ func reachTerminal(t *testing.T, ledger *Ledger, id int64, terminal RecordState) } return ledger.SetState(ctx, id, StateCompleted, "") } + +// A2: a cursor older than the feed's current epoch is never usable. A 410 with +// an epoch says the history below it is gone for good, so the position the +// walk was holding is not merely superseded — it is refused by the server +// forever, and a pass that ends for some other reason must not hand it back. +func TestInvariantA2APreEpochCursorIsNeverUsedAgain(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + clock := &walkClock{at: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)} + // The missing id is above the epoch, so the walk still has something to + // serve after the fence and follows the resume. + loss, err := ledger.RecordLoss(ctx, []int64{17099838700}, clock.at, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + require.NoError(t, ledger.SaveRepairCursor(ctx, loss.ID, "PRE-EPOCH-POSITION")) + loss.RepairCursor = "PRE-EPOCH-POSITION" + + const epoch = int64(17099838600) + polls := &scriptedPolls{errs: []error{ + // The stored cursor is below the epoch: the fence. + &eventfeed.PollError{ + Kind: eventfeed.PollGone, + EpochAfterID: epoch, + ResumeURL: "https://3.basecampapi.com/2914079/events.json?since=17099838600", + Err: errors.New("gone"), + }, + // The resume then fails transiently — a connection reset, not a + // verdict — and the pass ends. + errors.New("connection reset"), + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + + cursor, err := walker.walk(ctx, &loss) + require.NoError(t, err) + + assert.Empty(t, cursor, "the pass must not carry the refused position back to its caller") + assert.Empty(t, loss.RepairCursor) + stored, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + require.Len(t, stored, 1) + assert.Empty(t, stored[0].RepairCursor, "the pre-epoch cursor is gone from the ledger too") + assert.Equal(t, epoch, stored[0].RepairSince) + + // The next pass enters at the epoch, never at the refused position. + polls.errs = nil + _, err = walker.walk(ctx, &stored[0]) + require.NoError(t, err) + entered := polls.cursors[len(polls.cursors)-1] + assert.Equal(t, "17099838600", entered.Since) + // The first poll of the first pass is where the refusal was discovered. + // Nothing after it may name that position again. + for _, seen := range polls.cursors[1:] { + assert.NotEqual(t, "PRE-EPOCH-POSITION", seen.Position, "no pass may re-enter at the refused position") + } +} + +// The lifecycle carries the edges the later cards actually commit, so neither +// has to write state around SetState to make its own contract work. +func TestInvariantE4TheLifecycleCarriesTheEdgesLaterCardsCommit(t *testing.T) { + // Admission commits an admitted verdict AS queued when the conversation + // is live, so the record goes from seen to queued in one write. + t.Run("seen to queued", func(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + + require.NoError(t, ledger.SetState(ctx, 1, StateQueued, "")) + + record, ok, err := ledger.Get(ctx, 1) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateQueued, record.State) + }) + + // A dispatched record whose worker never started has its exposure + // withdrawn and returns to admitted. + t.Run("dispatched back to admitted", func(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + _, err := ledger.RecordSeen(ctx, testEvent(1), LanePoll) + require.NoError(t, err) + require.NoError(t, ledger.SetState(ctx, 1, StateAdmitted, "")) + require.NoError(t, ledger.SetState(ctx, 1, StateDispatched, "")) + + require.NoError(t, ledger.SetState(ctx, 1, StateAdmitted, "")) + + record, ok, err := ledger.Get(ctx, 1) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, StateAdmitted, record.State) + }) +} + +// updated_at is the retention clock. Writing the state a record already has is +// a repeat, and a repeat must not restart the window on a finished record. +func TestInvariantE4ARepeatDoesNotRestartRetention(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + at := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + ledger.now = func() time.Time { return at } + require.NoError(t, reachTerminal(t, ledger, 1, StateCompleted)) + + ledger.now = func() time.Time { return at.Add(90 * 24 * time.Hour) } + require.NoError(t, ledger.SetState(ctx, 1, StateCompleted, "")) + + dropped, err := ledger.DropContent(ctx, at.Add(time.Hour), at.Add(time.Hour)) + require.NoError(t, err) + assert.Equal(t, 1, dropped, "the repeat must not have pushed the record's retention forward") +} diff --git a/internal/connector/ledger_events.go b/internal/connector/ledger_events.go index 82d1fa0b8..1867f7213 100644 --- a/internal/connector/ledger_events.go +++ b/internal/connector/ledger_events.go @@ -167,11 +167,18 @@ func (l *Ledger) CountInState(ctx context.Context, state RecordState) (int, erro // Blocked is not terminal on purpose: it is retained and retried, so it has // edges back into the working states. Completed and discarded have none. var lifecycle = map[RecordState][]RecordState{ - StateSeen: {StateAdmitted, StateBlocked, StateDiscarded}, - StateAdmitted: {StateQueued, StateDispatched, StateBlocked, StateDiscarded}, - StateQueued: {StateDispatched, StateBlocked, StateDiscarded}, - StateBlocked: {StateAdmitted, StateQueued, StateDispatched, StateDiscarded}, - StateDispatched: {StateCompleted, StateBlocked}, + // seen to queued is one edge, not two: admission commits an admitted + // verdict AS queued when the conversation is already live, so the record + // never passes through admitted at all. + StateSeen: {StateAdmitted, StateQueued, StateBlocked, StateDiscarded}, + StateAdmitted: {StateQueued, StateDispatched, StateBlocked, StateDiscarded}, + StateQueued: {StateDispatched, StateBlocked, StateDiscarded}, + StateBlocked: {StateAdmitted, StateQueued, StateDispatched, StateDiscarded}, + // A dispatched record whose worker never started has its exposure + // withdrawn and returns to admitted. It is never discarded: a dispatched + // event ends completed, with an outcome, even when the outcome is + // unknown. + StateDispatched: {StateCompleted, StateBlocked, StateAdmitted}, StateCompleted: nil, StateDiscarded: nil, } @@ -218,15 +225,21 @@ func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reas return fmt.Errorf("connector: set state of %d: %q is not a ledger state", id, state) } froms := enterableFrom(state) - args := []any{string(state), reason, l.timestamp(), id} + args := []any{string(state), reason, string(state), l.timestamp(), id} for _, from := range froms { args = append(args, from) } // The only thing concatenated is a list of "?" as long as the lifecycle's // own edge list. Every value is bound. //nolint:gosec // G202: placeholders, not values - query := `UPDATE events SET state = ?, reason = ?, updated_at = ? WHERE id = ? AND state IN (` + - strings.TrimSuffix(strings.Repeat("?, ", len(froms)), ", ") + `)` + // + // updated_at is left alone when the state does not change. It is the + // retention clock DropContent reads, and a repeated write of the state a + // record already has would silently restart the window on a finished + // record. + query := `UPDATE events SET state = ?, reason = ?, + updated_at = CASE WHEN state = ? THEN updated_at ELSE ? END +WHERE id = ? AND state IN (` + strings.TrimSuffix(strings.Repeat("?, ", len(froms)), ", ") + `)` res, err := l.db.ExecContext(ctx, query, args...) if err != nil { return fmt.Errorf("connector: set state of %d: %w", id, err) @@ -243,6 +256,10 @@ func (l *Ledger) SetState(ctx context.Context, id int64, state RecordState, reas // explainRefusal says why an update changed nothing: there is no such record, // or the record is somewhere the lifecycle cannot leave for state. +// +// The refusal itself already happened, in the UPDATE. This is the message, and +// it reads the row a second time: under a concurrent writer it can name a +// state the record has since left. Diagnostics, not a verdict to act on. func (l *Ledger) explainRefusal(ctx context.Context, id int64, state RecordState) error { var current string switch err := l.db.QueryRowContext(ctx, `SELECT state FROM events WHERE id = ?`, id).Scan(¤t); { diff --git a/internal/connector/repair.go b/internal/connector/repair.go index 7a82f8d39..ccbd53f78 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -238,7 +238,13 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { cursor = eventfeed.Cursor{Position: loss.RepairCursor} } - last := loss.RepairCursor + // loss.RepairCursor is the walk's memory of what is DURABLE, and it is + // kept equal to what the ledger holds at every point below. A 410 is why: + // the epoch makes the pre-epoch position permanently unusable, and + // pollFailure clears it on disk — so a pass that then ends for any other + // reason must not hand a copy of that position back to its caller, which + // would put it back in memory and start every later pass at a position + // the feed has already refused. pass := &repairPass{followed: map[string]bool{}} walked := map[string]bool{} maxPages := w.maxPages @@ -249,26 +255,27 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { if err := ctx.Err(); err != nil { // Cancellation is a delay, never a verdict: the loss stays open // on disk for the next start. - return last, err + return loss.RepairCursor, err } if pages >= maxPages { // Distinct positions forever evade cycle detection. The pass ends // at the cap, and the next one resumes from the cursor saved on // the last page. w.log.Warn("a repair pass reached its page cap; resuming on the repair cadence", "loss_id", loss.ID, "pages", pages) - return last, nil + return loss.RepairCursor, nil } pages++ page, err := w.polls.Poll(ctx, cursor, w.filters) if err != nil { next, err := w.pollFailure(ctx, loss, cursor, err, pass) if err != nil || next == nil { - return last, err + // pollFailure has already written whatever this failure did + // to the durable cursor — cleared it at an epoch fence, or + // left it alone — so what is returned is that, never a copy + // of the position the failure just invalidated. + return loss.RepairCursor, err } cursor = *next - if cursor.PageURL == "" && cursor.Position == "" { - last = "" - } continue } @@ -277,15 +284,17 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { // what marks the missing id recovered, and it is one code path // rather than two that must agree. if err := w.ingest(ctx, event, LaneRepair); err != nil { - return last, err + return loss.RepairCursor, err } } if page.Position != "" { - last = page.Position + // Saved first, remembered second: the walk's memory of the cursor + // never runs ahead of the ledger's. if err := w.ledger.SaveRepairCursor(ctx, loss.ID, page.Position); err != nil { - return last, err + return loss.RepairCursor, err } + loss.RepairCursor = page.Position } if page.Next == "" { @@ -293,7 +302,7 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { // page cut short by the safety horizon withholds the link on // purpose, so the caller polls again rather than concluding // anything. - return last, nil + return loss.RepairCursor, nil } // An empty page with a `next` is ordinary — the walk crossed rows the // filters excluded — so the loop never stops on len(Events) == 0. @@ -301,12 +310,12 @@ func (w *repairWalker) walk(ctx context.Context, loss *Loss) (string, error) { // A next already walked this pass — itself, or a cycle — would // spin. The pass ends and the repair cadence is the backoff. w.log.Warn("a repair page's next repeats a URL this pass already walked; ending the pass", "loss_id", loss.ID) - return last, nil + return loss.RepairCursor, nil } walked[page.Next] = true if err := sameOrigin(w.origin, page.Next); err != nil { w.log.Error("a repair page's next URL leaves the API origin; the loss stays open for the next start", "loss_id", loss.ID) - return last, errReconciliationEnded + return loss.RepairCursor, errReconciliationEnded } cursor = eventfeed.Cursor{PageURL: page.Next} } From f427ce7695ba73c86cfb6e91352a663ac70a2725 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 04:28:34 +0200 Subject: [PATCH 45/49] Build the verdict stream's writer before the workers exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The line writer was built on the first verdict. Nothing in this package needs that: admission knows its sink before it starts, so the writer is built there and injected, and the struct keeps a writer and nothing else — no sink, no once, no first-use path for a pool of workers to take together. A stream whose lines stay whole rests on there being exactly one writer and one lock per sink. That is easier to keep true when there is no second place a writer can come into being. --- internal/connector/admission/commit_test.go | 66 +++++++++++++++++++-- internal/connector/admission/run.go | 33 +++++++---- 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/internal/connector/admission/commit_test.go b/internal/connector/admission/commit_test.go index 731f67a27..919cdca0d 100644 --- a/internal/connector/admission/commit_test.go +++ b/internal/connector/admission/commit_test.go @@ -1,6 +1,8 @@ package admission import ( + "bufio" + "bytes" "context" "encoding/json" "errors" @@ -396,7 +398,7 @@ func (stuckWriter) Write([]byte) (int, error) { return 0, nil } func TestALineIsWrittenWholeOrNotAtAll(t *testing.T) { w := &chunkWriter{chunk: 7} - require.NoError(t, (&lineWriter{w: w}).write(Verdict{EventID: 1, EventType: "card.created", State: StateDiscarded, Reason: ReasonNotInMatrix})) + require.NoError(t, newLineWriter(nil, w).write(Verdict{EventID: 1, EventType: "card.created", State: StateDiscarded, Reason: ReasonNotInMatrix})) var m map[string]any require.NoError(t, json.Unmarshal([]byte(strings.TrimSuffix(w.b.String(), "\n")), &m), "a writer that takes a line in pieces still gets all of it") assert.True(t, strings.HasSuffix(w.b.String(), "}\n")) @@ -404,7 +406,7 @@ func TestALineIsWrittenWholeOrNotAtAll(t *testing.T) { // Bounded: a writer loop that forgot this case would spin, and should fail // this test by name rather than hang the suite. done := make(chan error, 1) - go func() { done <- (&lineWriter{w: stuckWriter{}}).write(Verdict{EventID: 1, State: StateDiscarded}) }() + go func() { done <- newLineWriter(nil, stuckWriter{}).write(Verdict{EventID: 1, State: StateDiscarded}) }() select { case err := <-done: require.ErrorIs(t, err, io.ErrShortWrite) @@ -416,7 +418,7 @@ func TestALineIsWrittenWholeOrNotAtAll(t *testing.T) { func TestLinesCannotCarryTerminalControls(t *testing.T) { esc, csi, bel := string(rune(0x1b)), string(rune(0x9b)), string(rune(0x07)) var b strings.Builder - require.NoError(t, (&lineWriter{w: &b}).write(Verdict{ + require.NoError(t, newLineWriter(nil, &b).write(Verdict{ EventID: 1, EventType: "card.created" + esc + "]0;owned" + bel, RecordingURL: "https://app.basecamp.com/x" + csi + "31m" + esc + "[2J", @@ -462,9 +464,65 @@ func TestRunStopsOnALoadFailure(t *testing.T) { func TestLinesKeepLocalPathsAsWritten(t *testing.T) { var b strings.Builder - require.NoError(t, (&lineWriter{w: &b}).write(Verdict{EventID: 1, State: StateAdmitted, Route: " /work/My Projects\tA", Class: "in ternal"})) + require.NoError(t, newLineWriter(nil, &b).write(Verdict{EventID: 1, State: StateAdmitted, Route: " /work/My Projects\tA", Class: "in ternal"})) var m map[string]any require.NoError(t, json.Unmarshal([]byte(b.String()), &m)) assert.Equal(t, " /work/My Projects\tA", m["route"], "whitespace in a path is not a terminal control") assert.Equal(t, "in ternal", m["class"]) } + +// A sink that is safe for concurrent use, so the only race a run can report is +// one of admission's own making. +type lockedSink struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (s *lockedSink) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.buf.Write(p) +} + +// The verdict writer is built before any worker exists. Built on first use, a +// pool of workers reaching their first lines together would read and assign it +// at once — a data race on the ordinary path, not an edge case. +func TestTheVerdictWriterIsBuiltBeforeAnyWorkerWrites(t *testing.T) { + sink := &lockedSink{} + lines := newLineWriter(nil, sink) + + var wg sync.WaitGroup + for worker := range 8 { + wg.Go(func() { + for i := range 25 { + require.NoError(t, lines.write(Verdict{ + EventID: int64(worker*100 + i), + EventType: "comment.created", + State: StateAdmitted, + })) + } + }) + } + wg.Wait() + + scanner := bufio.NewScanner(bytes.NewReader(sink.buf.Bytes())) + count := 0 + for scanner.Scan() { + var line map[string]any + require.NoError(t, json.Unmarshal(scanner.Bytes(), &line), "torn line %d: %q", count, scanner.Text()) + count++ + } + require.NoError(t, scanner.Err()) + assert.Equal(t, 200, count) +} + +// One sink has one writer, whoever asks for it: the lock that keeps lines +// whole is only one lock if there is only one of it. +func TestOneSinkHasOneWriter(t *testing.T) { + sink := &lockedSink{} + + first := newLineWriter(nil, sink) + second := newLineWriter(nil, sink) + + assert.Same(t, first.out, second.out) +} diff --git a/internal/connector/admission/run.go b/internal/connector/admission/run.go index 1d1f9f8b0..421d94f02 100644 --- a/internal/connector/admission/run.go +++ b/internal/connector/admission/run.go @@ -72,7 +72,9 @@ func Run(ctx context.Context, opts RunOptions) error { if log == nil { log = slog.New(slog.DiscardHandler) } - lines := &lineWriter{w: opts.Lines, out: opts.LineWriter} + // Built here, before the workers exist, so nothing is initialized on a + // path two of them can take at once. + lines := newLineWriter(opts.LineWriter, opts.Lines) ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -182,22 +184,33 @@ func LineFor(v Verdict) Line { } } +// lineWriter is the verdict stream. It holds a writer and nothing else: no +// flag, no once, no first-use path. +// +// It used to build the writer on the first verdict, which is a race the moment +// admission has more than one worker — two of them reaching their first line +// together read and assign the same field. Laziness is not a thing to guard +// here; it is a thing to remove. The writer is built once, before any worker +// starts, and injected. type lineWriter struct { - w io.Writer + out *ndjson.Writer +} - once sync.Once - out *ndjson.Writer +// newLineWriter takes the writer a caller wiring admission beside intake +// passes in, or builds one for the sink. There is exactly one writer per sink +// either way: NewWriter returns the same writer for the same sink, so the one +// lock is the one lock. +func newLineWriter(out *ndjson.Writer, sink io.Writer) *lineWriter { + if out == nil && sink != nil { + out = ndjson.NewWriter(sink) + } + return &lineWriter{out: out} } func (l *lineWriter) write(v Verdict) error { - if l.w == nil && l.out == nil { + if l.out == nil { return nil } - l.once.Do(func() { - if l.out == nil { - l.out = ndjson.NewWriter(l.w) - } - }) if err := l.out.WriteLine(LineFor(v)); err != nil { return fmt.Errorf("admission: %w", err) } From 0d84ab6c445cd41b206cb176ab26a589be0e9e9a Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 04:37:57 +0200 Subject: [PATCH 46/49] Keep a server's patience inside the deadline it is asking against MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The absolute lifetime bounds the loss, but the waits between its attempts were unbounded: Retry-After is whatever the server names, so one response asking for thirty hours held the loss open — and its repair worker with it — well past the day the connector promises. Every repair wait is now capped at the time the loss has left. Woken at the deadline, the next turn around the loop sees it and closes the loss without polling first, which is the honest answer: the server asked for a wait this connector no longer has to give. --- internal/connector/repair.go | 23 ++++++++++++-- internal/connector/repair_bounds_test.go | 39 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/internal/connector/repair.go b/internal/connector/repair.go index ccbd53f78..caa006ab7 100644 --- a/internal/connector/repair.go +++ b/internal/connector/repair.go @@ -120,12 +120,31 @@ func (w *repairWalker) reconcile(ctx context.Context, loss Loss) error { // says nothing about the next one. w.retryAfter = 0 } - if err := w.wait(ctx, wait); err != nil { + if err := w.waitWithin(ctx, wait, loss); err != nil { return err } } } +// waitWithin sleeps for wait, but never past the loss's absolute deadline. +// +// The 24-hour lifetime bounds the LOSS, not the intervals between its +// attempts, and a server-directed Retry-After is not bounded by anything: one +// response asking for thirty hours would otherwise hold the loss open — and +// its repair worker with it — well past the deadline the connector promises. +// Woken at the deadline, the caller's next turn around the loop sees it and +// closes the loss without polling first, which is right: the server asked for +// a wait this connector no longer has to give. +func (w *repairWalker) waitWithin(ctx context.Context, wait time.Duration, loss Loss) error { + if remaining := loss.DetectedAt.Add(maxLossLifetime).Sub(w.now()); remaining < wait { + wait = remaining + } + if wait <= 0 { + return nil + } + return w.wait(ctx, wait) +} + // postpone moves a loss's window out by the wait the server asked for. // // This is the whole rule for a throttle, in one place: it is not a failure and @@ -164,7 +183,7 @@ func (w *repairWalker) finalPass(ctx context.Context, loss *Loss) error { w.retryAfter = 0 w.log.Warn("the last repair attempt was throttled; waiting as the server asked and trying again", "loss_id", loss.ID, "retry_after", wait) - if err := w.wait(ctx, wait); err != nil { + if err := w.waitWithin(ctx, wait, *loss); err != nil { return err } return w.reconcile(ctx, *loss) diff --git a/internal/connector/repair_bounds_test.go b/internal/connector/repair_bounds_test.go index d6d3736dc..34958ef77 100644 --- a/internal/connector/repair_bounds_test.go +++ b/internal/connector/repair_bounds_test.go @@ -2,6 +2,7 @@ package connector import ( "context" + "errors" "strconv" "sync" "testing" @@ -210,3 +211,41 @@ type alwaysThrottling struct{} func (alwaysThrottling) Poll(context.Context, eventfeed.Cursor, eventfeed.Filters) (eventfeed.PollPage, error) { return eventfeed.PollPage{}, &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 15 * time.Minute} } + +// A throttle is not a failure, and the wait it asks for is the server's to +// name — but the loss's absolute deadline is this connector's promise, and an +// uncapped Retry-After would hold the loss open, and its repair worker with +// it, well past the day it is allowed to live. +func TestAThrottleCannotHoldALossPastItsAbsoluteDeadline(t *testing.T) { + ledger := newTestLedger(t) + ctx := context.Background() + detected := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + clock := &walkClock{at: detected} + loss, err := ledger.RecordLoss(ctx, []int64{17099838509}, detected, time.Hour, eventfeed.Filters{}) + require.NoError(t, err) + + // Twenty-three hours in, the server asks for six more. + clock.at = detected.Add(23 * time.Hour) + var slept []time.Duration + polls := &scriptedPolls{errs: []error{ + &eventfeed.PollError{Kind: eventfeed.PollThrottled, RetryAfter: 6 * time.Hour, Err: errors.New("slow down")}, + }} + walker, _ := newTestWalker(t, ledger, polls, clock) + walker.sleep = func(_ context.Context, d time.Duration) error { + slept = append(slept, d) + clock.at = clock.at.Add(d) + return nil + } + + require.NoError(t, walker.reconcile(ctx, loss)) + + require.NotEmpty(t, slept) + for _, d := range slept { + assert.LessOrEqual(t, d, time.Hour, "no wait may reach past the loss's last hour") + } + assert.LessOrEqual(t, clock.at.Sub(detected), maxLossLifetime, "the loss outlived its deadline") + + open, err := ledger.OpenLosses(ctx) + require.NoError(t, err) + assert.Empty(t, open, "the loss closes at its deadline whatever the server asked for") +} From 90e644a06276fcf52f7e867c7f302e54e2d3ecfe Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 04:47:34 +0200 Subject: [PATCH 47/49] Let a panicking callback cost the report, not the queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A callback belongs to whoever built the queue, and one that panicked took the queue's own state with it: the pause callback runs before the offer's cleanup was in place, so the id's count and the wait it announced both stayed behind — a queue reporting an item nobody can take, paused with nothing able to lift it. Three changes, all of them about the same rule. The state a callback is told about is committed before it runs, so nothing it does can change it. An offer that leaves without sending takes its count back on every way out, not only on cancellation. And a panic in a callback is contained and logged: a reporting bug is not a reason to lose the feed. --- internal/connector/queue.go | 74 ++++++++++++++++++++++++++----- internal/connector/queue_test.go | 76 ++++++++++++++++++++++++++------ 2 files changed, 126 insertions(+), 24 deletions(-) diff --git a/internal/connector/queue.go b/internal/connector/queue.go index 8cdef4778..cb3eab04f 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -3,7 +3,11 @@ package connector import ( "context" "errors" + "fmt" + "log/slog" "sync" + + "github.com/basecamp/basecamp-cli/internal/richtext" ) // Backlog thresholds. Intake is the only work on the feed's delivery path, so @@ -64,6 +68,8 @@ type Queue struct { // it stops. The feed is not being consumed in between. OnPause func(depth int) OnResume func(depth int) + // Logger reports a callback that panicked. Optional; silent when unset. + Logger *slog.Logger } // NewQueue builds a queue that warns at warnAt and pauses the feed at pauseAt. @@ -90,8 +96,26 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { // can never be applied before the increment it belongs to, and a crossing // can never be lost to the order two operations happen to take the lock in. q.stage(1) + // An offer that leaves without sending must take its count back with it, + // on every way out — the ordinary cancellation, and an unwinding this + // function did not choose. A count left behind is a queue that reports an + // item nobody can take and, at the threshold, a pause nothing can lift. + settled := false + settle := func(sent bool) { + if settled { + return + } + settled = true + if !sent { + q.stage(-1) + q.deliver() + } + } + defer func() { settle(false) }() + select { case q.ids <- id: + settle(true) q.afterOp() q.deliver() return nil @@ -99,21 +123,25 @@ func (q *Queue) Offer(ctx context.Context, id int64) error { } q.stageWait(1) - q.deliver() + // Registered BEFORE the pause is delivered: the wait is recorded, and + // what ends it must already be in place when anything else runs. defer func() { q.stageWait(-1) q.deliver() }() + q.deliver() select { case q.ids <- id: + settle(true) q.afterOp() q.deliver() return nil case <-ctx.Done(): - // It never went in, so it is not backlog. - q.stage(-1) - q.deliver() + // It never went in, so it is not backlog. Taken back here rather than + // left to the defer, so the resume that follows reports the depth + // without it. + settle(false) return ctx.Err() } } @@ -232,19 +260,43 @@ func (q *Queue) deliver() { edge := q.pending[0] q.pending = q.pending[1:] q.edges.Unlock() - // The lock is retaken by a defer, not after the call: a callback that - // panics unwinds through here, and the cleanup above unlocks. Retaking - // it on the way out of every callback — returned or panicked — is what - // makes that unlock the one that pairs with this Lock. + // The lock is retaken by a defer, not after the call: fire contains a + // panic, but a future change to it must not be able to leave this + // loop, or the cleanup above, holding nothing. func() { defer q.edges.Lock() - if edge.fire != nil { - edge.fire(edge.depth) - } + q.fire(edge) }() } } +// fire runs one callback with the lock released and a panic contained. +// +// The callbacks belong to whoever built the queue, and the goroutine they run +// on is the feed's delivery path or a worker taking work off it. A callback +// that panics is a reporting bug; it is not a reason to lose the connection, +// and it must not leave the queue part-way through an update. The state the +// callback is told about was committed before it ran, so what it does cannot +// change it. +func (q *Queue) fire(edge queueEdge) { + defer func() { + if p := recover(); p != nil { + q.logger().Error("a backlog callback panicked; the queue carried on without it", + "panic", richtext.SanitizeSingleLine(fmt.Sprint(p))) + } + }() + if edge.fire != nil { + edge.fire(edge.depth) + } +} + +func (q *Queue) logger() *slog.Logger { + if q.Logger != nil { + return q.Logger + } + return slog.New(slog.DiscardHandler) +} + type queueEdge struct { fire func(int) depth int diff --git a/internal/connector/queue_test.go b/internal/connector/queue_test.go index d99c0461f..f52ed0228 100644 --- a/internal/connector/queue_test.go +++ b/internal/connector/queue_test.go @@ -1,7 +1,9 @@ package connector import ( + "bytes" "context" + "log/slog" "sync" "sync/atomic" "testing" @@ -149,32 +151,31 @@ func TestACrossingIsNotLostToAConcurrentTake(t *testing.T) { assert.Zero(t, queue.Depth()) } +// A callback belongs to whoever built the queue, and a panic in one is a +// reporting bug — not a reason to lose the feed. It is contained, and the +// queue's own state is correct afterwards because it was committed before the +// callback ran. func TestAPanickingBacklogCallbackLeavesTheQueueUsable(t *testing.T) { queue, err := NewQueue(1, 4) require.NoError(t, err) - var recoveries int - queue.OnWarn = func(int) { panic("a warning callback panics") } + var warnings, recoveries int + queue.OnWarn = func(int) { warnings++; panic("a warning callback panics") } queue.OnRecover = func(int) { recoveries++ } - // The panic belongs to the callback. It must reach the caller, who can - // recover it, and not take the process — or the drain — with it. - panicked := func() (caught bool) { - defer func() { caught = recover() != nil }() - require.NoError(t, queue.Offer(context.Background(), 1)) - return false - }() - require.True(t, panicked, "the callback's panic should reach the caller") + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1), "the callback's panic is not the offer's failure") + assert.Equal(t, 1, warnings) // The drain is unlatched and the queue still works: a second id goes in, // both come out, and the recovery edge is delivered. - require.NoError(t, queue.Offer(context.Background(), 2)) + require.NoError(t, queue.Offer(ctx, 2)) assert.Equal(t, 2, queue.Depth()) - first, err := queue.Take(context.Background()) + first, err := queue.Take(ctx) require.NoError(t, err) assert.Equal(t, int64(1), first) - second, err := queue.Take(context.Background()) + second, err := queue.Take(ctx) require.NoError(t, err) assert.Equal(t, int64(2), second) assert.Equal(t, 0, queue.Depth()) @@ -249,3 +250,52 @@ func TestPauseAndResumeAreObservedInTheOrderTheyHappened(t *testing.T) { assert.Equal(t, int64(3), third) assert.False(t, queue.Paused()) } + +// A pause callback that panics must leave nothing behind: the id it was +// waiting for still goes in, the depth is the queue's real depth, the pause +// lifts, and the next crossing is reported. +func TestAPanickingPauseCallbackLeavesNoPhantomBacklog(t *testing.T) { + queue, err := NewQueue(1, 1) + require.NoError(t, err) + + var logs bytes.Buffer + queue.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + var pauses, resumes atomic.Int32 + queue.OnPause = func(int) { + if pauses.Add(1) == 1 { + panic("a pause callback panics") + } + } + queue.OnResume = func(int) { resumes.Add(1) } + + ctx := context.Background() + require.NoError(t, queue.Offer(ctx, 1)) + + waiting := make(chan error, 1) + go func() { waiting <- queue.Offer(ctx, 2) }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + + first, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(1), first) + require.NoError(t, <-waiting, "the offer the panicking callback announced still completes") + + assert.Equal(t, 1, queue.Depth(), "no phantom item") + assert.False(t, queue.Paused(), "no phantom pause") + assert.Contains(t, logs.String(), "a backlog callback panicked") + + // And the next crossing is still reported, on both sides. + second, err := queue.Take(ctx) + require.NoError(t, err) + assert.Equal(t, int64(2), second) + require.NoError(t, queue.Offer(ctx, 3)) + blocked := make(chan error, 1) + go func() { blocked <- queue.Offer(ctx, 4) }() + require.Eventually(t, queue.Paused, time.Second, time.Millisecond) + _, err = queue.Take(ctx) + require.NoError(t, err) + require.NoError(t, <-blocked) + assert.Equal(t, int32(2), pauses.Load()) + assert.Equal(t, int32(2), resumes.Load()) + assert.Equal(t, 1, queue.Depth()) +} From 5e448b8757ed8778cfb46bdf85e45c8a7152f6a0 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 04:57:39 +0200 Subject: [PATCH 48/49] Refuse a filter this lane cannot honor, and say where a panic went MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reasons filter the inbox — why an event reached you — and the account feed does not carry them. The SDK refuses them when the feed is built, but that is too late: Run starts the repair workers first, so an open loss could walk under a filter set the lane cannot honor, and a dropped dimension widens a read rather than narrowing it. Configuration that cannot mean what it says is refused before any wire work. And intake now hands the queue its logger, so the callback panic the queue contains is reported somewhere. A queue built by a caller who did not think about it would have swallowed it silently. --- internal/connector/intake.go | 16 ++++++++ internal/connector/seam_contract_test.go | 50 ++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index 1c373c690..b729c3a31 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -223,6 +223,16 @@ func New(opts Options) (*Intake, error) { if err := opts.Filters.Validate(); err != nil { return nil, fmt.Errorf("connector: intake filters: %w", err) } + if len(opts.Filters.Reasons) > 0 { + // Reasons is the inbox lane's dimension — why an event reached YOU — + // and the account feed does not carry it. The SDK refuses it when the + // feed is built, but that is too late: Run starts the repair workers + // before the feed exists, so an open loss could walk under a filter + // set the account lane cannot honor, and a dropped dimension widens a + // read rather than narrowing it. Configuration that cannot mean what + // it says fails here, before any wire work. + return nil, errors.New("connector: intake filters: reasons filter the inbox, which the account feed does not carry") + } // The filter set is the checkpoint's identity, and it is also what every // subscription, recorded loss and repair walk runs under. A caller that // kept its slices could change all of those while the key stays frozen on @@ -244,6 +254,12 @@ func New(opts Options) (*Intake, error) { if opts.Logger == nil { opts.Logger = slog.New(slog.DiscardHandler) } + if opts.Queue.Logger == nil { + // The queue reports a callback that panicked, and a queue built by a + // caller who did not think about that would report it nowhere. One + // owner, one logger. + opts.Queue.Logger = opts.Logger + } if opts.RepairInterval <= 0 { opts.RepairInterval = DefaultRepairInterval } diff --git a/internal/connector/seam_contract_test.go b/internal/connector/seam_contract_test.go index 905b9ece6..509d259fa 100644 --- a/internal/connector/seam_contract_test.go +++ b/internal/connector/seam_contract_test.go @@ -1,8 +1,10 @@ package connector import ( + "bytes" "context" "errors" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -150,3 +152,51 @@ func TestSeamCallerCancellationIsNotATransportFailure(t *testing.T) { var pollErr *eventfeed.PollError assert.False(t, errors.As(err, &pollErr)) } + +// Reasons filter the inbox — why an event reached you — and the account feed +// does not carry them. The refusal is here rather than at the feed, because +// Run starts the repair workers before the feed exists: a loss walking under +// a filter set the lane cannot honor reads WIDER than the caller asked for. +func TestIntakeRefusesAnInboxOnlyFilter(t *testing.T) { + queue, err := NewQueue(DefaultBacklogWarn, DefaultBacklogPause) + require.NoError(t, err) + + _, err = New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: newTestLedger(t), + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + Filters: eventfeed.Filters{Reasons: []string{"mention"}}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "reasons filter the inbox") +} + +// The queue reports a callback that panicked, and intake owns the logger that +// says where. +func TestIntakeGivesTheQueueItsLogger(t *testing.T) { + var logs bytes.Buffer + log := slog.New(slog.NewTextHandler(&logs, nil)) + queue, err := NewQueue(1, 4) + require.NoError(t, err) + _, err = New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: newTestLedger(t), + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + Logger: log, + }) + require.NoError(t, err) + + queue.OnWarn = func(int) { panic("a callback panics") } + require.NoError(t, queue.Offer(context.Background(), 1)) + + assert.Contains(t, logs.String(), "a backlog callback panicked") +} From d13fda82a8b9e4b060f8b849de0255ca4c1ed4c7 Mon Sep 17 00:00:00 2001 From: Jorge Manrubia Date: Thu, 17 Sep 2026 05:07:03 +0200 Subject: [PATCH 49/49] Adopt the queue's logger under the queue's lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit had intake write the queue's logger field while the callback path read it unlocked, and a queue can already be live when intake adopts it — admission takes from the same one. The race detector found it three times in three on a queue in use during construction. The logger is now held under the queue's own lock: SetLogger for a caller, an adopt that checks and sets in one step for intake, and the one read made with the lock released around the callback it reports on. --- internal/connector/intake.go | 12 +++--- internal/connector/queue.go | 32 ++++++++++++++-- internal/connector/queue_test.go | 2 +- internal/connector/seam_contract_test.go | 48 ++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/internal/connector/intake.go b/internal/connector/intake.go index b729c3a31..e47d2e5b3 100644 --- a/internal/connector/intake.go +++ b/internal/connector/intake.go @@ -254,12 +254,12 @@ func New(opts Options) (*Intake, error) { if opts.Logger == nil { opts.Logger = slog.New(slog.DiscardHandler) } - if opts.Queue.Logger == nil { - // The queue reports a callback that panicked, and a queue built by a - // caller who did not think about that would report it nowhere. One - // owner, one logger. - opts.Queue.Logger = opts.Logger - } + // The queue reports a callback that panicked, and a queue built by a + // caller who did not think about that would report it nowhere. It is + // adopted under the queue's own lock, because the queue may already be + // in use — admission takes from it — and a logger the caller did set is + // kept. + opts.Queue.adoptLogger(opts.Logger) if opts.RepairInterval <= 0 { opts.RepairInterval = DefaultRepairInterval } diff --git a/internal/connector/queue.go b/internal/connector/queue.go index cb3eab04f..37132d094 100644 --- a/internal/connector/queue.go +++ b/internal/connector/queue.go @@ -68,8 +68,28 @@ type Queue struct { // it stops. The feed is not being consumed in between. OnPause func(depth int) OnResume func(depth int) - // Logger reports a callback that panicked. Optional; silent when unset. - Logger *slog.Logger + // log reports a callback that panicked. Held under edges: a queue can be + // adopted by intake while another component is already using it, and a + // field written by one and read by the other is a race. + log *slog.Logger +} + +// SetLogger sets where a callback that panicked is reported. Safe to call +// while the queue is in use. +func (q *Queue) SetLogger(log *slog.Logger) { + q.edges.Lock() + defer q.edges.Unlock() + q.log = log +} + +// adoptLogger sets log only if nobody has set one, as one step: a check and a +// set taken apart would let two adopters both find it unset. +func (q *Queue) adoptLogger(log *slog.Logger) { + q.edges.Lock() + defer q.edges.Unlock() + if q.log == nil { + q.log = log + } } // NewQueue builds a queue that warns at warnAt and pauses the feed at pauseAt. @@ -290,9 +310,13 @@ func (q *Queue) fire(edge queueEdge) { } } +// logger is read under edges, so it must never be called holding them. Its one +// caller, fire, runs with the lock released. func (q *Queue) logger() *slog.Logger { - if q.Logger != nil { - return q.Logger + q.edges.Lock() + defer q.edges.Unlock() + if q.log != nil { + return q.log } return slog.New(slog.DiscardHandler) } diff --git a/internal/connector/queue_test.go b/internal/connector/queue_test.go index f52ed0228..8a764b063 100644 --- a/internal/connector/queue_test.go +++ b/internal/connector/queue_test.go @@ -259,7 +259,7 @@ func TestAPanickingPauseCallbackLeavesNoPhantomBacklog(t *testing.T) { require.NoError(t, err) var logs bytes.Buffer - queue.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + queue.SetLogger(slog.New(slog.NewTextHandler(&logs, nil))) var pauses, resumes atomic.Int32 queue.OnPause = func(int) { if pauses.Add(1) == 1 { diff --git a/internal/connector/seam_contract_test.go b/internal/connector/seam_contract_test.go index 509d259fa..a41d9ebf0 100644 --- a/internal/connector/seam_contract_test.go +++ b/internal/connector/seam_contract_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" @@ -200,3 +201,50 @@ func TestIntakeGivesTheQueueItsLogger(t *testing.T) { assert.Contains(t, logs.String(), "a backlog callback panicked") } + +// A queue can already be in use when intake adopts it — admission takes from +// the same queue — so adopting its logger must not race the callback path that +// reads it. +func TestAdoptingAQueueInUseDoesNotRace(t *testing.T) { + queue, err := NewQueue(1, 1000) + require.NoError(t, err) + queue.OnWarn = func(int) { panic("a callback panics") } + queue.OnRecover = func(int) { panic("a callback panics") } + ledger := newTestLedger(t) + + ctx := context.Background() + stop := make(chan struct{}) + var wg sync.WaitGroup + // The queue is in use, and every crossing reads its logger, for the + // whole time intake is adopting it. + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + if err := queue.Offer(ctx, 1); err != nil { + return + } + if _, err := queue.Take(ctx); err != nil { + return + } + } + }) + for range 20 { + _, err := New(Options{ + Origin: "https://3.basecampapi.com", + AccountID: "2914079", + ConsumerNamespace: "connector-test", + Ledger: ledger, + Queue: queue, + Minter: stubMinter{}, + Polls: &scriptedPolls{}, + Logger: slog.New(slog.DiscardHandler), + }) + require.NoError(t, err) + } + close(stop) + wg.Wait() +}