diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c3a4fc9..6b3f90c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,9 +33,39 @@ jobs: - run: go build ./... - run: go vet ./... - # Deterministic tests. e2e_test.go skips without OPENROUTER_API_KEY. + # Deterministic tests. e2e_test.go skips without OPENROUTER_API_KEY + + # OPENROUTER_AGENT_E2E=1, so this stays free and hermetic. - run: go test ./... + # Live end-to-end tests against the real OpenRouter API: streaming, a real + # tool round, approval pause/resume, lifecycle hooks, state serialization + # round-trip. Costs a few cents per run (small model, short prompts). + # + # Warns and exits 0 when the secret is missing (e.g. PRs from forks, where + # GitHub withholds secrets) instead of failing — same pattern as upstream + # typescript-agent's e2e job. + e2e: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: true + + - name: Live e2e tests + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENROUTER_AGENT_E2E: "1" + run: | + if [ -z "$OPENROUTER_API_KEY" ]; then + echo "::warning::OPENROUTER_API_KEY is not set; skipping live e2e tests." + exit 0 + fi + go test -run 'TestE2E' -v . + # Reports the port's own mechanical gate. Advisory here, BLOCKING inside the # sync job (scripts/upstream) where it gates whether state.yaml advances. # diff --git a/e2e_test.go b/e2e_test.go index 2726f15..d150e96 100644 --- a/e2e_test.go +++ b/e2e_test.go @@ -1,36 +1,362 @@ package agent +// Live end-to-end tests against the real OpenRouter API, mirroring upstream +// typescript-agent's packages/agent/tests/e2e coverage: streaming, a real +// tool round, approval pause/resume across two CallModel invocations, +// lifecycle hooks firing on live traffic, and conversation-state +// serialization surviving a round trip. +// +// Skipped without OPENROUTER_API_KEY. These assert behavior (a tool ran, a +// hook fired, state advanced), never model quality, so prompts pin outputs +// as hard as possible. Model overridable via OPENROUTER_E2E_MODEL. + import ( "context" "os" "strings" "testing" "time" + + "github.com/OpenRouterTeam/go-sdk/models/components" ) -func TestE2ESimpleResponsesCall(t *testing.T) { +func e2eModel() string { + if m := os.Getenv("OPENROUTER_E2E_MODEL"); m != "" { + return m + } + return "anthropic/claude-haiku-4.5" +} + +func e2eSetup(t *testing.T) (context.Context, *OpenRouter) { + t.Helper() if os.Getenv("OPENROUTER_API_KEY") == "" { t.Skip("OPENROUTER_API_KEY is not set") } + // Double gate: a plain `go test ./...` with a key exported must not spend + // money by surprise. CI's e2e job sets this explicitly. if os.Getenv("OPENROUTER_AGENT_E2E") != "1" { t.Skip("set OPENROUTER_AGENT_E2E=1 to run live OpenRouter e2e") } - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + return ctx, NewOpenRouter(OpenRouterOptions{}) +} + +// requiredToolChoice pins the first turn to a tool call so model +// nondeterminism can't skip an approval pause the test is asserting. +func requiredToolChoice() *components.OpenAIResponsesToolChoiceUnion { + choice := components.CreateOpenAIResponsesToolChoiceUnionOpenAIResponsesToolChoiceRequired( + components.OpenAIResponsesToolChoiceRequiredRequired, + ) + return &choice +} + +type e2eToolInput struct { + Text string `json:"text,omitempty"` + ID int `json:"id,omitempty"` +} + +type e2eMemoryState struct { + current *ConversationState + saves int +} + +func (a *e2eMemoryState) Load(context.Context) (*ConversationState, error) { + return a.current, nil +} + +func (a *e2eMemoryState) Save(_ context.Context, s ConversationState) error { + a.saves++ + a.current = &s + return nil +} + +func TestE2ETextAndStreamAgree(t *testing.T) { + ctx, client := e2eSetup(t) - client := NewOpenRouter(OpenRouterOptions{}) result, err := CallModel(ctx, client, CallModelInput{ - Model: "openai/gpt-4o-mini", - Input: "Reply with exactly: OK", + Model: e2eModel(), + Input: "Reply with exactly the word: pong", }) if err != nil { t.Fatal(err) } + + stream, wait := result.TextStream(ctx) + var chunks []string + for chunk := range stream { + chunks = append(chunks, chunk) + } + if err := wait(); err != nil { + t.Fatal(err) + } text, err := result.Text(ctx) if err != nil { t.Fatal(err) } - if !strings.Contains(strings.ToUpper(text), "OK") { - t.Fatalf("expected OK in response, got %q", text) + + if !strings.Contains(strings.ToLower(text), "pong") { + t.Fatalf("expected pong in response, got %q", text) + } + if joined := strings.Join(chunks, ""); joined != text { + t.Fatalf("stream chunks %q != final text %q", joined, text) + } +} + +func TestE2EToolLoopExecutesAndFeedsResultBack(t *testing.T) { + ctx, client := e2eSetup(t) + + var calls int + secretTool := MustNewTool(ToolConfig[e2eToolInput]{ + Name: "get_secret", + Description: "Returns the secret code. Call this to answer any question about the secret code.", + Execute: func(context.Context, e2eToolInput, ToolExecuteContext) (any, error) { + calls++ + return map[string]any{"secret": "BANANA-42"}, nil + }, + }) + + result, err := CallModel(ctx, client, CallModelInput{ + Model: e2eModel(), + Input: "What is the secret code? Use the get_secret tool, then repeat the code back verbatim.", + Tools: []Tool{secretTool}, + }) + if err != nil { + t.Fatal(err) + } + text, err := result.Text(ctx) + if err != nil { + t.Fatal(err) + } + + if calls == 0 { + t.Fatal("model never called the tool") + } + if !strings.Contains(text, "BANANA-42") { + t.Fatalf("tool output did not reach the final answer: %q", text) + } +} + +func TestE2EApprovalPauseAndResumeAcrossCalls(t *testing.T) { + ctx, client := e2eSetup(t) + + var executed int + deleteTool := MustNewTool(ToolConfig[e2eToolInput]{ + Name: "delete_record", + Description: "Deletes the record. Requires approval.", + RequireApproval: true, + Execute: func(context.Context, e2eToolInput, ToolExecuteContext) (any, error) { + executed++ + return map[string]any{"deleted": true}, nil + }, + }) + + accessor := &e2eMemoryState{} + first, err := CallModel(ctx, client, CallModelInput{ + Model: e2eModel(), + Input: "Delete the record with id 7 using the delete_record tool.", + Tools: []Tool{deleteTool}, + StateAccessor: accessor, + Request: components.ResponsesRequest{ToolChoice: requiredToolChoice()}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := first.Response(ctx); err != nil { + t.Fatal(err) + } + + if accessor.current == nil { + t.Fatal("no state was saved") + } + if accessor.current.Status != ConversationStatusAwaitingApproval { + t.Fatalf("expected awaiting_approval, got %s", accessor.current.Status) + } + if executed != 0 { + t.Fatal("tool must not run before approval") + } + + pending, err := first.PendingToolCalls(ctx) + if err != nil { + t.Fatal(err) + } + if len(pending) != 1 { + t.Fatalf("expected 1 pending call, got %d", len(pending)) + } + + resumed, err := CallModel(ctx, client, CallModelInput{ + Model: e2eModel(), + Tools: []Tool{deleteTool}, + StateAccessor: accessor, + ApproveToolCalls: []string{pending[0].CallID}, + }) + if err != nil { + t.Fatal(err) + } + text, err := resumed.Text(ctx) + if err != nil { + t.Fatal(err) + } + + if executed != 1 { + t.Fatalf("approved tool executed %d times, want 1", executed) + } + if accessor.current.Status != ConversationStatusComplete { + t.Fatalf("expected complete, got %s", accessor.current.Status) + } + if strings.TrimSpace(text) == "" { + t.Fatal("expected a non-empty final answer after resume") + } +} + +func TestE2EHooksFireOnRealTraffic(t *testing.T) { + ctx, client := e2eSetup(t) + + var fired []string + var totals *SessionUsageTotals + + manager := NewHooksManager() + manager.OnSessionStart(HookEntry[SessionStartPayload, EmptyHookResult]{ + Handler: func(_ SessionStartPayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + fired = append(fired, "SessionStart") + return HookHandlerResult[EmptyHookResult]{}, nil + }, + }) + manager.OnPreToolUse(HookEntry[PreToolUsePayload, PreToolUseResult]{ + Handler: func(_ PreToolUsePayload, _ LifecycleHookContext) (HookHandlerResult[PreToolUseResult], error) { + fired = append(fired, "PreToolUse") + return HookHandlerResult[PreToolUseResult]{}, nil + }, + }) + manager.OnPostToolUse(HookEntry[PostToolUsePayload, EmptyHookResult]{ + Handler: func(_ PostToolUsePayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + fired = append(fired, "PostToolUse") + return HookHandlerResult[EmptyHookResult]{}, nil + }, + }) + manager.OnPostModelCall(HookEntry[PostModelCallPayload, EmptyHookResult]{ + Handler: func(_ PostModelCallPayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + fired = append(fired, "PostModelCall") + return HookHandlerResult[EmptyHookResult]{}, nil + }, + }) + manager.OnSessionEnd(HookEntry[SessionEndPayload, EmptyHookResult]{ + Handler: func(payload SessionEndPayload, _ LifecycleHookContext) (HookHandlerResult[EmptyHookResult], error) { + fired = append(fired, "SessionEnd") + totals = payload.TotalUsage + return HookHandlerResult[EmptyHookResult]{}, nil + }, + }) + + echoTool := MustNewTool(ToolConfig[e2eToolInput]{ + Name: "echo", + Description: "Echoes back the given text.", + Execute: func(_ context.Context, in e2eToolInput, _ ToolExecuteContext) (any, error) { + return map[string]any{"echoed": in.Text}, nil + }, + }) + + result, err := CallModel(ctx, client, CallModelInput{ + Model: e2eModel(), + Input: "Use the echo tool with text 'hi', then say done.", + Tools: []Tool{echoTool}, + Hooks: manager, + }) + if err != nil { + t.Fatal(err) + } + if _, err := result.Text(ctx); err != nil { + t.Fatal(err) + } + + if len(fired) == 0 || fired[0] != "SessionStart" { + t.Fatalf("SessionStart must fire first, got %v", fired) + } + if fired[len(fired)-1] != "SessionEnd" { + t.Fatalf("SessionEnd must fire last, got %v", fired) + } + for _, want := range []string{"PreToolUse", "PostToolUse", "PostModelCall"} { + found := false + for _, name := range fired { + if name == want { + found = true + break + } + } + if !found { + t.Fatalf("hook %s never fired: %v", want, fired) + } + } + // SessionEnd carries aggregated real usage — a live call must cost tokens. + if totals == nil || totals.TotalTokens <= 0 || totals.ModelCalls <= 0 { + t.Fatalf("SessionEnd usage totals empty: %+v", totals) + } +} + +func TestE2EStateSerializationRoundTripResumes(t *testing.T) { + ctx, client := e2eSetup(t) + + var executed int + launchTool := MustNewTool(ToolConfig[e2eToolInput]{ + Name: "launch", + Description: "Launches the rocket. Requires approval.", + RequireApproval: true, + Execute: func(context.Context, e2eToolInput, ToolExecuteContext) (any, error) { + executed++ + return map[string]any{"launched": true}, nil + }, + }) + + accessor := &e2eMemoryState{} + first, err := CallModel(ctx, client, CallModelInput{ + Model: e2eModel(), + Input: "Launch the rocket using the launch tool.", + Tools: []Tool{launchTool}, + StateAccessor: accessor, + Request: components.ResponsesRequest{ToolChoice: requiredToolChoice()}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := first.Response(ctx); err != nil { + t.Fatal(err) + } + if accessor.current == nil || accessor.current.Status != ConversationStatusAwaitingApproval { + t.Fatalf("expected awaiting_approval pause, got %+v", accessor.current) + } + pending, err := first.PendingToolCalls(ctx) + if err != nil || len(pending) != 1 { + t.Fatalf("expected 1 pending call, got %d (err %v)", len(pending), err) + } + + // Round-trip through the wire format, as a durable store would. + raw, err := SerializeConversationState(*accessor.current) + if err != nil { + t.Fatal(err) + } + restoredState, err := DeserializeConversationState(raw) + if err != nil { + t.Fatal(err) + } + restored := &e2eMemoryState{current: &restoredState} + + resumed, err := CallModel(ctx, client, CallModelInput{ + Model: e2eModel(), + Tools: []Tool{launchTool}, + StateAccessor: restored, + ApproveToolCalls: []string{pending[0].CallID}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := resumed.Text(ctx); err != nil { + t.Fatal(err) + } + + if executed != 1 { + t.Fatalf("approved tool executed %d times after round-trip, want 1", executed) + } + if restored.current.Status != ConversationStatusComplete { + t.Fatalf("expected complete, got %s", restored.current.Status) } } diff --git a/flatten_message_content_test.go b/flatten_message_content_test.go new file mode 100644 index 0000000..1f47b1a --- /dev/null +++ b/flatten_message_content_test.go @@ -0,0 +1,80 @@ +package agent + +// Deterministic regression guard for flattenMessageContentForUnion, the +// workaround for go-sdk v0.5.4's InputsUnion1 unmarshaller panicking on +// array-content assistant messages. The live e2e suite exercises it against +// real traffic; these hardcoded shapes make regressions in the workaround +// itself visible on every PR without spending API credits. + +import ( + "encoding/json" + "testing" + + "github.com/OpenRouterTeam/go-sdk/models/components" +) + +func TestFlattenMessageContentForUnion(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + { + name: "array of output_text blocks flattens to concatenated string", + in: `{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Hello, ","annotations":[]},{"type":"output_text","text":"world"}],"id":"msg_1","status":"completed"}`, + want: `{"content":"Hello, world","id":"msg_1","role":"assistant","status":"completed","type":"message"}`, + }, + { + name: "string content passes through untouched", + in: `{"type":"message","role":"assistant","content":"hi","id":"msg_1"}`, + want: `{"type":"message","role":"assistant","content":"hi","id":"msg_1"}`, + }, + { + name: "non-message type passes through untouched", + in: `{"type":"function_call","id":"fc_1","call_id":"call_1","name":"t","arguments":"{}"}`, + want: `{"type":"function_call","id":"fc_1","call_id":"call_1","name":"t","arguments":"{}"}`, + }, + { + name: "non-output_text block bails and returns original", + in: `{"type":"message","role":"user","content":[{"type":"input_image","image_url":"https://example.com/x.png"}],"id":"msg_2"}`, + want: `{"type":"message","role":"user","content":[{"type":"input_image","image_url":"https://example.com/x.png"}],"id":"msg_2"}`, + }, + { + name: "invalid JSON returns original bytes", + in: `{not json`, + want: `{not json`, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := string(flattenMessageContentForUnion([]byte(tc.in))) + // Compare semantically where both sides are JSON; byte-for-byte otherwise. + var gotAny, wantAny any + if json.Unmarshal([]byte(got), &gotAny) == nil && json.Unmarshal([]byte(tc.want), &wantAny) == nil { + gb, _ := json.Marshal(gotAny) + wb, _ := json.Marshal(wantAny) + if string(gb) != string(wb) { + t.Fatalf("got %s, want %s", gb, wb) + } + return + } + if got != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } +} + +// The end-to-end property the workaround exists for: an array-content +// assistant message, after flattening, decodes into InputsUnion1 without +// panicking. (Without the flatten this exact shape reflect-panics in +// go-sdk v0.5.4 — the reason every live tool round crashed.) +func TestFlattenedMessageDecodesIntoInputsUnion(t *testing.T) { + raw := []byte(`{"type":"message","role":"assistant","content":[{"type":"output_text","text":"hi","annotations":[]}],"id":"msg_1","status":"completed"}`) + flattened := flattenMessageContentForUnion(raw) + var u components.InputsUnion1 + if err := json.Unmarshal(flattened, &u); err != nil { + t.Fatalf("flattened message failed to decode into InputsUnion1: %v", err) + } +} diff --git a/model_result.go b/model_result.go index 27e2c6d..c955511 100644 --- a/model_result.go +++ b/model_result.go @@ -133,9 +133,11 @@ func CallModel(ctx context.Context, client ResponseSender, input CallModelInput) } else if len(state.Messages) > 0 { req.Input = openrouter.Pointer(components.CreateInputsUnionArrayOfInputsUnion1(state.Messages)) } - if state.PreviousResponseID != nil { - req.PreviousResponseID = optionalnullable.From(state.PreviousResponseID) - } + // PreviousResponseID is tracked in state for observability but never sent + // on the wire: this port always sends full message history (mirroring + // upstream, which stores previousResponseId in state only), and the live + // Responses API rejects previous_response_id on stateless requests + // ("expected null, received string"). if input.AdditionalInstructions != "" { req.Instructions = optionalnullable.From(openrouter.String(input.AdditionalInstructions)) } @@ -459,7 +461,6 @@ func (m *ModelResult) run() { base = append(base, outputs...) req.Input = openrouter.Pointer(components.CreateInputsUnionArrayOfInputsUnion1(base)) if resp.ID != "" { - req.PreviousResponseID = optionalnullable.From(openrouter.String(resp.ID)) m.state.PreviousResponseID = &resp.ID } } @@ -648,9 +649,6 @@ func (m *ModelResult) prepareResumeRequest(req *components.ResponsesRequest) (bo m.state.UnsentToolResults = nil m.state.Status = ConversationStatusInProgress req.Input = openrouter.Pointer(components.CreateInputsUnionArrayOfInputsUnion1(items)) - if m.state.PreviousResponseID != nil { - req.PreviousResponseID = optionalnullable.From(m.state.PreviousResponseID) - } return false, nil } @@ -1383,6 +1381,7 @@ func responseInputItemsWithError(resp components.OpenResponsesResult) ([]compone if err != nil { return nil, fmt.Errorf("marshal response output item: %w", err) } + b = flattenMessageContentForUnion(b) var input components.InputsUnion1 if err := json.Unmarshal(b, &input); err != nil { return nil, fmt.Errorf("convert response output item to input item: %w", err) @@ -1392,6 +1391,47 @@ func responseInputItemsWithError(resp components.OpenResponsesResult) ([]compone return items, nil } +// flattenMessageContentForUnion rewrites a marshaled `message` item whose +// content is an array of output_text blocks into the equivalent +// string-content form. go-sdk v0.5.4's InputsUnion1 unmarshaller panics +// (reflect on slice value) on array-content assistant messages — the exact +// shape every live response echoes back on follow-up turns — but accepts the +// same message with string content. Text is preserved; annotations are not +// (they are response-side metadata the next request does not need). Remove +// once the pinned go-sdk's union decoding handles array content. +func flattenMessageContentForUnion(b []byte) []byte { + var raw map[string]any + if err := json.Unmarshal(b, &raw); err != nil { + return b + } + if raw["type"] != "message" { + return b + } + blocks, ok := raw["content"].([]any) + if !ok { + return b + } + var text strings.Builder + for _, block := range blocks { + m, ok := block.(map[string]any) + if !ok { + return b // unknown block shape: leave untouched + } + if m["type"] != "output_text" { + return b // non-text content: leave untouched + } + if s, ok := m["text"].(string); ok { + text.WriteString(s) + } + } + raw["content"] = text.String() + flattened, err := json.Marshal(raw) + if err != nil { + return b + } + return flattened +} + func appendResponseItemsToState(state ConversationState, resp components.OpenResponsesResult, items []components.InputsUnion1) ConversationState { state = AppendToMessages(state, items...) if resp.ID != "" { diff --git a/model_result_test.go b/model_result_test.go index 06c9e8f..ac3ea8f 100644 --- a/model_result_test.go +++ b/model_result_test.go @@ -114,8 +114,11 @@ func TestApprovalResumeReplaysFunctionCallBeforeOutput(t *testing.T) { if len(items) < 2 || (items[0].FunctionCallItem == nil && items[0].OutputFunctionCallItem == nil) || items[1].FunctionCallOutputItem == nil { t.Fatalf("resume input must replay function_call before function_call_output: %#v", items) } - if previousID, ok := resumeSender.requests[0].PreviousResponseID.GetOrZero(); !ok || previousID == "" { - t.Fatalf("resume request should carry previous_response_id") + // previous_response_id must NOT go on the wire: the live Responses API + // rejects it on stateless requests ("expected null, received string"). + // It is tracked in state only, mirroring upstream. + if _, ok := resumeSender.requests[0].PreviousResponseID.GetOrZero(); ok { + t.Fatalf("resume request must not carry previous_response_id on the wire") } }