Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 16 additions & 40 deletions internal/execution/bifrost/conversion_fidelity.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,13 @@ func finishConvertedPreparation(spec execution.AttemptSpec, providerKind channel
(spec.Operation != execution.OperationChatCompletion && spec.Operation != execution.OperationResponsesCreate) {
return prepared, nil
}
var toolConstraintsValid bool
var needsToolHistoryCheck bool
Comment thread
coderabbitai[bot] marked this conversation as resolved.
prepared, needsToolHistoryCheck, toolConstraintsValid = prepareConvertedToolConstraints(spec.ClientProtocol, providerKind, spec.UpstreamModel, prepared)
if !toolConstraintsValid {
failure := notSentConversionFailure(execution.ErrorCodeCriticalSemanticLoss, "conversion cannot preserve requested tools or tool choice")
return preparedAttempt{}, &failure
}
vertexChat := providerKind == channel.ProviderGoogleVertex && vertexUsesChatFallback(spec.UpstreamModel)
if providerKind == channel.ProviderAnthropic || providerKind == channel.ProviderGemini ||
providerKind == channel.ProviderAWSBedrock || (providerKind == channel.ProviderGoogleVertex && !vertexChat) {
Expand All @@ -24,16 +31,23 @@ func finishConvertedPreparation(spec execution.AttemptSpec, providerKind channel
chatFallback := providerKind == channel.ProviderOpenAICompatible || providerKind == channel.ProviderGroq ||
providerKind == channel.ProviderDeepSeek || vertexChat
if chatFallback {
if chatFallbackDropsTools(prepared.responsesRequest) {
dropsTools, newlyAllowed := chatFallbackToolCompatibility(prepared.responsesRequest)
needsToolHistoryCheck = needsToolHistoryCheck || newlyAllowed
Comment thread
tbphp marked this conversation as resolved.
// Compatible 保持 SDK 的尽力转换行为:不支持的工具及选择可被过滤,
// 不因此拒绝整个请求;普通函数白名单仍由前面的准备阶段适配。
if dropsTools && providerKind != channel.ProviderOpenAICompatible {
failure := notSentConversionFailure(execution.ErrorCodeCriticalSemanticLoss, "Chat conversion cannot preserve requested tools or tool choice")
return preparedAttempt{}, &failure
}
normalizeChatFallbackToolMode(prepared.responsesRequest)
if providerKind == channel.ProviderDeepSeek && deepSeekConversionDisablesThinking(prepared.responsesRequest) {
failure := notSentConversionFailure(execution.ErrorCodeCriticalSemanticLoss, "DeepSeek conversion cannot preserve explicit thinking with this tool choice or history")
return preparedAttempt{}, &failure
}
}
if needsToolHistoryCheck && !convertedTargetPreservesToolHistory(providerKind, spec.UpstreamModel, prepared.responsesRequest) {
failure := notSentConversionFailure(execution.ErrorCodeCriticalSemanticLoss, "conversion cannot preserve requested tool history")
return preparedAttempt{}, &failure
}
wantSystems := dialect.CountMidConversationSystemMessages(spec.ClientProtocol, spec.Body)
if wantSystems == 0 {
return prepared, nil
Expand Down Expand Up @@ -95,44 +109,6 @@ func preserveResponsesGlobalInstructions(request *schemas.BifrostResponsesReques
request.Params.Instructions = nil
}

func chatFallbackDropsTools(request *schemas.BifrostResponsesRequest) bool {
if request == nil || request.Params == nil {
return false
}
// 仅检查 SDK 的参数映射;不重复转换消息,也不自行维护工具能力表。
converted := (&schemas.BifrostResponsesRequest{Params: request.Params}).ToChatRequest()
if len(converted.Params.Tools) != len(request.Params.Tools) {
return true
}
choice := request.Params.ToolChoice
if choice == nil || converted.Params.ToolChoice != nil {
return false
}
// 没有声明工具时,省略 auto/none 不会改变行为;强制调用约束仍不能丢失。
mode := ""
if choice.ResponsesToolChoiceStr != nil {
mode = *choice.ResponsesToolChoiceStr
} else if value := choice.ResponsesToolChoiceStruct; value != nil && len(value.Tools) == 0 && value.Name == nil && value.ServerLabel == nil {
mode = string(value.Type)
}
return mode != "auto" && mode != "none"
}

func normalizeChatFallbackToolMode(request *schemas.BifrostResponsesRequest) {
if request == nil || request.Params == nil || request.Params.ToolChoice == nil {
return
}
choice := request.Params.ToolChoice.ResponsesToolChoiceStruct
if choice == nil || len(choice.Tools) != 0 || choice.Name != nil || choice.ServerLabel != nil {
return
}
switch choice.Type {
case schemas.ResponsesToolChoiceTypeAuto, schemas.ResponsesToolChoiceTypeNone, schemas.ResponsesToolChoiceTypeRequired:
// SDK 的结构化 mode:auto 会误映射为 any;采用同义字符串保留原始约束。
request.Params.ToolChoice = &schemas.ResponsesToolChoice{ResponsesToolChoiceStr: schemas.Ptr(string(choice.Type))}
}
}

func deepSeekConversionDisablesThinking(request *schemas.BifrostResponsesRequest) bool {
if request == nil || request.Params == nil || request.Params.Reasoning == nil {
return false
Expand Down
4 changes: 2 additions & 2 deletions internal/execution/bifrost/conversion_fidelity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ func TestChatFallbackDoesNotDropRequestedTools(t *testing.T) {
{"no tools required", `[]`, `"required"`, true},
{"web search", `[{"type":"web_search_preview"}]`, `"auto"`, true},
{"function", `[{"type":"function","name":"lookup","parameters":{"type":"object"}}]`, `"required"`, false},
{"allowed tools", `[{"type":"function","name":"lookup","parameters":{"type":"object"}}]`, `{"type":"allowed_tools","mode":"required","tools":[{"type":"function","name":"lookup"}]}`, true},
{"allowed tools", `[{"type":"function","name":"lookup","parameters":{"type":"object"}}]`, `{"type":"allowed_tools","mode":"required","tools":[{"type":"function","name":"lookup"}]}`, false},
} {
for _, stream := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/%s/stream=%t", channelID, test.name, stream), func(t *testing.T) {
Expand All @@ -199,7 +199,7 @@ func TestChatFallbackDoesNotDropRequestedTools(t *testing.T) {
} else {
evidence = runtime.Execute(t.Context(), spec).Error
}
if test.wantError {
if test.wantError && channelID != channel.OpenAICompatible {
if calls.Load() != 0 || evidence == nil || evidence.Kind != execution.ErrorKindConversionUnsupported || evidence.Code != execution.ErrorCodeCriticalSemanticLoss {
t.Fatalf("tool requirement was lost: calls=%d error=%+v", calls.Load(), evidence)
}
Expand Down
42 changes: 42 additions & 0 deletions internal/execution/bifrost/count_tokens_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"net/http/httptest"
"testing"

"github.com/tidwall/gjson"

"gpt-load/internal/channel"
"gpt-load/internal/execution"
"gpt-load/internal/protocol"
Expand Down Expand Up @@ -183,6 +185,46 @@ func TestConvertedCountTokensUsesBifrostProviderEndpointAndClientShape(t *testin
}
}

func TestConvertedCountTokensUsesGenerationToolConstraints(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name string
clientProtocol protocol.Protocol
operation execution.Operation
path string
body string
}{
{
name: "OpenAI Responses", clientProtocol: protocol.OpenAIResponses,
operation: execution.OperationResponsesInputTokens, path: "/v1/responses/input_tokens",
body: `{"model":"client-model","input":"hello","store":false,"tools":[{"type":"function","name":"lookup"},{"type":"function","name":"summarize"},{"type":"function","name":"remove_record"}],"tool_choice":{"type":"allowed_tools","mode":"required","tools":[{"type":"function","name":"lookup"},{"type":"function","name":"summarize"}]}}`,
},
{
name: "Gemini", clientProtocol: protocol.Gemini,
operation: execution.OperationCountTokens, path: "/v1beta/models/client-model:countTokens",
body: `{"contents":[{"role":"user","parts":[{"text":"hello"}]}],"tools":[{"functionDeclarations":[{"name":"lookup"},{"name":"summarize"},{"name":"remove_record"}]}],"toolConfig":{"functionCallingConfig":{"mode":"ANY","allowedFunctionNames":["lookup","summarize"]}}}`,
},
} {
t.Run(test.name, func(t *testing.T) {
runtime := newProtocolTestRuntime(t, testRuntimeOptions{})
spec := countTokensSpec(channel.Anthropic, test.clientProtocol, test.path, test.body, execution.RouteConverted)
spec.Operation = test.operation
spec = freezeTestAttempt(spec)
prepared, failure := runtime.prepare(spec, false)
if failure != nil {
t.Fatalf("prepare token-count request: %+v", failure.Error)
}
wire := convertedToolTargetWire(t, channel.ProviderAnthropic, spec.UpstreamModel, prepared.countTokensRequest)
if gjson.GetBytes(wire, "tools.#").Int() != 2 ||
gjson.GetBytes(wire, "tools.0.name").String() != "lookup" ||
gjson.GetBytes(wire, "tools.1.name").String() != "summarize" ||
gjson.GetBytes(wire, "tool_choice.type").String() != "any" {
t.Fatalf("token-count constraints differ from generation: %s", wire)
}
})
}
}

func TestCountTokensUnsupportedIsRequestRejected(t *testing.T) {
t.Parallel()

Expand Down
17 changes: 17 additions & 0 deletions internal/execution/bifrost/executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -774,6 +774,23 @@ func (r *Runtime) prepare(spec execution.AttemptSpec, stream bool) (preparedAtte
failure := notSentUnaryFailure(execution.ErrorKindInvalidRequest, conversionErr.Error())
return preparedAttempt{}, &failure
}
if spec.RouteMode == execution.RouteConverted {
toolPrepared, needsToolHistoryCheck, toolConstraintsValid := prepareConvertedToolConstraints(
spec.ClientProtocol,
providerKind,
spec.UpstreamModel,
preparedAttempt{responsesRequest: request},
)
if !toolConstraintsValid {
failure := notSentConversionFailure(execution.ErrorCodeCriticalSemanticLoss, "conversion cannot preserve requested tools or tool choice")
return preparedAttempt{}, &failure
}
if needsToolHistoryCheck && !convertedTargetPreservesToolHistory(providerKind, spec.UpstreamModel, toolPrepared.responsesRequest) {
failure := notSentConversionFailure(execution.ErrorCodeCriticalSemanticLoss, "conversion cannot preserve requested tool history")
return preparedAttempt{}, &failure
}
request = toolPrepared.responsesRequest
}
typedURL, upstreamProtocol, targetErr := countTokensTypedTarget(
providerKind,
customTargetBaseURL,
Expand Down
Loading