Fix reasoning-model temperature quirk matching and validation
- Omit temperature for OpenRouter's namespaced OpenAI reasoning models (openai/o1, openai/o1-mini, openai/o3-mini, openai/o1:online). - Match o1/o3 only as the bare family name or with a "-" variant, so o123 and o3custom keep their temperature. - Validate the caller's request before clearing temperature, so a negative temperature is still a validation error, not a sent request. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -46,14 +46,16 @@ const (
|
|||||||
KeyNone
|
KeyNone
|
||||||
)
|
)
|
||||||
|
|
||||||
// TemperatureRejectingModels is the one list of model-id patterns that
|
// The reasoning models that reject the temperature field are OpenAI's o1
|
||||||
// reject the temperature field: OpenAI's o1 and o3 reasoning models. A
|
// and o3 families: the bare family name or the name followed by a "-"
|
||||||
// preset whose OmitTemperature holds it never sends a temperature to a
|
// variant (o1, o1-mini, o3-2025-04-16), never a longer word such as o123.
|
||||||
// matching model.
|
// OpenRouter names them with an "openai/" prefix and may add a ":" variant
|
||||||
var TemperatureRejectingModels = []*regexp.Regexp{
|
// (openai/o1:online). A preset whose OmitTemperature holds a pattern never
|
||||||
regexp.MustCompile(`^o1`),
|
// sends a temperature to a matching model.
|
||||||
regexp.MustCompile(`^o3`),
|
var (
|
||||||
}
|
openAIReasoningModels = regexp.MustCompile(`^o[13](-.*)?$`)
|
||||||
|
openRouterReasoningModels = regexp.MustCompile(`^openai/o[13]([-:].*)?$`)
|
||||||
|
)
|
||||||
|
|
||||||
// Preset is one vendor's fixed configuration.
|
// Preset is one vendor's fixed configuration.
|
||||||
type Preset struct {
|
type Preset struct {
|
||||||
@@ -127,7 +129,7 @@ var all = map[string]Preset{
|
|||||||
Key: KeyRequired,
|
Key: KeyRequired,
|
||||||
StructuredOutput: openai.JSONSchema,
|
StructuredOutput: openai.JSONSchema,
|
||||||
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema},
|
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema},
|
||||||
OmitTemperature: TemperatureRejectingModels,
|
OmitTemperature: []*regexp.Regexp{openAIReasoningModels},
|
||||||
},
|
},
|
||||||
"openrouter": {
|
"openrouter": {
|
||||||
Name: "openrouter",
|
Name: "openrouter",
|
||||||
@@ -139,6 +141,7 @@ var all = map[string]Preset{
|
|||||||
StructuredOutput: openai.JSONSchema,
|
StructuredOutput: openai.JSONSchema,
|
||||||
FallbackStructuredOutput: openai.JSONObject,
|
FallbackStructuredOutput: openai.JSONObject,
|
||||||
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject},
|
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject},
|
||||||
|
OmitTemperature: []*regexp.Regexp{openRouterReasoningModels},
|
||||||
},
|
},
|
||||||
"ollama": {
|
"ollama": {
|
||||||
Name: "ollama",
|
Name: "ollama",
|
||||||
@@ -151,7 +154,9 @@ var all = map[string]Preset{
|
|||||||
Key: KeyOptional,
|
Key: KeyOptional,
|
||||||
StructuredOutput: openai.JSONSchema,
|
StructuredOutput: openai.JSONSchema,
|
||||||
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject, openai.PromptOnly},
|
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject, openai.PromptOnly},
|
||||||
OmitTemperature: TemperatureRejectingModels,
|
// The server may be OpenAI itself or a gateway that names models
|
||||||
|
// the OpenRouter way.
|
||||||
|
OmitTemperature: []*regexp.Regexp{openAIReasoningModels, openRouterReasoningModels},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -246,7 +251,13 @@ type configured struct {
|
|||||||
temperature *float64
|
temperature *float64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Generate validates req as sent by the caller before changing its
|
||||||
|
// temperature, so clearing it for a model that rejects temperature never
|
||||||
|
// hides an invalid one.
|
||||||
func (c *configured) Generate(ctx context.Context, req llm.Request) (llm.Response, error) {
|
func (c *configured) Generate(ctx context.Context, req llm.Request) (llm.Response, error) {
|
||||||
|
if err := req.Validate(); err != nil {
|
||||||
|
return llm.Response{}, fmt.Errorf("presets: generate: %w", err)
|
||||||
|
}
|
||||||
if c.temperature == nil {
|
if c.temperature == nil {
|
||||||
req.Temperature = nil
|
req.Temperature = nil
|
||||||
} else if req.Temperature == nil {
|
} else if req.Temperature == nil {
|
||||||
|
|||||||
@@ -223,11 +223,19 @@ func TestOpenAICompatiblePreset(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTemperatureOmittedForRejectingModels(t *testing.T) {
|
func TestTemperatureOmittedForRejectingModels(t *testing.T) {
|
||||||
for _, provider := range []string{"openai", "openai_compatible"} {
|
cases := map[string][]string{
|
||||||
for _, model := range []string{"o1", "o1-mini", "o3-mini"} {
|
"openai": {"o1", "o1-mini", "o1-2024-12-17", "o3", "o3-mini"},
|
||||||
s := llm.Settings{Provider: provider, BaseURL: "http://llm.lan/v1", Model: model, Temperature: 0.8}
|
"openai_compatible": {"o1", "o1-mini", "o3-mini", "openai/o1"},
|
||||||
if provider == "openai" {
|
"openrouter": {"openai/o1", "openai/o1-mini", "openai/o3", "openai/o3-mini", "openai/o1:online"},
|
||||||
s.BaseURL = ""
|
}
|
||||||
|
for provider, models := range cases {
|
||||||
|
for _, model := range models {
|
||||||
|
s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8}
|
||||||
|
switch provider {
|
||||||
|
case "openai_compatible":
|
||||||
|
s.BaseURL = "http://llm.lan/v1"
|
||||||
|
case "openrouter":
|
||||||
|
s.PublicURL = "https://mealquest.example"
|
||||||
}
|
}
|
||||||
for _, req := range []llm.Request{plainRequest(), func() llm.Request { r := plainRequest(); r.Temperature = ptr(0.2); return r }()} {
|
for _, req := range []llm.Request{plainRequest(), func() llm.Request { r := plainRequest(); r.Temperature = ptr(0.2); return r }()} {
|
||||||
if got, ok := temperature(t, generate(t, s, testKey, req)); ok {
|
if got, ok := temperature(t, generate(t, s, testKey, req)); ok {
|
||||||
@@ -239,10 +247,19 @@ func TestTemperatureOmittedForRejectingModels(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTemperatureSentForOtherModels(t *testing.T) {
|
func TestTemperatureSentForOtherModels(t *testing.T) {
|
||||||
for _, model := range []string{"gpt-4o", "gpt-4o-mini", "chatgpt-o1-like"} {
|
cases := map[string][]string{
|
||||||
c := generate(t, llm.Settings{Provider: "openai", Model: model, Temperature: 0.8}, testKey, plainRequest())
|
"openai": {"gpt-4o", "gpt-4o-mini", "chatgpt-o1-like", "o123", "o3custom", "o1mini", "openai/o1"},
|
||||||
if got, ok := temperature(t, c); !ok || got != 0.8 {
|
"openrouter": {"openai/gpt-4o", "openai/o123", "openai/o3custom", "acme/o1", "o1-mini"},
|
||||||
t.Errorf("%s: temperature = %v, %v; want 0.8", model, got, ok)
|
}
|
||||||
|
for provider, models := range cases {
|
||||||
|
for _, model := range models {
|
||||||
|
s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8}
|
||||||
|
if provider == "openrouter" {
|
||||||
|
s.PublicURL = "https://mealquest.example"
|
||||||
|
}
|
||||||
|
if got, ok := temperature(t, generate(t, s, testKey, plainRequest())); !ok || got != 0.8 {
|
||||||
|
t.Errorf("%s %s: temperature = %v, %v; want 0.8", provider, model, got, ok)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// A request's own temperature wins over the household's.
|
// A request's own temperature wins over the household's.
|
||||||
@@ -254,8 +271,35 @@ func TestTemperatureSentForOtherModels(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A request is validated before the temperature quirk clears its
|
||||||
|
// temperature, so a negative one is an error and nothing is sent.
|
||||||
|
func TestNegativeTemperatureRejectedForRejectingModels(t *testing.T) {
|
||||||
|
rt := &recordingTransport{}
|
||||||
|
orig := http.DefaultTransport
|
||||||
|
http.DefaultTransport = rt
|
||||||
|
t.Cleanup(func() { http.DefaultTransport = orig })
|
||||||
|
|
||||||
|
for _, s := range []llm.Settings{
|
||||||
|
{Provider: "openai", Model: "o1"},
|
||||||
|
{Provider: "openrouter", Model: "openai/o1", PublicURL: "https://mealquest.example"},
|
||||||
|
} {
|
||||||
|
p, err := FromSettings(s, testKey)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: FromSettings: %v", s.Model, err)
|
||||||
|
}
|
||||||
|
req := plainRequest()
|
||||||
|
req.Temperature = ptr(-0.5)
|
||||||
|
if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, errStop) || !strings.Contains(err.Error(), "temperature") {
|
||||||
|
t.Errorf("%s: Generate = %v, want a temperature validation error", s.Model, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(rt.reqs) != 0 {
|
||||||
|
t.Fatalf("sent %d requests, want 0", len(rt.reqs))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTemperatureQuirkPresets(t *testing.T) {
|
func TestTemperatureQuirkPresets(t *testing.T) {
|
||||||
want := map[string]bool{"openai": true, "openai_compatible": true}
|
want := map[string]bool{"openai": true, "openrouter": true, "openai_compatible": true}
|
||||||
for name, p := range all {
|
for name, p := range all {
|
||||||
if got := len(p.OmitTemperature) > 0; got != want[name] {
|
if got := len(p.OmitTemperature) > 0; got != want[name] {
|
||||||
t.Errorf("%s: lists temperature quirk = %v, want %v", name, got, want[name])
|
t.Errorf("%s: lists temperature quirk = %v, want %v", name, got, want[name])
|
||||||
|
|||||||
Reference in New Issue
Block a user