From f69c76f4889382392cd8a7c52970149d2775486d Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:56:16 +0000 Subject: [PATCH 1/6] Use the chat API that's more widely supported The responses API is more recent has support for more features such as tool use, however it isn't supported as widely by cloud providers. Specifically, the qwen models we're considering that is hosted by Cloudflare only supports the chat API. I think the chat API is sufficient for our use case. This also makes the mocking easier in the tests with a helper function. Responses vs chat API: https://developers.openai.com/api/docs/guides/migrate-to-responses Co-Authored-By: Claude Opus 5 (1M context) --- ...riendly_error_messages_inference_client.rb | 8 ++-- ...ly_error_messages_inference_client_spec.rb | 43 +++++++++---------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/lib/friendly_error_messages_inference_client.rb b/lib/friendly_error_messages_inference_client.rb index 915942bfd..057b8ea7b 100644 --- a/lib/friendly_error_messages_inference_client.rb +++ b/lib/friendly_error_messages_inference_client.rb @@ -37,15 +37,15 @@ def initialize def get_message(message, trace, runtime, code) return 'This is a placeholder friendly error message.' if ENV.fetch('BYPASS_FEM_INFERENCE', nil) == 'true' - result = @client.responses.create( + result = @client.chat.completions.create( model: 'qwen3.5:9b', - input: [ + messages: [ { role: 'system', content: SYSTEM_PROMPT_BASE }, { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN }, { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } ], - reasoning: { effort: :none } - ).output_text + reasoning_effort: :none + ).choices.first.message.content.to_s raise Error, 'No message returned by inference service.' if result.empty? raise Error, "The message by the inference service was too long - max #{MAX_RESULT_LENGTH_CHARS} chars, actual #{result.length} chars." if result.length > MAX_RESULT_LENGTH_CHARS diff --git a/spec/lib/friendly_error_messages_inference_client_spec.rb b/spec/lib/friendly_error_messages_inference_client_spec.rb index a449dab50..ba47f50b6 100644 --- a/spec/lib/friendly_error_messages_inference_client_spec.rb +++ b/spec/lib/friendly_error_messages_inference_client_spec.rb @@ -6,6 +6,15 @@ subject(:client) { described_class.new } let(:mock_client) { instance_double(OpenAI::Client) } + let(:chat) { instance_double(OpenAI::Resources::Chat) } + let(:completions) { instance_double(OpenAI::Resources::Chat::Completions) } + + def stub_completion(content) + message = instance_double(OpenAI::Models::Chat::ChatCompletionMessage, content:) + choice = instance_double(OpenAI::Models::Chat::ChatCompletion::Choice, message:) + completion = instance_double(OpenAI::Models::Chat::ChatCompletion, choices: [choice]) + allow(completions).to receive(:create).and_return(completion) + end context 'when BYPASS_FEM_INFERENCE is false' do around do |example| @@ -14,51 +23,39 @@ end end - it 'calls the inference server' do - responses = instance_double(OpenAI::Resources::Responses) - response = instance_double(OpenAI::Models::Responses::Response, output_text: 'A friendly error message') + before do allow(OpenAI::Client).to receive(:new).and_return(mock_client) - allow(mock_client).to receive(:responses).and_return(responses) - allow(responses).to receive(:create).and_return(response) + allow(mock_client).to receive(:chat).and_return(chat) + allow(chat).to receive(:completions).and_return(completions) + end + + it 'calls the inference server' do + stub_completion('A friendly error message') result = client.get_message('', '', '', '') expect(result).to eq 'A friendly error message' end it 'throws an error when the inference call times out' do - responses = instance_double(OpenAI::Resources::Responses) - allow(OpenAI::Client).to receive(:new).and_return(mock_client) - allow(mock_client).to receive(:responses).and_return(responses) - allow(responses).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url')) + allow(completions).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url')) expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) end it 'throws an error when the inference server is down' do - responses = instance_double(OpenAI::Resources::Responses) - allow(OpenAI::Client).to receive(:new).and_return(mock_client) - allow(mock_client).to receive(:responses).and_return(responses) - allow(responses).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url')) + allow(completions).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url')) expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) end it 'throws an error when the inference server returns nothing' do - responses = instance_double(OpenAI::Resources::Responses) - response = instance_double(OpenAI::Models::Responses::Response, output_text: '') - allow(OpenAI::Client).to receive(:new).and_return(mock_client) - allow(mock_client).to receive(:responses).and_return(responses) - allow(responses).to receive(:create).and_return(response) + stub_completion(nil) expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) end it 'throws an error when the inference server returns a response that is too long' do - responses = instance_double(OpenAI::Resources::Responses) - response = instance_double(OpenAI::Models::Responses::Response, output_text: 'a' * 1000) - allow(OpenAI::Client).to receive(:new).and_return(mock_client) - allow(mock_client).to receive(:responses).and_return(responses) - allow(responses).to receive(:create).and_return(response) + stub_completion('a' * 1000) expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) end From 0c3692e11ab5a11a0d27adc7197dc1e7a3f591ac Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:06:10 +0000 Subject: [PATCH 2/6] Configure Cloudflare cloud model This qwen model is a suggestion as it's one of the closest to the ones we have previously tested and has a reasonable cost. We will run the evaluator against it to confirm it's suitable. Note I took out the reasoning config as it changes the structure of the response and causes errors (I'm not sure why). I'll add this back in a future commit --- lib/friendly_error_messages_inference_client.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/friendly_error_messages_inference_client.rb b/lib/friendly_error_messages_inference_client.rb index 057b8ea7b..9119d8887 100644 --- a/lib/friendly_error_messages_inference_client.rb +++ b/lib/friendly_error_messages_inference_client.rb @@ -27,7 +27,7 @@ class Error < StandardError; end def initialize @client = OpenAI::Client.new( - base_url: ENV.fetch('FEM_INFERENCE_API_URL'), + base_url: "#{ENV.fetch('FEM_INFERENCE_API_URL', '')}/ai/v1", api_key: ENV.fetch('FEM_INFERENCE_API_KEY', ''), timeout: 15.0, max_retries: 0 @@ -38,13 +38,13 @@ def get_message(message, trace, runtime, code) return 'This is a placeholder friendly error message.' if ENV.fetch('BYPASS_FEM_INFERENCE', nil) == 'true' result = @client.chat.completions.create( - model: 'qwen3.5:9b', + model: '@cf/qwen/qwen3-30b-a3b-fp8', messages: [ { role: 'system', content: SYSTEM_PROMPT_BASE }, { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN }, { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } ], - reasoning_effort: :none + request_options: { extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') } } ).choices.first.message.content.to_s raise Error, 'No message returned by inference service.' if result.empty? From b121160e7fc90149f13d2cfc36f0153d768d3d9a Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:07:59 +0000 Subject: [PATCH 3/6] Refactor to extract API request params I thought the other method was getting quite complex --- ...riendly_error_messages_inference_client.rb | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/friendly_error_messages_inference_client.rb b/lib/friendly_error_messages_inference_client.rb index 9119d8887..32bbfe160 100644 --- a/lib/friendly_error_messages_inference_client.rb +++ b/lib/friendly_error_messages_inference_client.rb @@ -38,13 +38,7 @@ def get_message(message, trace, runtime, code) return 'This is a placeholder friendly error message.' if ENV.fetch('BYPASS_FEM_INFERENCE', nil) == 'true' result = @client.chat.completions.create( - model: '@cf/qwen/qwen3-30b-a3b-fp8', - messages: [ - { role: 'system', content: SYSTEM_PROMPT_BASE }, - { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN }, - { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } - ], - request_options: { extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') } } + build_request(message, trace, runtime, code) ).choices.first.message.content.to_s raise Error, 'No message returned by inference service.' if result.empty? @@ -54,4 +48,16 @@ def get_message(message, trace, runtime, code) rescue OpenAI::Errors::APIError => e raise Error, e.message end + + def build_request(message, trace, runtime, code) + { + model: '@cf/qwen/qwen3-30b-a3b-fp8', + messages: [ + { role: 'system', content: SYSTEM_PROMPT_BASE }, + { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN }, + { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } + ], + request_options: { extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') } } + } + end end From b906bba278dd70810545ca003862b3d41fe85b9c Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:37:22 +0000 Subject: [PATCH 4/6] Disable reasoning When testing we found that reasoning increased cost and time while providing little benefit for the kinds of data we expect. Initially I tried disabling this using the 'reasoning' parameter but this changes the structure of the response. I've not been able to work out why, but assume it is the way this particular model is hosted or configured. Instead I've switched to an alternative way to disable reasoning using the /no_think command. --- lib/friendly_error_messages_inference_client.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/friendly_error_messages_inference_client.rb b/lib/friendly_error_messages_inference_client.rb index 32bbfe160..680cf852c 100644 --- a/lib/friendly_error_messages_inference_client.rb +++ b/lib/friendly_error_messages_inference_client.rb @@ -25,6 +25,8 @@ class Error < StandardError; end Do not write any code in this reply. No code blocks and no corrected lines. PROMPT + NO_REASONING_COMMAND = '/no_think' + def initialize @client = OpenAI::Client.new( base_url: "#{ENV.fetch('FEM_INFERENCE_API_URL', '')}/ai/v1", @@ -55,6 +57,7 @@ def build_request(message, trace, runtime, code) messages: [ { role: 'system', content: SYSTEM_PROMPT_BASE }, { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN }, + { role: 'system', content: NO_REASONING_COMMAND }, { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } ], request_options: { extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') } } From 15ade92746823d9af99035326b4b9c19fb351029 Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:37:30 +0000 Subject: [PATCH 5/6] Set a low temperature Having a low temperature makes the models relatively more predictable which is preferred for our needs. I tested this by asking the model to name an animal. I still saw variance in responses with a low temperature, but I saw less than with a higher temperature so I assume it's working. --- lib/friendly_error_messages_inference_client.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/friendly_error_messages_inference_client.rb b/lib/friendly_error_messages_inference_client.rb index 680cf852c..a56953ee7 100644 --- a/lib/friendly_error_messages_inference_client.rb +++ b/lib/friendly_error_messages_inference_client.rb @@ -60,6 +60,7 @@ def build_request(message, trace, runtime, code) { role: 'system', content: NO_REASONING_COMMAND }, { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } ], + temperature: 0, request_options: { extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') } } } end From d50aaf72cd8a2636ea11b1956d87aff497d425c6 Mon Sep 17 00:00:00 2001 From: Chris Zetter <253059100+zetter-rpf@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:43:07 +0000 Subject: [PATCH 6/6] Set maximum completion tokens to reduce risk of costly responses In english, it's common to estimate each token as approximately 4 characters long so should be set to about 1/4 of the max result length. I've added a generous buffer on top of this. --- lib/friendly_error_messages_inference_client.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/friendly_error_messages_inference_client.rb b/lib/friendly_error_messages_inference_client.rb index a56953ee7..ff4254b5d 100644 --- a/lib/friendly_error_messages_inference_client.rb +++ b/lib/friendly_error_messages_inference_client.rb @@ -4,6 +4,7 @@ class FriendlyErrorMessagesInferenceClient class Error < StandardError; end MAX_RESULT_LENGTH_CHARS = 500 + MAX_COMPLETION_TOKENS = MAX_RESULT_LENGTH_CHARS * 0.66 SYSTEM_PROMPT_BASE = <<~PROMPT You help young people learn to program in Python. @@ -61,6 +62,7 @@ def build_request(message, trace, runtime, code) { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } ], temperature: 0, + max_completion_tokens: MAX_COMPLETION_TOKENS, request_options: { extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') } } } end