Use a cloud based model

lib/friendly_error_messages_inference_client.rb

@@ -4,6 +4,7 @@ class FriendlyErrorMessagesInferenceClient
   class Error < StandardError; end
 
   MAX_RESULT_LENGTH_CHARS = 500
+  MAX_COMPLETION_TOKENS = MAX_RESULT_LENGTH_CHARS * 0.66
 
   SYSTEM_PROMPT_BASE = <<~PROMPT
     You help young people learn to program in Python.
@@ -25,9 +26,11 @@ class Error < StandardError; end
     Do not write any code in this reply. No code blocks and no corrected lines.
   PROMPT
 
+  NO_REASONING_COMMAND = '/no_think'
+
   def initialize
     @client = OpenAI::Client.new(
-      base_url: ENV.fetch('FEM_INFERENCE_API_URL'),
+      base_url: "#{ENV.fetch('FEM_INFERENCE_API_URL', '')}/ai/v1",
       api_key: ENV.fetch('FEM_INFERENCE_API_KEY', ''),
       timeout: 15.0,
       max_retries: 0
@@ -37,15 +40,9 @@ def initialize
   def get_message(message, trace, runtime, code)
     return 'This is a placeholder friendly error message.' if ENV.fetch('BYPASS_FEM_INFERENCE', nil) == 'true'
 
-    result = @client.responses.create(
-      model: 'qwen3.5:9b',
-      input: [
-        { role: 'system', content: SYSTEM_PROMPT_BASE },
-        { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN },
-        { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) }
-      ],
-      reasoning: { effort: :none }
-    ).output_text
+    result = @client.chat.completions.create(
+      build_request(message, trace, runtime, code)
+    ).choices.first.message.content.to_s
 
     raise Error, 'No message returned by inference service.' if result.empty?
     raise Error, "The message by the inference service was too long - max #{MAX_RESULT_LENGTH_CHARS} chars, actual #{result.length} chars." if result.length > MAX_RESULT_LENGTH_CHARS
@@ -54,4 +51,19 @@ def get_message(message, trace, runtime, code)
   rescue OpenAI::Errors::APIError => e
     raise Error, e.message
   end
+
+  def build_request(message, trace, runtime, code)
+    {
+      model: '@cf/qwen/qwen3-30b-a3b-fp8',
+      messages: [
+        { role: 'system', content: SYSTEM_PROMPT_BASE },
+        { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN },
+        { role: 'system', content: NO_REASONING_COMMAND },
+        { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) }
+      ],
+      temperature: 0,
+      max_completion_tokens: MAX_COMPLETION_TOKENS,
+      request_options: { extra_headers: { 'cf-aig-gateway-id' => ENV.fetch('FEM_GATEWAY_NAME', '') } }
+    }
+  end
 end

spec/lib/friendly_error_messages_inference_client_spec.rb

@@ -6,6 +6,15 @@
   subject(:client) { described_class.new }
 
   let(:mock_client) { instance_double(OpenAI::Client) }
+  let(:chat) { instance_double(OpenAI::Resources::Chat) }
+  let(:completions) { instance_double(OpenAI::Resources::Chat::Completions) }
+
+  def stub_completion(content)
+    message = instance_double(OpenAI::Models::Chat::ChatCompletionMessage, content:)
+    choice = instance_double(OpenAI::Models::Chat::ChatCompletion::Choice, message:)
+    completion = instance_double(OpenAI::Models::Chat::ChatCompletion, choices: [choice])
+    allow(completions).to receive(:create).and_return(completion)
+  end
 
   context 'when BYPASS_FEM_INFERENCE is false' do
     around do |example|
@@ -14,51 +23,39 @@
       end
     end
 
-    it 'calls the inference server' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      response = instance_double(OpenAI::Models::Responses::Response, output_text: 'A friendly error message')
+    before do
       allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_return(response)
+      allow(mock_client).to receive(:chat).and_return(chat)
+      allow(chat).to receive(:completions).and_return(completions)
+    end
+
+    it 'calls the inference server' do
+      stub_completion('A friendly error message')
 
       result = client.get_message('', '', '', '')
       expect(result).to eq 'A friendly error message'
     end
 
     it 'throws an error when the inference call times out' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url'))
+      allow(completions).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url'))
 
       expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
     end
 
     it 'throws an error when the inference server is down' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url'))
+      allow(completions).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url'))
 
       expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
     end
 
     it 'throws an error when the inference server returns nothing' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      response = instance_double(OpenAI::Models::Responses::Response, output_text: '')
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_return(response)
+      stub_completion(nil)
 
       expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
     end
 
     it 'throws an error when the inference server returns a response that is too long' do
-      responses = instance_double(OpenAI::Resources::Responses)
-      response = instance_double(OpenAI::Models::Responses::Response, output_text: 'a' * 1000)
-      allow(OpenAI::Client).to receive(:new).and_return(mock_client)
-      allow(mock_client).to receive(:responses).and_return(responses)
-      allow(responses).to receive(:create).and_return(response)
+      stub_completion('a' * 1000)
 
       expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error)
     end