diff --git a/.env.example b/.env.example index 2166cd2e3..49570089d 100644 --- a/.env.example +++ b/.env.example @@ -70,3 +70,8 @@ SCRATCH_ASSET_IMPORT_BASE_URL=https://example.com/assets/ HOSTNAME=localhost LEARNER_EXPERIENCE_TESTS_DISPATCH_TOKEN=changeme + +# Friendly error messages +FEM_INFERENCE_API_URL='http://localhost:11434/v1/' +FEM_INFERENCE_API_KEY='' +BYPASS_FEM_INFERENCE=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd6bd2110..4e2a18dd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,7 @@ jobs: SALESFORCE_CONNECT_USER: postgres SALESFORCE_CONNECT_PASSWORD: password SALESFORCE_CONNECT_DB: salesforce_test + FEM_INFERENCE_API_URL: '' services: postgres: image: postgres:12 diff --git a/Gemfile b/Gemfile index 1b63d9974..0e58aff31 100644 --- a/Gemfile +++ b/Gemfile @@ -87,3 +87,5 @@ group :test do gem 'webdrivers' gem 'webmock' end + +gem 'openai', '~> 0.115.0' diff --git a/Gemfile.lock b/Gemfile.lock index b47104069..b1bf4d804 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -325,6 +325,9 @@ GEM stringio time uri + openai (0.115.0) + connection_pool (>= 2.2.3) + logger paper_trail (17.0.0) activerecord (>= 7.1) request_store (~> 1.4) @@ -604,6 +607,7 @@ DEPENDENCIES omniauth-rails_csrf_protection (~> 2.0.1) omniauth-rpi! open-uri + openai (~> 0.115.0) paper_trail pg (~> 1.7) postmark-rails diff --git a/app/controllers/api/features_controller.rb b/app/controllers/api/features_controller.rb index 87dc47452..1d0e761fe 100644 --- a/app/controllers/api/features_controller.rb +++ b/app/controllers/api/features_controller.rb @@ -9,7 +9,7 @@ def index school = current_user&.schools&.first enabled_feature_keys = Flipper.features - .select { |feature| Flipper.enabled?(feature.key, current_user, school) } + .select { |feature| FeatureFlagHelpers.enabled_for_user?(feature.key, current_user, school) } .map(&:key) render json: { enabled: enabled_feature_keys } diff --git a/app/controllers/api/friendly_error_messages_controller.rb b/app/controllers/api/friendly_error_messages_controller.rb new file mode 100644 index 000000000..cd2c7d102 --- /dev/null +++ b/app/controllers/api/friendly_error_messages_controller.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Api + class FriendlyErrorMessagesController < ApiController + MAX_REQUEST_SIZE_KB = 100 + + before_action :authorize_user + skip_authorization_check only: :show + + rescue_from FriendlyErrorMessagesInferenceClient::Error, with: :bad_gateway + + def show + return render json: {}, status: :unauthorized unless FeatureFlagHelpers.enabled_for_user? :ai_friendly_errors, current_user, current_user&.schools&.first + return render json: { error: "The request was over the max size of #{MAX_REQUEST_SIZE_KB} KB" }, status: :content_too_large if request.body.size > MAX_REQUEST_SIZE_KB.kilobytes + + params.require(%i[message trace runtime code]) + + fem = FriendlyErrorMessagesInferenceClient.new.get_message( + params[:message], + params[:trace], + params[:runtime], + params[:code] + ) + + render json: { fem: }, status: :ok + end + end +end diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb index 3f9b1a984..d3399b2cd 100644 --- a/app/controllers/api_controller.rb +++ b/app/controllers/api_controller.rb @@ -60,6 +60,11 @@ def authorization_not_performed(exception) render json: { error: 'Internal server error' }, status: :internal_server_error end + def bad_gateway(exception) + Sentry.capture_exception(exception) + render_error_as_json exception, :bad_gateway + end + def render_error_as_json(exception, status) render json: { error: "#{exception.class}: #{exception.message}" }, status: end diff --git a/config/routes.rb b/config/routes.rb index 2548a9bd1..a804f0596 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -118,6 +118,8 @@ post '/join/:join_code', to: 'join#create' resources :events, only: %i[create] + + post '/friendly_error_messages', to: 'friendly_error_messages#show' end resource :github_webhooks, only: :create, defaults: { formats: :json } diff --git a/lib/friendly_error_messages_inference_client.rb b/lib/friendly_error_messages_inference_client.rb new file mode 100644 index 000000000..915942bfd --- /dev/null +++ b/lib/friendly_error_messages_inference_client.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +class FriendlyErrorMessagesInferenceClient + class Error < StandardError; end + + MAX_RESULT_LENGTH_CHARS = 500 + + SYSTEM_PROMPT_BASE = <<~PROMPT + You help young people learn to program in Python. + + A learner will show you some Python code, and usually an error message. Respond with a + short piece of feedback, addressed to them, that helps them find and fix the problem + themselves. + PROMPT + + SYSTEM_PROMPT_L1_EXPLAIN = <<~PROMPT + The learner has only seen the raw Python error. Nothing has explained it to them yet. + + Say in plain language what the error means, and suggest which part of their code to + look at. Pose it as a suggestion, not an instruction — "it might be worth checking…" + rather than "change…". + + Three or four short sentences. + + Do not write any code in this reply. No code blocks and no corrected lines. + PROMPT + + def initialize + @client = OpenAI::Client.new( + base_url: ENV.fetch('FEM_INFERENCE_API_URL'), + api_key: ENV.fetch('FEM_INFERENCE_API_KEY', ''), + timeout: 15.0, + max_retries: 0 + ) + end + + def get_message(message, trace, runtime, code) + return 'This is a placeholder friendly error message.' if ENV.fetch('BYPASS_FEM_INFERENCE', nil) == 'true' + + result = @client.responses.create( + model: 'qwen3.5:9b', + input: [ + { role: 'system', content: SYSTEM_PROMPT_BASE }, + { role: 'system', content: SYSTEM_PROMPT_L1_EXPLAIN }, + { role: 'user', content: JSON.dump({ message:, trace:, runtime:, code: }) } + ], + reasoning: { effort: :none } + ).output_text + + raise Error, 'No message returned by inference service.' if result.empty? + raise Error, "The message by the inference service was too long - max #{MAX_RESULT_LENGTH_CHARS} chars, actual #{result.length} chars." if result.length > MAX_RESULT_LENGTH_CHARS + + result + rescue OpenAI::Errors::APIError => e + raise Error, e.message + end +end diff --git a/lib/helpers/feature_flag_helpers.rb b/lib/helpers/feature_flag_helpers.rb new file mode 100644 index 000000000..8b1a906eb --- /dev/null +++ b/lib/helpers/feature_flag_helpers.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class FeatureFlagHelpers + def self.enabled_for_user?(feature_key, user, school) + Flipper.enabled?(feature_key, user, school) + end +end diff --git a/spec/lib/friendly_error_messages_inference_client_spec.rb b/spec/lib/friendly_error_messages_inference_client_spec.rb new file mode 100644 index 000000000..a449dab50 --- /dev/null +++ b/spec/lib/friendly_error_messages_inference_client_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe FriendlyErrorMessagesInferenceClient do + subject(:client) { described_class.new } + + let(:mock_client) { instance_double(OpenAI::Client) } + + context 'when BYPASS_FEM_INFERENCE is false' do + around do |example| + ClimateControl.modify(BYPASS_FEM_INFERENCE: 'false') do + example.run + end + end + + it 'calls the inference server' do + responses = instance_double(OpenAI::Resources::Responses) + response = instance_double(OpenAI::Models::Responses::Response, output_text: 'A friendly error message') + allow(OpenAI::Client).to receive(:new).and_return(mock_client) + allow(mock_client).to receive(:responses).and_return(responses) + allow(responses).to receive(:create).and_return(response) + + result = client.get_message('', '', '', '') + expect(result).to eq 'A friendly error message' + end + + it 'throws an error when the inference call times out' do + responses = instance_double(OpenAI::Resources::Responses) + allow(OpenAI::Client).to receive(:new).and_return(mock_client) + allow(mock_client).to receive(:responses).and_return(responses) + allow(responses).to receive(:create).and_raise(OpenAI::Errors::APITimeoutError.new(url: 'http://some_url')) + + expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) + end + + it 'throws an error when the inference server is down' do + responses = instance_double(OpenAI::Resources::Responses) + allow(OpenAI::Client).to receive(:new).and_return(mock_client) + allow(mock_client).to receive(:responses).and_return(responses) + allow(responses).to receive(:create).and_raise(OpenAI::Errors::APIConnectionError.new(url: 'http://some_url')) + + expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) + end + + it 'throws an error when the inference server returns nothing' do + responses = instance_double(OpenAI::Resources::Responses) + response = instance_double(OpenAI::Models::Responses::Response, output_text: '') + allow(OpenAI::Client).to receive(:new).and_return(mock_client) + allow(mock_client).to receive(:responses).and_return(responses) + allow(responses).to receive(:create).and_return(response) + + expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) + end + + it 'throws an error when the inference server returns a response that is too long' do + responses = instance_double(OpenAI::Resources::Responses) + response = instance_double(OpenAI::Models::Responses::Response, output_text: 'a' * 1000) + allow(OpenAI::Client).to receive(:new).and_return(mock_client) + allow(mock_client).to receive(:responses).and_return(responses) + allow(responses).to receive(:create).and_return(response) + + expect { client.get_message('', '', '', '') }.to raise_exception(FriendlyErrorMessagesInferenceClient::Error) + end + end + + context 'when BYPASS_FEM_INFERENCE is true' do + around do |example| + ClimateControl.modify(BYPASS_FEM_INFERENCE: 'true') do + example.run + end + end + + it 'returns a mock response' do + result = client.get_message('', '', '', '') + expect(result).to eq 'This is a placeholder friendly error message.' + end + end +end diff --git a/spec/requests/friendly_error_messages_spec.rb b/spec/requests/friendly_error_messages_spec.rb new file mode 100644 index 000000000..088321f08 --- /dev/null +++ b/spec/requests/friendly_error_messages_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe 'FriendlyErrorMessages' do + describe 'POST /friendly_error_messages' do + let(:headers) { { Authorization: UserProfileMock::TOKEN } } + let(:school) { create(:school) } + let(:student) { create(:student, school:) } + let(:mock_client) { instance_double(FriendlyErrorMessagesInferenceClient) } + + before do + Flipper.enable(:ai_friendly_errors) + end + + around do |example| + ClimateControl.modify(BYPASS_FEM_INFERENCE: 'true') do + example.run + end + end + + it 'does not allow logged-out users' do + post '/api/friendly_error_messages' + + expect(response).to have_http_status(:unauthorized) + end + + it 'disallows requests over 100KB in size' do + authenticated_in_hydra_as(student) + + post('/api/friendly_error_messages', headers:, params: { content: 'a' * 100.kilobytes }) + + expect(response).to have_http_status(:content_too_large) + end + + it 'requires message, trace, runtime and code params' do + authenticated_in_hydra_as(student) + + post('/api/friendly_error_messages', headers:, params: { + message: 'message', + trace: 'trace', + runtime: 'runtime' + }) + + expect(response).to have_http_status(:bad_request) + end + + it 'is protected by a feature flag' do + Flipper.disable(:ai_friendly_errors) + + authenticated_in_hydra_as(student) + + post('/api/friendly_error_messages', headers:) + + expect(response).to have_http_status(:unauthorized) + end + + it 'returns a result for a well-formed request' do + authenticated_in_hydra_as(student) + + post('/api/friendly_error_messages', headers:, params: { + message: 'message', + trace: 'trace', + runtime: 'runtime', + code: 'code' + }) + + expect(response).to have_http_status(:ok) + expect(response.parsed_body[:fem]).not_to be_nil + end + + it 'returns a 503 when the client raises an error' do + authenticated_in_hydra_as(student) + + allow(FriendlyErrorMessagesInferenceClient).to receive(:new).and_return(mock_client) + allow(mock_client).to receive(:get_message).and_raise(FriendlyErrorMessagesInferenceClient::Error) + + post('/api/friendly_error_messages', headers:, params: { + message: 'message', + trace: 'trace', + runtime: 'runtime', + code: 'code' + }) + + expect(response).to have_http_status(:bad_gateway) + end + end +end