diff --git a/lib/imp/clients/req_llm.ex b/lib/imp/clients/req_llm.ex index 40a6b53f..689d45f7 100644 --- a/lib/imp/clients/req_llm.ex +++ b/lib/imp/clients/req_llm.ex @@ -10,6 +10,10 @@ defmodule Imp.Clients.ReqLLM do leaving the endpoint to choose it. Explicit output limits are passed through ReqLLM unchanged; Imp does not reserve the catalog's full output maximum. + On an OpenRouter call made with the caller's own provider key (`"is_byok"`), + the charge is OpenRouter's fee plus the upstream charge it reports in + `"cost_details"`, and `nil` when either figure is missing. + `:input_envelope` is an Imp-owned safety option. It accepts a positive `:max_bytes` guard and an optional positive `:reservation_tokens` value. Imp measures the rendered message content before cache lookup or transport, @@ -345,12 +349,22 @@ defmodule Imp.Clients.ReqLLM do defp cache_hit_result(value), do: value defp mark_cache_hit(%{req_llm: provider_meta} = metadata) when is_map(provider_meta) do - provider_meta = provider_meta |> Map.put(:usage, %{}) |> Map.put(:cache_hit, true) + provider_meta = + provider_meta + |> Map.put(:usage, %{}) + |> Map.put(:cache_hit, true) + |> Map.drop([:charge, "charge"]) + Map.put(metadata, :req_llm, provider_meta) end defp mark_cache_hit(%{"req_llm" => provider_meta} = metadata) when is_map(provider_meta) do - provider_meta = provider_meta |> Map.put("usage", %{}) |> Map.put("cache_hit", true) + provider_meta = + provider_meta + |> Map.put("usage", %{}) + |> Map.put("cache_hit", true) + |> Map.drop([:charge, "charge"]) + Map.put(metadata, "req_llm", provider_meta) end @@ -1986,19 +2000,48 @@ defmodule Imp.Clients.ReqLLM do defp from_response(other, _model_spec), do: other + # `"cost"` on a bring-your-own-key call is the gateway fee. The upstream + # charge is `"cost_details"` under `:upstream_inference_cost`. Both numbers + # are the charge; either missing or not a number makes it unknown. + # `"is_byok"` identifies the call for every model spec this client accepts. + # Any other usage reports its `"cost"`, or `nil` when the provider reported + # none. Callers always store the figure as `:charge`. + @doc false + @spec charge(term()) :: term() + def charge(%{"is_byok" => true} = usage), do: openrouter_byok_charge(usage) + def charge(%{"cost" => cost}), do: cost + def charge(_usage), do: nil + + defp openrouter_byok_charge(usage) do + fee = Map.get(usage, "cost") + upstream = usage |> Map.get("cost_details") |> upstream_inference_cost() + + if is_number(fee) and is_number(upstream), do: fee + upstream, else: nil + end + + defp upstream_inference_cost(details) when is_map(details) do + Map.get(details, :upstream_inference_cost, Map.get(details, "upstream_inference_cost")) + end + + defp upstream_inference_cost(_details), do: nil + defp response_metadata(%ReqLLM.Response{} = response, model_spec) do provider_meta = response.provider_meta || %{} logprobs = sanitize_logprobs(map_value(provider_meta, :logprobs)) + usage = sanitize_usage(ReqLLM.Response.usage(response)) + + charge = charge(usage) %{ provider: provider_name(model_spec), model: response.model, api: map_value(provider_meta, :api_type), finish_reason: response.finish_reason, - usage: sanitize_usage(ReqLLM.Response.usage(response)), + usage: usage, content: ReqLLM.Response.text(response) || "", logprobs: logprobs, - provider_meta: sanitize_provider_meta(provider_meta, logprobs) + provider_meta: sanitize_provider_meta(provider_meta, logprobs), + charge: charge } end diff --git a/lib/imp/core.ex b/lib/imp/core.ex index 1dac38fc..e0e2699c 100644 --- a/lib/imp/core.ex +++ b/lib/imp/core.ex @@ -75,18 +75,9 @@ defmodule Imp.Core do @moduledoc """ Provider-neutral LM response: normalized outputs, usage, cost, and raw data. - `cost` is what the provider says it charged for this call, in USD, as a - non-negative float, or `nil` when the provider reported no charge. `nil` - means the charge is unknown, not that the call was free. OpenRouter reports - its charge, unasked, as the `"cost"` field of its usage object; - ReqLLM carries that field through unchanged, and Imp reads it from there. - A provider whose response carries no charge — the Anthropic, OpenAI and - Google APIs called directly among them — gives a `nil` cost. On an - OpenRouter call made with the caller's own provider key (`"is_byok"`), - `cost` is OpenRouter's fee plus the upstream charge it reports in - `"cost_details"`, and `nil` when that upstream charge is missing. An LM - client other than ReqLLM reports its charge as `:cost` in its response - metadata. + `cost` is the charge the client reported for this call, in USD, as a + non-negative float, or `nil` when none was reported. `nil` means the + charge is unknown, not that the call was free. `estimated_cost` is ReqLLM's estimate for the call: the reported token counts priced from its model catalog, as a non-negative float, or `nil` @@ -151,7 +142,7 @@ defmodule Imp.Core do %LMResponse{ outputs: outputs, usage: usage, - cost: cost_number(reported_cost(metadata, usage)), + cost: cost_number(reported_cost(metadata)), estimated_cost: cost_number(estimate), billing: billing_breakdown(estimate), metadata: metadata, @@ -294,30 +285,36 @@ defmodule Imp.Core do end end - # In ReqLLM's usage map the atom keys are ReqLLM's own: its usage step - # prices the token counts from the model catalog and stores that estimate as - # `:cost` and `:total_cost`. The provider's wire fields that ReqLLM does not - # interpret stay under their string keys, so OpenRouter's charge is - # `"cost"`. OpenRouter includes it without being asked (its - # `usage: %{include: true}` request option is not needed for it). A client - # that is not ReqLLM reports its charge as `:cost` in its own metadata. - # - # With the caller's own provider key (`"is_byok"`), OpenRouter's `"cost"` is - # only its fee; the provider billed the key separately, and OpenRouter - # reports that as `"cost_details"."upstream_inference_cost"`. Counting the fee - # alone would understate spend, so without the upstream figure the charge - # is unknown. - defp reported_cost(_metadata, %{"is_byok" => true} = usage) do - upstream = usage |> Map.get("cost_details") |> map_value(:upstream_inference_cost, nil) - - case Map.get(usage, "cost") do - fee when is_number(fee) and is_number(upstream) -> fee + upstream - _incomplete -> nil + # `:charge` on the client metadata is the charge that client resolved. + # Present, it is the charge, `nil` included: nil is unknown, not a cue to + # read another figure. With none, the figure is `:cost` on the response + # metadata, or unknown when that is absent too. Usage `"cost"` is for the + # client to resolve into `:charge`; core does not read it. Atom `:cost` on + # the usage map is the catalog estimate, read as `estimated_cost`, not as + # the charge. + defp reported_cost(metadata) do + case resolved_charge(metadata) do + {:resolved, charge} -> charge + :unresolved -> map_value(metadata, :cost, nil) end end - defp reported_cost(metadata, usage) do - Map.get(usage, "cost", map_value(metadata, :cost, nil)) + defp resolved_charge(metadata) do + provider = map_value(metadata, :req_llm, %{}) + + cond do + not is_map(provider) -> + :unresolved + + Map.has_key?(provider, :charge) -> + {:resolved, Map.get(provider, :charge)} + + Map.has_key?(provider, "charge") -> + {:resolved, Map.get(provider, "charge")} + + true -> + :unresolved + end end # A cost breakdown is ReqLLM's map. Anything else under `:cost` is a value, diff --git a/lib/imp/streaming/execution.ex b/lib/imp/streaming/execution.ex index 4490061d..55080c06 100644 --- a/lib/imp/streaming/execution.ex +++ b/lib/imp/streaming/execution.ex @@ -220,12 +220,18 @@ defmodule Imp.Streaming.Execution do # records it, and otherwise the id the client was configured with. defp normalize_metadata(%Imp.Clients.ReqLLM{model: model}, metadata) do {provider, model_id} = Imp.Clients.ReqLLM.model_identity(model) + usage = metadata[:usage] || metadata["usage"] + + # A stream never passes through response metadata. Record the same + # charge the client would, so the recorded response reads that one figure. + charge = Imp.Clients.ReqLLM.charge(usage) req_llm = %{ provider: provider, model: metadata[:model] || metadata["model"] || model_id, - usage: metadata[:usage] || metadata["usage"], - finish_reason: metadata[:finish_reason] || metadata["finish_reason"] + usage: usage, + finish_reason: metadata[:finish_reason] || metadata["finish_reason"], + charge: charge } metadata diff --git a/test/model_response_cost_test.exs b/test/model_response_cost_test.exs index 49768513..f8082ff2 100644 --- a/test/model_response_cost_test.exs +++ b/test/model_response_cost_test.exs @@ -207,7 +207,7 @@ defmodule Imp.ModelResponseCostTest do raw = %{ __imp_lm_output__: "pong", __imp_lm_metadata__: %{ - req_llm: %{usage: %{:cost => %{@breakdown | total: total}, "cost" => total}} + req_llm: %{usage: %{:cost => %{@breakdown | total: total}}, charge: total} } } @@ -228,34 +228,24 @@ defmodule Imp.ModelResponseCostTest do assert response.billing == nil end - # With the caller's own provider key, OpenRouter's "cost" is its fee only, - # and the provider's own charge is reported beside it. - test "an OpenRouter call on the caller's own key costs the fee plus the upstream charge" do - raw = %{ - __imp_lm_output__: "pong", - __imp_lm_metadata__: %{ - req_llm: %{ - usage: %{ + # A raw envelope carries no charge the client resolved. Core reads `:charge` + # and then metadata `:cost`, never usage `"cost"`, so a fee or a wire figure + # left only on the usage map is unknown rather than that lower number. + test "a raw envelope with no resolved charge has no cost" do + for usage <- [ + %{"cost" => 0.0001, cost: @breakdown}, + %{ + "cost" => 0.0001, + "cost_details" => %{"upstream_inference_cost" => 0.002}, + cost: @breakdown + }, + %{ "is_byok" => true, "cost" => 0.0001, "cost_details" => %{"upstream_inference_cost" => 0.002}, cost: @breakdown } - } - } - } - - assert {:ok, response} = Imp.Core.response(raw) - assert_in_delta response.cost, 0.0021, 1.0e-12 - assert response.estimated_cost == 0.001858 - end - - test "an OpenRouter call on the caller's own key with no upstream charge has no cost" do - for details <- [nil, %{}, %{"upstream_inference_cost" => nil}] do - usage = - %{"is_byok" => true, "cost" => 0.0001, cost: @breakdown} - |> then(&if(details, do: Map.put(&1, "cost_details", details), else: &1)) - + ] do raw = %{__imp_lm_output__: "pong", __imp_lm_metadata__: %{req_llm: %{usage: usage}}} assert {:ok, response} = Imp.Core.response(raw) @@ -276,7 +266,7 @@ defmodule Imp.ModelResponseCostTest do for reported <- ["free", %{total: "free"}, %{total: nil}, -0.5, %{total: -0.5}] do raw = %{ __imp_lm_output__: "pong", - __imp_lm_metadata__: %{req_llm: %{usage: %{:cost => reported, "cost" => reported}}} + __imp_lm_metadata__: %{req_llm: %{usage: %{:cost => reported}, charge: reported}} } assert {:ok, response} = Imp.Core.response(raw) diff --git a/test/req_llm_client_test.exs b/test/req_llm_client_test.exs index 37e7cf9b..76a8b2b2 100644 --- a/test/req_llm_client_test.exs +++ b/test/req_llm_client_test.exs @@ -95,6 +95,22 @@ defmodule ReqLLMClientTest do end end + defmodule ChargeStub do + def generate_text(model, messages, opts) do + {:ok, + %ReqLLM.Response{ + id: "resp_charge", + model: response_model(model), + context: ReqLLM.Context.new(messages), + message: ReqLLM.Context.assistant("pong"), + usage: Keyword.fetch!(opts, :stub_usage) + }} + end + + defp response_model(model) when is_binary(model), do: model + defp response_model(model), do: inspect(model) + end + defmodule StreamingUsageStub do def generate_text(_model, _messages, _opts), do: {:error, :stream_expected} @@ -930,6 +946,115 @@ defmodule ReqLLMClientTest do refute_received {:req_llm_generate, ^model, _messages, _opts} end + @catalog %{total: 0.001858} + + defp byok_response(model, usage, opts) do + lm = Imp.req_llm(model, [req_module: ChargeStub] ++ opts) + + request = + Imp.LM.new_request(lm, [%{role: :user, content: "ping"}], [stub_usage: usage], "byok") + + assert {:ok, response} = Imp.Clients.ReqLLM.request(lm, request) + response + end + + # With the caller's own provider key, OpenRouter's "cost" is its fee only, + # and the provider's own charge is reported beside it. + test "an OpenRouter call on the caller's own key costs the fee plus the upstream charge" do + usage = %{ + "is_byok" => true, + "cost" => 0.0001, + "cost_details" => %{"upstream_inference_cost" => 0.002}, + cost: @catalog + } + + response = byok_response("openrouter:byok", usage, cache: false) + + assert_in_delta response.cost, 0.0021, 1.0e-12 + assert response.estimated_cost == 0.001858 + assert response.usage["cost"] == 0.0001 + assert response.usage["is_byok"] == true + end + + test "an OpenRouter call on the caller's own key with no upstream charge has no cost" do + for usage <- [ + %{"is_byok" => true, "cost" => 0.0001, "cost_details" => nil, cost: @catalog}, + %{"is_byok" => true, "cost" => 0.0001, "cost_details" => %{}, cost: @catalog}, + %{ + "is_byok" => true, + "cost" => 0.0001, + "cost_details" => %{"upstream_inference_cost" => nil}, + cost: @catalog + }, + %{ + "is_byok" => true, + "cost" => "0.0001", + "cost_details" => %{"upstream_inference_cost" => 0.002}, + cost: @catalog + }, + %{ + "is_byok" => true, + "cost" => 0.0001, + "cost_details" => %{"upstream_inference_cost" => "0.002"}, + cost: @catalog + } + ] do + response = byok_response("openrouter:byok", usage, cache: false) + + assert response.cost == nil, inspect(usage["cost_details"] || usage["cost"]) + assert response.usage["cost"] == usage["cost"] + assert response.estimated_cost == 0.001858 + end + end + + # `openrouter:` strings are not the only spec this client accepts. + # `"is_byok"` is what identifies the call, so each of these costs the fee + # plus the upstream charge, and the client records that figure as `:charge`. + test "a byok response on an OpenRouter model spec costs the fee plus the upstream charge" do + usage = %{ + "is_byok" => true, + "cost" => 0.0001, + "cost_details" => %{"upstream_inference_cost" => 0.002}, + cost: @catalog + } + + for model <- [ + {:openrouter, "id"}, + {:openrouter, "id", []}, + %{"provider" => "openrouter"} + ] do + response = byok_response(model, usage, cache: false) + + assert_in_delta response.cost, 0.0021, 1.0e-12 + assert_in_delta response.metadata.req_llm.charge, 0.0021, 1.0e-12 + assert response.usage["cost"] == 0.0001 + end + end + + test "a repeated OpenRouter call on the caller's own key does not bill the cached charge" do + Imp.Cache.clear() + model = "openrouter:byok-cache-#{System.unique_integer([:positive])}" + + usage = %{ + "is_byok" => true, + "cost" => 0.0001, + "cost_details" => %{"upstream_inference_cost" => 0.002}, + cost: @catalog + } + + lm = Imp.req_llm(model, req_module: ChargeStub) + + request = + Imp.LM.new_request(lm, [%{role: :user, content: "ping"}], [stub_usage: usage], "byok") + + assert {:ok, first} = Imp.Clients.ReqLLM.request(lm, request) + assert_in_delta first.cost, 0.0021, 1.0e-12 + + assert {:ok, second} = Imp.Clients.ReqLLM.request(lm, request) + assert second.cost == nil + assert second.usage == %{} + end + test "ReqLLM cache behavior isolates endpoints and semantic secret-shaped values" do model = "openai:gpt-cache-routing-#{System.unique_integer([:positive])}" messages = [%{role: :user, content: "same prompt"}] diff --git a/test/req_llm_stream_end_test.exs b/test/req_llm_stream_end_test.exs index 39fd250a..1fd6c891 100644 --- a/test/req_llm_stream_end_test.exs +++ b/test/req_llm_stream_end_test.exs @@ -31,6 +31,20 @@ defmodule Imp.ReqLLMStreamEndTest do } } + @byok_usage %{ + "id" => "gen-1", + "model" => "local-model", + "choices" => [], + "usage" => %{ + "prompt_tokens" => 7, + "completion_tokens" => 1, + "total_tokens" => 8, + "cost" => 0.0001, + "is_byok" => true, + "cost_details" => %{"upstream_inference_cost" => 0.002} + } + } + defp sse(events, done? \\ true) do body = Enum.map_join(events, "", &("data: " <> Jason.encode!(&1) <> "\n\n")) if done?, do: body <> "data: [DONE]\n\n", else: body @@ -220,6 +234,27 @@ defmodule Imp.ReqLLMStreamEndTest do assert response.metadata.cost == 0.00042 end + test "an OpenRouter stream on the caller's own key costs the fee plus the upstream charge" do + answer = "[[ ## answer ## ]]\nParis\n\n[[ ## completed ## ]]" + content = put_in(@content, ["choices", Access.at(0), "delta", "content"], answer) + client = lm(sse([content, @finish, @byok_usage])) + + assert %StreamResponse{done: true, metadata: %{usage: usage}} = + client |> stream() |> List.last() + + assert usage["cost"] == 0.0001 + + program = Imp.predict("question -> answer", lm: client) + {:ok, run} = Imp.Run.start(%Collecting{program: program}, %{question: "Capital of France?"}) + assert {:ok, _prediction} = Task.await(run.task) + events = Imp.Run.events(run) + Imp.Run.stop(run) + + assert [response] = Enum.filter(events, &(&1.kind == :model_response)) + assert_in_delta response.metadata.cost, 0.0021, 1.0e-12 + assert response.metadata.usage["cost"] == 0.0001 + end + defp run(program) do {:ok, run} = Imp.Run.start(%Collecting{program: program}, %{question: "Capital of France?"}) @@ -242,6 +277,28 @@ defmodule Imp.ReqLLMStreamEndTest do assert response.metadata.cost == 0.00042 end + test "a stream that fails after a byok usage chunk records the resolved charge" do + error = %{"error" => %{"message" => "upstream died"}} + + charged = + Imp.predict("question -> answer", lm: lm(sse([@content, @byok_usage, error], false))) + + assert {{:error, %Imp.LMError{}}, _usage, events} = run(charged) + assert [response] = Enum.filter(events, &(&1.kind == :model_response)) + assert_in_delta response.metadata.cost, 0.0021, 1.0e-12 + assert response.metadata.usage["cost"] == 0.0001 + + bare = %{@byok_usage | "usage" => Map.delete(@byok_usage["usage"], "cost_details")} + + missing = + Imp.predict("question -> answer", lm: lm(sse([@content, bare, error], false))) + + assert {{:error, %Imp.LMError{}}, _usage, missing_events} = run(missing) + assert [partial] = Enum.filter(missing_events, &(&1.kind == :model_response)) + assert partial.metadata.cost == nil + assert partial.metadata.usage["cost"] == 0.0001 + end + test "a failure that carries a partial response counts its usage and returns the reason" do lm = Imp.req_llm(%{provider: :openrouter, id: "m", model: "m"}, cache: false) request = Imp.LM.new_request(lm, [%{role: :user, content: "ping"}], [], "test")