From 033004bdb631b5cd224ae9e832d262e78d48f8f7 Mon Sep 17 00:00:00 2001 From: Laurianti Date: Fri, 25 Sep 2026 00:17:21 +0200 Subject: [PATCH 1/4] Fixed `RawRepresentationFactory` settings being overwritten in `AsChatClient` --- .../LLamaExecutorChatClientTests.cs | 104 ++++++++++++++++++ LLama/Extensions/LLamaExecutorExtensions.cs | 17 ++- 2 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 LLama.Unittest/LLamaExecutorChatClientTests.cs diff --git a/LLama.Unittest/LLamaExecutorChatClientTests.cs b/LLama.Unittest/LLamaExecutorChatClientTests.cs new file mode 100644 index 000000000..f7ba0fd9f --- /dev/null +++ b/LLama.Unittest/LLamaExecutorChatClientTests.cs @@ -0,0 +1,104 @@ +using System.Runtime.CompilerServices; +using LLama.Abstractions; +using LLama.Common; +using LLama.Native; +using LLama.Sampling; +using Microsoft.Extensions.AI; + +namespace LLama.Unittest; + +public sealed class LLamaExecutorChatClientTests +{ + [Fact] + public async Task RawRepresentationSettingsAreKept() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + RawRepresentationFactory = _ => new InferenceParams + { + MaxTokens = 1024, + SamplingPipeline = new DefaultSamplingPipeline { Temperature = 0.1f, TopP = 0.5f, TopK = 5, Seed = 42 }, + }, + }); + + var pipeline = Assert.IsType(executor.InferenceParams?.SamplingPipeline); + Assert.Equal(1024, executor.InferenceParams!.MaxTokens); + Assert.Equal(0.1f, pipeline.Temperature); + Assert.Equal(0.5f, pipeline.TopP); + Assert.Equal(5, pipeline.TopK); + Assert.Equal(42u, pipeline.Seed); + } + + [Fact] + public async Task ChatOptionsOverrideRawRepresentationSettings() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + MaxOutputTokens = 64, + Temperature = 0.9f, + TopP = 0.8f, + TopK = 20, + Seed = 7, + RawRepresentationFactory = _ => new InferenceParams + { + MaxTokens = 1024, + SamplingPipeline = new DefaultSamplingPipeline { Temperature = 0.1f, TopP = 0.5f, TopK = 5, Seed = 42 }, + }, + }); + + var pipeline = Assert.IsType(executor.InferenceParams?.SamplingPipeline); + Assert.Equal(64, executor.InferenceParams!.MaxTokens); + Assert.Equal(0.9f, pipeline.Temperature); + Assert.Equal(0.8f, pipeline.TopP); + Assert.Equal(20, pipeline.TopK); + Assert.Equal(7u, pipeline.Seed); + } + + [Fact] + public async Task RawRepresentationSamplingPipelineIsKept() + { + var executor = new CapturingExecutor(); + var greedy = new GreedySamplingPipeline(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + RawRepresentationFactory = _ => new InferenceParams { SamplingPipeline = greedy }, + }); + + Assert.Same(greedy, executor.InferenceParams?.SamplingPipeline); + } + + [Fact] + public async Task MaxTokensDefaultsTo256WithoutALimit() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello"); + + Assert.Equal(256, executor.InferenceParams!.MaxTokens); + } + + /// + /// Records the inference parameters it is given, without a model. + /// + private sealed class CapturingExecutor : ILLamaExecutor + { + public IInferenceParams? InferenceParams { get; private set; } + + public LLamaContext Context => throw new NotSupportedException(); + public bool IsMultiModal => false; + public MtmdWeights? ClipModel => null; + public List Embeds { get; } = []; + + public async IAsyncEnumerable InferAsync(string text, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken token = default) + { + InferenceParams = inferenceParams; + await Task.Yield(); + yield return "hi"; + } + } +} diff --git a/LLama/Extensions/LLamaExecutorExtensions.cs b/LLama/Extensions/LLamaExecutorExtensions.cs index 7fd799f6a..e866105dd 100644 --- a/LLama/Extensions/LLamaExecutorExtensions.cs +++ b/LLama/Extensions/LLamaExecutorExtensions.cs @@ -141,7 +141,14 @@ private InferenceParams CreateInferenceParams(ChatOptions? options) InferenceParams ip = options?.RawRepresentationFactory?.Invoke(this) as InferenceParams ?? new(); ip.AntiPrompts = [.. s_antiPrompts, .. ip.AntiPrompts]; - ip.MaxTokens = options?.MaxOutputTokens ?? 256; // arbitrary upper limit + ip.MaxTokens = options?.MaxOutputTokens ?? (ip.MaxTokens > 0 ? ip.MaxTokens : 256); // arbitrary upper limit when none is given + + // A sampling pipeline other than DefaultSamplingPipeline is kept as given: the chat options cannot be applied to it. + if (ip.SamplingPipeline is not null and not DefaultSamplingPipeline) + { + return ip; + } + ip.SamplingPipeline = new DefaultSamplingPipeline() { FrequencyPenalty = options?.FrequencyPenalty ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.FrequencyPenalty ?? s_defaultPipeline.FrequencyPenalty, @@ -155,10 +162,10 @@ private InferenceParams CreateInferenceParams(ChatOptions? options) LogitBias = (ip.SamplingPipeline as DefaultSamplingPipeline)?.LogitBias ?? s_defaultPipeline.LogitBias, MinKeep = (ip.SamplingPipeline as DefaultSamplingPipeline)?.MinKeep ?? s_defaultPipeline.MinKeep, MinP = (ip.SamplingPipeline as DefaultSamplingPipeline)?.MinP ?? s_defaultPipeline.MinP, - Seed = options?.Seed is long seed ? (uint)seed : (uint)(t_random ??= new()).Next(), - Temperature = options?.Temperature ?? s_defaultPipeline.Temperature, - TopP = options?.TopP ?? s_defaultPipeline.TopP, - TopK = options?.TopK ?? s_defaultPipeline.TopK, + Seed = options?.Seed is long seed ? (uint)seed : (ip.SamplingPipeline as DefaultSamplingPipeline)?.Seed ?? (uint)(t_random ??= new()).Next(), + Temperature = options?.Temperature ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.Temperature ?? s_defaultPipeline.Temperature, + TopP = options?.TopP ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.TopP ?? s_defaultPipeline.TopP, + TopK = options?.TopK ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.TopK ?? s_defaultPipeline.TopK, TypicalP = (ip.SamplingPipeline as DefaultSamplingPipeline)?.TypicalP ?? s_defaultPipeline.TypicalP, }; From b486ee70f7732e5adb2e78b13574e62034620d27 Mon Sep 17 00:00:00 2001 From: Laurianti Date: Fri, 25 Sep 2026 06:44:00 +0200 Subject: [PATCH 2/4] Kept a `MaxTokens` of -1 (unlimited) from `RawRepresentationFactory` --- LLama.Unittest/LLamaExecutorChatClientTests.cs | 13 +++++++++++++ LLama/Extensions/LLamaExecutorExtensions.cs | 5 +++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/LLama.Unittest/LLamaExecutorChatClientTests.cs b/LLama.Unittest/LLamaExecutorChatClientTests.cs index f7ba0fd9f..95fba1940 100644 --- a/LLama.Unittest/LLamaExecutorChatClientTests.cs +++ b/LLama.Unittest/LLamaExecutorChatClientTests.cs @@ -31,6 +31,19 @@ public async Task RawRepresentationSettingsAreKept() Assert.Equal(42u, pipeline.Seed); } + [Fact] + public async Task RawRepresentationUnlimitedMaxTokensIsKept() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + RawRepresentationFactory = _ => new InferenceParams { MaxTokens = -1 }, + }); + + Assert.Equal(-1, executor.InferenceParams!.MaxTokens); + } + [Fact] public async Task ChatOptionsOverrideRawRepresentationSettings() { diff --git a/LLama/Extensions/LLamaExecutorExtensions.cs b/LLama/Extensions/LLamaExecutorExtensions.cs index e866105dd..1597a985b 100644 --- a/LLama/Extensions/LLamaExecutorExtensions.cs +++ b/LLama/Extensions/LLamaExecutorExtensions.cs @@ -138,10 +138,11 @@ private string CreatePrompt(IEnumerable messages, ChatOptions? opti /// Convert the chat options to inference parameters. private InferenceParams CreateInferenceParams(ChatOptions? options) { - InferenceParams ip = options?.RawRepresentationFactory?.Invoke(this) as InferenceParams ?? new(); + var raw = options?.RawRepresentationFactory?.Invoke(this) as InferenceParams; + InferenceParams ip = raw ?? new(); ip.AntiPrompts = [.. s_antiPrompts, .. ip.AntiPrompts]; - ip.MaxTokens = options?.MaxOutputTokens ?? (ip.MaxTokens > 0 ? ip.MaxTokens : 256); // arbitrary upper limit when none is given + ip.MaxTokens = options?.MaxOutputTokens ?? (raw is null ? 256 : ip.MaxTokens); // arbitrary upper limit when no InferenceParams are given // A sampling pipeline other than DefaultSamplingPipeline is kept as given: the chat options cannot be applied to it. if (ip.SamplingPipeline is not null and not DefaultSamplingPipeline) From 70b604401e44c2c8e18e6fec99cd8b04123bc43e Mon Sep 17 00:00:00 2001 From: Laurianti Date: Fri, 25 Sep 2026 15:07:55 +0200 Subject: [PATCH 3/4] Reused a `DefaultSamplingPipeline` from `RawRepresentationFactory` when only the seed changes --- LLama.Unittest/LLamaExecutorChatClientTests.cs | 17 +++++++++++++++++ LLama/Extensions/LLamaExecutorExtensions.cs | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/LLama.Unittest/LLamaExecutorChatClientTests.cs b/LLama.Unittest/LLamaExecutorChatClientTests.cs index 95fba1940..9573bb45f 100644 --- a/LLama.Unittest/LLamaExecutorChatClientTests.cs +++ b/LLama.Unittest/LLamaExecutorChatClientTests.cs @@ -85,6 +85,23 @@ public async Task RawRepresentationSamplingPipelineIsKept() Assert.Same(greedy, executor.InferenceParams?.SamplingPipeline); } + [Fact] + public async Task RawRepresentationDefaultSamplingPipelineIsReused() + { + var executor = new CapturingExecutor(); + var sampling = new DefaultSamplingPipeline { Temperature = 0.1f, Seed = 42 }; + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + Seed = 7, + RawRepresentationFactory = _ => new InferenceParams { SamplingPipeline = sampling }, + }); + + Assert.Same(sampling, executor.InferenceParams?.SamplingPipeline); + Assert.Equal(0.1f, sampling.Temperature); + Assert.Equal(7u, sampling.Seed); + } + [Fact] public async Task MaxTokensDefaultsTo256WithoutALimit() { diff --git a/LLama/Extensions/LLamaExecutorExtensions.cs b/LLama/Extensions/LLamaExecutorExtensions.cs index 1597a985b..97312822c 100644 --- a/LLama/Extensions/LLamaExecutorExtensions.cs +++ b/LLama/Extensions/LLamaExecutorExtensions.cs @@ -150,6 +150,20 @@ private InferenceParams CreateInferenceParams(ChatOptions? options) return ip; } + // A DefaultSamplingPipeline is reused when the chat options change nothing but the seed, which can be + // set on it; the other settings can only be given when it is constructed. + if (ip.SamplingPipeline is DefaultSamplingPipeline existing + && options is not { FrequencyPenalty: not null } and not { PresencePenalty: not null } + and not { Temperature: not null } and not { TopP: not null } and not { TopK: not null }) + { + if (options?.Seed is long existingSeed) + { + existing.Seed = (uint)existingSeed; + } + + return ip; + } + ip.SamplingPipeline = new DefaultSamplingPipeline() { FrequencyPenalty = options?.FrequencyPenalty ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.FrequencyPenalty ?? s_defaultPipeline.FrequencyPenalty, From 1f107081acd5870eb7d935da8e955779ef54982d Mon Sep 17 00:00:00 2001 From: Laurianti Date: Fri, 25 Sep 2026 17:23:19 +0200 Subject: [PATCH 4/4] Reverted reusing a `DefaultSamplingPipeline`: its native chain is cached on first use --- LLama.Unittest/LLamaExecutorChatClientTests.cs | 17 ----------------- LLama/Extensions/LLamaExecutorExtensions.cs | 14 -------------- 2 files changed, 31 deletions(-) diff --git a/LLama.Unittest/LLamaExecutorChatClientTests.cs b/LLama.Unittest/LLamaExecutorChatClientTests.cs index 9573bb45f..95fba1940 100644 --- a/LLama.Unittest/LLamaExecutorChatClientTests.cs +++ b/LLama.Unittest/LLamaExecutorChatClientTests.cs @@ -85,23 +85,6 @@ public async Task RawRepresentationSamplingPipelineIsKept() Assert.Same(greedy, executor.InferenceParams?.SamplingPipeline); } - [Fact] - public async Task RawRepresentationDefaultSamplingPipelineIsReused() - { - var executor = new CapturingExecutor(); - var sampling = new DefaultSamplingPipeline { Temperature = 0.1f, Seed = 42 }; - - await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions - { - Seed = 7, - RawRepresentationFactory = _ => new InferenceParams { SamplingPipeline = sampling }, - }); - - Assert.Same(sampling, executor.InferenceParams?.SamplingPipeline); - Assert.Equal(0.1f, sampling.Temperature); - Assert.Equal(7u, sampling.Seed); - } - [Fact] public async Task MaxTokensDefaultsTo256WithoutALimit() { diff --git a/LLama/Extensions/LLamaExecutorExtensions.cs b/LLama/Extensions/LLamaExecutorExtensions.cs index 97312822c..1597a985b 100644 --- a/LLama/Extensions/LLamaExecutorExtensions.cs +++ b/LLama/Extensions/LLamaExecutorExtensions.cs @@ -150,20 +150,6 @@ private InferenceParams CreateInferenceParams(ChatOptions? options) return ip; } - // A DefaultSamplingPipeline is reused when the chat options change nothing but the seed, which can be - // set on it; the other settings can only be given when it is constructed. - if (ip.SamplingPipeline is DefaultSamplingPipeline existing - && options is not { FrequencyPenalty: not null } and not { PresencePenalty: not null } - and not { Temperature: not null } and not { TopP: not null } and not { TopK: not null }) - { - if (options?.Seed is long existingSeed) - { - existing.Seed = (uint)existingSeed; - } - - return ip; - } - ip.SamplingPipeline = new DefaultSamplingPipeline() { FrequencyPenalty = options?.FrequencyPenalty ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.FrequencyPenalty ?? s_defaultPipeline.FrequencyPenalty,