Fix RawRepresentationFactory settings being overwritten in AsChatClient

LLama.Unittest/LLamaExecutorChatClientTests.cs

@@ -0,0 +1,117 @@
+using System.Runtime.CompilerServices;
+using LLama.Abstractions;
+using LLama.Common;
+using LLama.Native;
+using LLama.Sampling;
+using Microsoft.Extensions.AI;
+
+namespace LLama.Unittest;
+
+public sealed class LLamaExecutorChatClientTests
+{
+    [Fact]
+    public async Task RawRepresentationSettingsAreKept()
+    {
+        var executor = new CapturingExecutor();
+
+        await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions
+        {
+            RawRepresentationFactory = _ => new InferenceParams
+            {
+                MaxTokens = 1024,
+                SamplingPipeline = new DefaultSamplingPipeline { Temperature = 0.1f, TopP = 0.5f, TopK = 5, Seed = 42 },
+            },
+        });
+
+        var pipeline = Assert.IsType<DefaultSamplingPipeline>(executor.InferenceParams?.SamplingPipeline);
+        Assert.Equal(1024, executor.InferenceParams!.MaxTokens);
+        Assert.Equal(0.1f, pipeline.Temperature);
+        Assert.Equal(0.5f, pipeline.TopP);
+        Assert.Equal(5, pipeline.TopK);
+        Assert.Equal(42u, pipeline.Seed);
+    }
+
+    [Fact]
+    public async Task RawRepresentationUnlimitedMaxTokensIsKept()
+    {
+        var executor = new CapturingExecutor();
+
+        await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions
+        {
+            RawRepresentationFactory = _ => new InferenceParams { MaxTokens = -1 },
+        });
+
+        Assert.Equal(-1, executor.InferenceParams!.MaxTokens);
+    }
+
+    [Fact]
+    public async Task ChatOptionsOverrideRawRepresentationSettings()
+    {
+        var executor = new CapturingExecutor();
+
+        await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions
+        {
+            MaxOutputTokens = 64,
+            Temperature = 0.9f,
+            TopP = 0.8f,
+            TopK = 20,
+            Seed = 7,
+            RawRepresentationFactory = _ => new InferenceParams
+            {
+                MaxTokens = 1024,
+                SamplingPipeline = new DefaultSamplingPipeline { Temperature = 0.1f, TopP = 0.5f, TopK = 5, Seed = 42 },
+            },
+        });
+
+        var pipeline = Assert.IsType<DefaultSamplingPipeline>(executor.InferenceParams?.SamplingPipeline);
+        Assert.Equal(64, executor.InferenceParams!.MaxTokens);
+        Assert.Equal(0.9f, pipeline.Temperature);
+        Assert.Equal(0.8f, pipeline.TopP);
+        Assert.Equal(20, pipeline.TopK);
+        Assert.Equal(7u, pipeline.Seed);
+    }
+
+    [Fact]
+    public async Task RawRepresentationSamplingPipelineIsKept()
+    {
+        var executor = new CapturingExecutor();
+        var greedy = new GreedySamplingPipeline();
+
+        await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions
+        {
+            RawRepresentationFactory = _ => new InferenceParams { SamplingPipeline = greedy },
+        });
+
+        Assert.Same(greedy, executor.InferenceParams?.SamplingPipeline);
+    }
+
+    [Fact]
+    public async Task MaxTokensDefaultsTo256WithoutALimit()
+    {
+        var executor = new CapturingExecutor();
+
+        await executor.AsChatClient().GetResponseAsync("hello");
+
+        Assert.Equal(256, executor.InferenceParams!.MaxTokens);
+    }
+
+    /// <summary>
+    /// Records the inference parameters it is given, without a model.
+    /// </summary>
+    private sealed class CapturingExecutor : ILLamaExecutor
+    {
+        public IInferenceParams? InferenceParams { get; private set; }
+
+        public LLamaContext Context => throw new NotSupportedException();
+        public bool IsMultiModal => false;
+        public MtmdWeights? ClipModel => null;
+        public List<SafeMtmdEmbed> Embeds { get; } = [];
+
+        public async IAsyncEnumerable<string> InferAsync(string text, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken token = default)
+        {
+            InferenceParams = inferenceParams;
+            await Task.Yield();
+            yield return "hi";
+        }
+    }
+}

LLama/Extensions/LLamaExecutorExtensions.cs

@@ -138,10 +138,18 @@ private string CreatePrompt(IEnumerable<ChatMessage> messages, ChatOptions? opti
         /// <summary>Convert the chat options to inference parameters.</summary>
         private InferenceParams CreateInferenceParams(ChatOptions? options)
         {
-            InferenceParams ip = options?.RawRepresentationFactory?.Invoke(this) as InferenceParams ?? new();
+            var raw = options?.RawRepresentationFactory?.Invoke(this) as InferenceParams;
+            InferenceParams ip = raw ?? new();
 
             ip.AntiPrompts = [.. s_antiPrompts, .. ip.AntiPrompts];
-            ip.MaxTokens = options?.MaxOutputTokens ?? 256; // arbitrary upper limit
+            ip.MaxTokens = options?.MaxOutputTokens ?? (raw is null ? 256 : ip.MaxTokens); // arbitrary upper limit when no InferenceParams are given
+
+            // A sampling pipeline other than DefaultSamplingPipeline is kept as given: the chat options cannot be applied to it.
+            if (ip.SamplingPipeline is not null and not DefaultSamplingPipeline)
+            {
+                return ip;
+            }
+
             ip.SamplingPipeline = new DefaultSamplingPipeline()
             {
                 FrequencyPenalty = options?.FrequencyPenalty ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.FrequencyPenalty ?? s_defaultPipeline.FrequencyPenalty,
@@ -155,10 +163,10 @@ private InferenceParams CreateInferenceParams(ChatOptions? options)
                 LogitBias = (ip.SamplingPipeline as DefaultSamplingPipeline)?.LogitBias ?? s_defaultPipeline.LogitBias,
                 MinKeep = (ip.SamplingPipeline as DefaultSamplingPipeline)?.MinKeep ?? s_defaultPipeline.MinKeep,
                 MinP = (ip.SamplingPipeline as DefaultSamplingPipeline)?.MinP ?? s_defaultPipeline.MinP,
-                Seed = options?.Seed is long seed ? (uint)seed : (uint)(t_random ??= new()).Next(),
-                Temperature = options?.Temperature ?? s_defaultPipeline.Temperature,
-                TopP = options?.TopP ?? s_defaultPipeline.TopP,
-                TopK = options?.TopK ?? s_defaultPipeline.TopK,
+                Seed = options?.Seed is long seed ? (uint)seed : (ip.SamplingPipeline as DefaultSamplingPipeline)?.Seed ?? (uint)(t_random ??= new()).Next(),
+                Temperature = options?.Temperature ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.Temperature ?? s_defaultPipeline.Temperature,
+                TopP = options?.TopP ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.TopP ?? s_defaultPipeline.TopP,
+                TopK = options?.TopK ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.TopK ?? s_defaultPipeline.TopK,
                 TypicalP = (ip.SamplingPipeline as DefaultSamplingPipeline)?.TypicalP ?? s_defaultPipeline.TypicalP,
             };