diff --git a/LLama.Unittest/LLamaExecutorChatClientTests.cs b/LLama.Unittest/LLamaExecutorChatClientTests.cs new file mode 100644 index 000000000..95fba1940 --- /dev/null +++ b/LLama.Unittest/LLamaExecutorChatClientTests.cs @@ -0,0 +1,117 @@ +using System.Runtime.CompilerServices; +using LLama.Abstractions; +using LLama.Common; +using LLama.Native; +using LLama.Sampling; +using Microsoft.Extensions.AI; + +namespace LLama.Unittest; + +public sealed class LLamaExecutorChatClientTests +{ + [Fact] + public async Task RawRepresentationSettingsAreKept() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + RawRepresentationFactory = _ => new InferenceParams + { + MaxTokens = 1024, + SamplingPipeline = new DefaultSamplingPipeline { Temperature = 0.1f, TopP = 0.5f, TopK = 5, Seed = 42 }, + }, + }); + + var pipeline = Assert.IsType(executor.InferenceParams?.SamplingPipeline); + Assert.Equal(1024, executor.InferenceParams!.MaxTokens); + Assert.Equal(0.1f, pipeline.Temperature); + Assert.Equal(0.5f, pipeline.TopP); + Assert.Equal(5, pipeline.TopK); + Assert.Equal(42u, pipeline.Seed); + } + + [Fact] + public async Task RawRepresentationUnlimitedMaxTokensIsKept() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + RawRepresentationFactory = _ => new InferenceParams { MaxTokens = -1 }, + }); + + Assert.Equal(-1, executor.InferenceParams!.MaxTokens); + } + + [Fact] + public async Task ChatOptionsOverrideRawRepresentationSettings() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + MaxOutputTokens = 64, + Temperature = 0.9f, + TopP = 0.8f, + TopK = 20, + Seed = 7, + RawRepresentationFactory = _ => new InferenceParams + { + MaxTokens = 1024, + SamplingPipeline = new DefaultSamplingPipeline { Temperature = 0.1f, TopP = 0.5f, TopK = 5, Seed = 42 }, + }, + }); + + var pipeline = Assert.IsType(executor.InferenceParams?.SamplingPipeline); + Assert.Equal(64, executor.InferenceParams!.MaxTokens); + Assert.Equal(0.9f, pipeline.Temperature); + Assert.Equal(0.8f, pipeline.TopP); + Assert.Equal(20, pipeline.TopK); + Assert.Equal(7u, pipeline.Seed); + } + + [Fact] + public async Task RawRepresentationSamplingPipelineIsKept() + { + var executor = new CapturingExecutor(); + var greedy = new GreedySamplingPipeline(); + + await executor.AsChatClient().GetResponseAsync("hello", new ChatOptions + { + RawRepresentationFactory = _ => new InferenceParams { SamplingPipeline = greedy }, + }); + + Assert.Same(greedy, executor.InferenceParams?.SamplingPipeline); + } + + [Fact] + public async Task MaxTokensDefaultsTo256WithoutALimit() + { + var executor = new CapturingExecutor(); + + await executor.AsChatClient().GetResponseAsync("hello"); + + Assert.Equal(256, executor.InferenceParams!.MaxTokens); + } + + /// + /// Records the inference parameters it is given, without a model. + /// + private sealed class CapturingExecutor : ILLamaExecutor + { + public IInferenceParams? InferenceParams { get; private set; } + + public LLamaContext Context => throw new NotSupportedException(); + public bool IsMultiModal => false; + public MtmdWeights? ClipModel => null; + public List Embeds { get; } = []; + + public async IAsyncEnumerable InferAsync(string text, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken token = default) + { + InferenceParams = inferenceParams; + await Task.Yield(); + yield return "hi"; + } + } +} diff --git a/LLama/Extensions/LLamaExecutorExtensions.cs b/LLama/Extensions/LLamaExecutorExtensions.cs index 7fd799f6a..1597a985b 100644 --- a/LLama/Extensions/LLamaExecutorExtensions.cs +++ b/LLama/Extensions/LLamaExecutorExtensions.cs @@ -138,10 +138,18 @@ private string CreatePrompt(IEnumerable messages, ChatOptions? opti /// Convert the chat options to inference parameters. private InferenceParams CreateInferenceParams(ChatOptions? options) { - InferenceParams ip = options?.RawRepresentationFactory?.Invoke(this) as InferenceParams ?? new(); + var raw = options?.RawRepresentationFactory?.Invoke(this) as InferenceParams; + InferenceParams ip = raw ?? new(); ip.AntiPrompts = [.. s_antiPrompts, .. ip.AntiPrompts]; - ip.MaxTokens = options?.MaxOutputTokens ?? 256; // arbitrary upper limit + ip.MaxTokens = options?.MaxOutputTokens ?? (raw is null ? 256 : ip.MaxTokens); // arbitrary upper limit when no InferenceParams are given + + // A sampling pipeline other than DefaultSamplingPipeline is kept as given: the chat options cannot be applied to it. + if (ip.SamplingPipeline is not null and not DefaultSamplingPipeline) + { + return ip; + } + ip.SamplingPipeline = new DefaultSamplingPipeline() { FrequencyPenalty = options?.FrequencyPenalty ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.FrequencyPenalty ?? s_defaultPipeline.FrequencyPenalty, @@ -155,10 +163,10 @@ private InferenceParams CreateInferenceParams(ChatOptions? options) LogitBias = (ip.SamplingPipeline as DefaultSamplingPipeline)?.LogitBias ?? s_defaultPipeline.LogitBias, MinKeep = (ip.SamplingPipeline as DefaultSamplingPipeline)?.MinKeep ?? s_defaultPipeline.MinKeep, MinP = (ip.SamplingPipeline as DefaultSamplingPipeline)?.MinP ?? s_defaultPipeline.MinP, - Seed = options?.Seed is long seed ? (uint)seed : (uint)(t_random ??= new()).Next(), - Temperature = options?.Temperature ?? s_defaultPipeline.Temperature, - TopP = options?.TopP ?? s_defaultPipeline.TopP, - TopK = options?.TopK ?? s_defaultPipeline.TopK, + Seed = options?.Seed is long seed ? (uint)seed : (ip.SamplingPipeline as DefaultSamplingPipeline)?.Seed ?? (uint)(t_random ??= new()).Next(), + Temperature = options?.Temperature ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.Temperature ?? s_defaultPipeline.Temperature, + TopP = options?.TopP ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.TopP ?? s_defaultPipeline.TopP, + TopK = options?.TopK ?? (ip.SamplingPipeline as DefaultSamplingPipeline)?.TopK ?? s_defaultPipeline.TopK, TypicalP = (ip.SamplingPipeline as DefaultSamplingPipeline)?.TypicalP ?? s_defaultPipeline.TypicalP, };