diff --git a/Assets/icon.png b/Assets/icon.png new file mode 100644 index 000000000..8745d7e2a Binary files /dev/null and b/Assets/icon.png differ diff --git a/LLama/Batched/Conversation.cs b/LLama/Batched/Conversation.cs index 01e8934a9..15a698618 100644 --- a/LLama/Batched/Conversation.cs +++ b/LLama/Batched/Conversation.cs @@ -23,9 +23,6 @@ public sealed class Conversation /// private bool _forked; private MtmdChunkSequence? _pendingMtmdSequence; - private readonly List _embed_inps = new(); - private readonly List _session_tokens = new(); - private int _consumedTokensCount; /// /// Stores the indices to sample from. Contains valid items. @@ -251,6 +248,12 @@ private void AssertCanBePrompted() throw new AlreadyPromptedConversationException(); } + /// + /// Add text to this conversation + /// + /// Text to tokenize and append + /// Whether to add a bos to the text. + /// Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. public void Prompt(string promptText, bool addBos = true, bool special = true) { var tokens = Executor.Context.Tokenize(promptText, addBos, special); @@ -498,18 +501,6 @@ internal void OnMtmdEvaluationCompleted(int newPast, MtmdChunkSequence sequence) _requiredEpoch = Executor.Epoch + 1; _forked = false; - if (sequence.TextTokens.Count > 0) - { - _embed_inps.AddRange(sequence.TextTokens); - _session_tokens.AddRange(sequence.TextTokens); - } - - var fillerToken = GetFillerToken(Executor.GetMtmdMarker()); - var fillerCount = Math.Max(0, sequence.TotalPositions - sequence.TotalTokens); - for (var i = 0; i < fillerCount; i++) - _embed_inps.Add(fillerToken); - - _consumedTokensCount = _embed_inps.Count; sequence.Dispose(); } diff --git a/LLama/LLamaInteractExecutor.cs b/LLama/LLamaInteractExecutor.cs index 104758dbd..3afb137fb 100644 --- a/LLama/LLamaInteractExecutor.cs +++ b/LLama/LLamaInteractExecutor.cs @@ -21,12 +21,7 @@ namespace LLama public class InteractiveExecutor : StatefulExecutorBase { // Indicates whether the executor is currently evaluating the initial prompt or a follow-up turn. - private bool _is_prompt_run = true; - - // MTMD multimodal state - private SafeMtmdInputChunks? _mtmdChunks; // Pending chunk collection produced by the multimodal tokenizer. - private string? _mtmdMarker; // Cached multimodal marker returned by the native helper. - + private bool _isPromptRun = true; /// /// Create an interactive executor for text-only inference. @@ -55,15 +50,15 @@ public override ExecutorBaseState GetStateData() InteractiveExecutorState state = new() { ConsumedSessionCount = _n_session_consumed, - EmbedInps = _embed_inps.ToArray(), - IsPromptRun = _is_prompt_run, + EmbedInps = [ .. _embed_inps ], + IsPromptRun = _isPromptRun, ConsumedTokensCount = _consumedTokensCount, - Embeds = _embeds.ToArray(), - LastTokens = _last_n_tokens.ToArray(), + Embeds = [ .. _embeds ], + LastTokens = [ .. _last_n_tokens ], MatchingSessionTokensCount = _n_matching_session_tokens, PastTokensCount = _pastTokensCount, SessionFilePath = _pathSession, - SessionTokens = _session_tokens.ToArray(), + SessionTokens = [ .. _session_tokens ], LastTokensCapacity = _last_n_tokens.Capacity, }; return state; @@ -76,15 +71,15 @@ public override Task LoadState(ExecutorBaseState data, CancellationToken cancell if (data is InteractiveExecutorState state) { _n_session_consumed = state.ConsumedSessionCount; - _embed_inps = state.EmbedInps!.ToList(); - _is_prompt_run = state.IsPromptRun; + _embed_inps = [ .. state.EmbedInps ]; + _isPromptRun = state.IsPromptRun; _consumedTokensCount = state.ConsumedTokensCount; - _embeds = state.Embeds!.ToList(); - _last_n_tokens = new FixedSizeQueue(state.LastTokensCapacity, state.LastTokens!); + _embeds = [ .. state.Embeds ]; + _last_n_tokens = new FixedSizeQueue(state.LastTokensCapacity, state.LastTokens); _n_matching_session_tokens = state.MatchingSessionTokensCount; _pastTokensCount = state.PastTokensCount; _pathSession = state.SessionFilePath; - _session_tokens = state.SessionTokens!.ToList(); + _session_tokens = [ .. state.SessionTokens ]; } else throw new ArgumentException("Invalid state data type."); @@ -119,7 +114,7 @@ public override async Task LoadState(string filename, CancellationToken cancella /// true to keep generating; otherwise false. protected override Task GetLoopCondition(InferStateArgs args, CancellationToken cancellationToken = default) { - return Task.FromResult(args.RemainedTokens != 0 && !args.WaitForInput || _is_prompt_run); + return Task.FromResult(args.RemainedTokens != 0 && !args.WaitForInput || _isPromptRun); } /// @@ -130,7 +125,7 @@ protected override Task GetLoopCondition(InferStateArgs args, Cancellation /// protected override Task PreprocessInputs(string? text, InferStateArgs args, CancellationToken cancellationToken = default) { - if (_is_prompt_run) + if (_isPromptRun) { // When running the first input (prompt) in interactive mode, we should specially process it. if (text == null) @@ -140,7 +135,7 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc if (!IsMultiModal) { - _embed_inps = Context.Tokenize(text, true, true).ToList(); + _embed_inps = [ .. Context.Tokenize(text, true, true) ]; } else { @@ -159,9 +154,9 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc if (!IsMultiModal) { - var line_inp = Context.Tokenize(text, false, true); - _embed_inps.AddRange(line_inp); - args.RemainedTokens -= line_inp.Length; + var lineInput = Context.Tokenize(text, false, true); + _embed_inps.AddRange(lineInput); + args.RemainedTokens -= lineInput.Length; } else { @@ -180,7 +175,7 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc /// Mutable inference state. /// /// Tuple describing whether to stop and any additional outputs to emit. - protected override Task<(bool, IReadOnlyList)> PostProcess(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken) + protected override Task<(bool, IReadOnlyList)> PostProcess(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken = default) { if (_embed_inps.Count <= _consumedTokensCount) { @@ -211,13 +206,13 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc } /// - protected override async Task InferInternal(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken) + protected override async Task InferInternal(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken = default) { var batch = new LLamaBatch(); if (_embeds.Count > 0) { - _is_prompt_run = false; + _isPromptRun = false; if (_pastTokensCount + _embeds.Count > Context.ContextSize) { // number of tokens to keep when resetting context @@ -248,7 +243,7 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In } else { - var result = await Context.DecodeAsync(_embeds, LLamaSeqId.Zero, batch, _pastTokensCount); + var result = await Context.DecodeAsync(_embeds, LLamaSeqId.Zero, batch, _pastTokensCount, cancellationToken); _pastTokensCount = result.Item3; if (result.Item1 != DecodeResult.Ok) throw new LLamaDecodeError(result.Item1); @@ -262,7 +257,7 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In } else if (IsMultiModal && MtmdChunks is not null) { - _is_prompt_run = false; + _isPromptRun = false; var nPast = _pastTokensCount; var previousConsumed = _consumedTokensCount; EvaluateMtmdChunks(ref nPast, previousConsumed, nameof(InteractiveExecutor)); @@ -297,8 +292,8 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In id = Context.NativeHandle.ModelHandle.Vocab.Newline!.Value; if (args.Antiprompts is not null && args.Antiprompts.Count > 0) { - var first_antiprompt = Context.Tokenize(args.Antiprompts[0], false, true); - _embed_inps.AddRange(first_antiprompt); + var firstAntiprompt = Context.Tokenize(args.Antiprompts[0], false, true); + _embed_inps.AddRange(firstAntiprompt); } } @@ -320,15 +315,13 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In } } } - - return; } /// /// Serializable state specific to the interactive executor. /// public class InteractiveExecutorState - : StatefulExecutorBase.ExecutorBaseState + : ExecutorBaseState { /// /// Whether the executor is running for the first time (running the prompt). diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 9acd98bfa..12d42ee44 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -14,7 +14,7 @@ MIT, SciSharp STACK $([System.DateTime]::UtcNow.ToString(yyyy)) https://github.com/SciSharp/LLamaSharp git - https://avatars3.githubusercontent.com/u/44989469?s=200&v=4 + https://avatars3.githubusercontent.com/u/44989469 LLama, LLM, GPT, ChatGPT, NLP, AI, Chat Bot, SciSharp LLamaSharp is a cross-platform library to run 🦙LLaMA/Mtmd model (and others) in your local device. diff --git a/LLama/MtmdWeights.cs b/LLama/MtmdWeights.cs index 32dd6b572..b25844a13 100644 --- a/LLama/MtmdWeights.cs +++ b/LLama/MtmdWeights.cs @@ -110,12 +110,6 @@ public int Tokenize(string text, bool addSpecial, bool parseSpecial, ReadOnlySpa public int EvaluateChunks(SafeMtmdInputChunks chunks, SafeLLamaContextHandle llamaContext, ref int nPast, int seqId, int nBatch, bool logitsLast) => NativeHandle.EvaluateChunks(chunks, llamaContext, ref nPast, seqId, nBatch, logitsLast); - public int EvaluateChunk(IntPtr chunkPtr, SafeLLamaContextHandle llamaContext, ref int nPast, int seqId, int nBatch, bool logitsLast) - => NativeHandle.EvaluateChunk(chunkPtr, llamaContext, ref nPast, seqId, nBatch, logitsLast); - - public int DecodeImageChunk(IntPtr chunkPtr, SafeLLamaContextHandle llamaContext, IntPtr encodedEmbeddings, ref int nPast, int seqId, int nBatch) - => NativeHandle.DecodeImageChunk(chunkPtr, llamaContext, encodedEmbeddings, ref nPast, seqId, nBatch); - /// /// Indicates whether the model supports vision inputs. /// diff --git a/LLama/Native/NativeApi.Mtmd.cs b/LLama/Native/NativeApi.Mtmd.cs index 685200bee..cdecfd9f2 100644 --- a/LLama/Native/NativeApi.Mtmd.cs +++ b/LLama/Native/NativeApi.Mtmd.cs @@ -244,7 +244,7 @@ internal struct mtmd_decoder_pos // tokenize ---------------------------------------------------------- /// - /// Native text structure consumed by . + /// Native text structure consumed by NativeApi.mtmd_tokenize /// internal unsafe struct mtmd_input_text_native {