From 4c715fef1f0a0da49fb266006debdc1b05f584a7 Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Thu, 15 Oct 2026 00:40:37 +0100 Subject: [PATCH] - Removed unused fields in `Conversation` - Removed unused state in `InteractiveExecutor` - Removed 2 unused methods in `MtmdWeights` --- Assets/icon.png | Bin 0 -> 6055 bytes LLama/Batched/Conversation.cs | 21 ++++-------- LLama/LLamaInteractExecutor.cs | 57 +++++++++++++++------------------ LLama/LLamaSharp.csproj | 2 +- LLama/MtmdWeights.cs | 6 ---- LLama/Native/NativeApi.Mtmd.cs | 2 +- 6 files changed, 33 insertions(+), 55 deletions(-) create mode 100644 Assets/icon.png diff --git a/Assets/icon.png b/Assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..8745d7e2a77f56eb9fcee7a91d14120a3242a1f4 GIT binary patch literal 6055 zcmV;Y7g*?tP)!UsKp zbn%&r`&|V#!OGH6syJ+fvrqh8#gG` zUrjmG5YV}lqLu?HC`biBivX}c&)9!UnnOeY_yFo7AU>+5F+d6Wh=M*!XfME>wFqL* zoYU^!t=OvM=~7K$ItfBHHf*_Gk6Oog)S(CPS8RJ6qggQS)o|?cpZxcvMs*y z@=*y8<0}*&5!EyojDQ3~AS3KA2n2$FB&R$5M_RhJe=FTanP6H7va$Yw+bAV3u=_Nz zoEEvEcq<{-R6wjOfmrH>=(ZzPXh+OtMa-Fd7Dp&x4NI^FBCw7`Vey3^_<|7n1|jtH zgX`!686DL&QwSw@?CRe3yPQu*JHLDnaF2KMB^=0G0ClsxWz}+YfBb5MOB@jGGgGFp zg+w?8M^X4*7o59aN)-D+_?0c4j~>tG*m)Iu*Vccfhy#9|AVxuD9jfeApf<~lB?d#( zWr45GgPz+jN96ymP854UUa|5g@@cz`@r=aNA@tqlW-g=mtF^JVR2HO(LyWs-8ByiAD@CHEW$n< zhIKFm!5@NU&<}nn2!3E#m6AArFfJ)YVrk7qcG>_>A!uO%sK*7l)B~y14Y{NcVv!56 zVizLCsri}vY!H?XX?SVT4@Ta<2)(wGM*y`K&vs}(2rvzE=-TVat+{e;_!-4C_*>tNjZm&wn8>aw_BPy?i!rmNYTPt}pQ$1)&!b|=HQ+6JODBukx(?k5>Pi1KEt1t=LP}dmq?CKyG50(#FeB zSTj#oDH`zO+`+w|G`Cv~cRX+_=G2#s=i3i;e!K zO;prwMemM#5h``)eye<=1Mcr71TFG*RhQ$MZ1z;LGAM2EKCljB#ISqHP*BsLt3wq9 zgU4HQP7J}q4YLpy)1G9hxyCANo8$sr+%vvAhL`(|MDAfihi84H!p(XtxnYZ)CTY7 z)uub1ILCAwFr9-c^g9WGpk%@2Ae_D%e|HNVv1sMOBcV+rFw*Imj*j=`?+f4+2 zx(8KzPUy;If#AZ%k0r9~-aL_sZjv@ZdcC2w0G6_}v#b)1V*PR{jRxWGKACRUrcr-I zt)73pnfk_Rmn+6UI^RvrKeTZD>+j(j(i=mE8;T*VUXjQyDe*R}O}Fo~2=ayUtpx-` zbyQFp`MgECI!pl^Jo&D=WaSGBR?bh#M?*1mzn7YSXqUOut(}l!85lOj7%HE8*>IDs z@2*v<;|1bwD*wRlSvp;(MG(mueu#C=^mJ$8ieyg8Arl$@iR)9KeVLGG#ETm#TeY9oskMP4ID zE@Pt6mWDS(X-T4ir}DcK{h7MtV+}9XDfW6+6a1 zx{hSRKe{sG9~!uR{Cn5~`fcU@74@K%+kX2uQy{Dz!X7vMfgU--P7Z3HWYK& zQl-dgdjzHpo&M32UAVP=^#3;T<&!D%1{ zpTDvWO6^=VO@X!d);*}VylRlrxX20e*FlL|uc&^ZD(-6G>o^8vppmm1T&y)~Pl1|+ zl{)@0&^?5*x{@i|Ex9FGJZEoIa43|r9fR+zKecIvY49%`KS{{!`xoZlS1U=vY1O6K;X>8&o0H7e46%R@XIuBG`bWB`m{*R!-*bQ3 z?WPmoy#2fAd0#KyYC$+2J^%bgor#`PC>l3E2h!WmvLblW*wVhOd$N9}T(V@@^3_B< zhP$>=M+}DHj^h~-vQo4?%d5aAN zbM?FXtP=5x~|8Tx9C@gW|BkON4RriwXn^1S@MTK`{Ab^)s?DR2Q`~R;-MZ?8y-oCbe95L8v znhblM+Xr9&i0S&USkU?8msJn?iLi!6G~D|=AgbyteZvLLhUbrN)hRHBQr0LZtYO4p z>4J({L#G8_&&jNa!4Mb<;aA&!Z>sJC8a?9uy>0lzldn$MhVi^x)?J@e_U11R;EyfF zo+LSeD57ldarFML%aV#?4hy2UUIW{2)j3xc1;Td#__?;Z;gUrO8Goe=wenaLq5kyE z)|tX$6V$!%i7;arte95@cUeK&?XrdQ*RLAdxCU>$bUe)w#cJk}DMa_6@`G;B6}t zNy5p}dhPE?1o=$G{VsrfTKz$=z*Vld8ixk@AVsG)bf9T@@{KiXhiY1$Y|lG64a-#m z!l`b!d-cM=z>;dv0;ig$AZx$W_@$&hKFGy51!vMQ=@Cza*OplBAwue_@(Vtw3$p@5#^qpTl2@DL zEeg&M-X#Y6=Ozq;+T}`&nDf8NC5rZ+!hEGdO;e6Q65`K);6yJH>LZ1cWc5sXLqy{I zB@+f%I@o^(RMsC8V#rtW2(7u@e?EZzj?`PkP1^mBgWWmV|40Y!xn4EqQH_HPOK=8a zaQMQAwvYR&`H1NH>BKfk1bMD=YlH$n!dS>z(7>ebD5-#8cjom%CZb8Re{Oc!(eAsS z+&{g`6>qh{QC6WV51u~*MT$S@6l8Zx-`1p43UpGIz@a5^u1BC(&o7v=RWP?Mx3ixF zhK_9}maB4aDT*tL(r%YaF#XMFt0K^M@t*RE+@k$m*W$^J?I$ScpP_C%`4JuQ={j{+ zH-Ix>__>Dan4ooRdn>hTs+NC9qKwXWv)n&V1BBxY6>4@rV}bGe`TEYD9q(!D7#kM} zw=0nh7)Vu|#R^x2-ujzsXqTP9U=aQ7sriSN?{psSQJvzZ3v319t#)1c#P}FO$u6U| zhPEu9%kit?;3!jRt3v%fC~jD&8^BZBgy7k(w0w8nWd&HZ<|@EK55l zYd!<=HmymK|w{WdJnW<_ln)KILMEf(w(X=Q@*{FSA%raBN+T$CsZFKEsEwX%9-|V z6#tP~{9yj~KM2PgssvO~;Jf&Ch&&~xPBk?}ke2ps-2`}srKM;j$sUt{V6($rs~7pw zMON`28Z%3_@gHX3scTeE%+rFY3W?ARO7YB--QPAf8PoA0L>8GMqrQJk!BIrL3qG5i zU@*CKP`{)`AOB(EN!RUvM5VmjuNWbW&UH~FapKz%rgCcDk$`D}>^$>CFCk=y!GPmX zUtfc`!*rmvLovqOeI)h9lD1v@!7i03`9w*>LU6oZE}HnjQjo`=?%i&BXDqE4!_lz3 z9e^rBGR;#AhjewE71eMQm!;Ef5;6(wkH>#x9)F$*>_w$;8o1X=qrr(<`B}@TV_Q1? zrbUq5{X0U0;1P_8bi=1~)?-ycQjdc*2b7TB*9lbODg zb^PEInCvK#$j|^rFN`b5hpW3>&!yXUCMP*4gd<-`>?#hn{l!oW_S7vfJmPOEELfQ= z?urp}f&E(k!HB`y<^%DE%{T&i@Z`9c3A;De#lI61tnErEqAXHaZTr zWR+cVrD06TWY7g>ns7`Y96_TtelTCl6(Oqm$nYqUuC|Fh^~Wt;+y5(*Au>J`dX&h;2;?D7 zP%;h+%a|ZPJoV@R0k&X_!K<&Ufi0`3{t{SS`t??e+Qn5m#(d(<%=XWj$eU|eTyA(K zvF2WAJ3Q{|On}WhPw&t(R+Gr;M7dpE+n?Rw-TWy4t0TTajP$m{U6cGqu-ChE;j)xR z{5d@iw7&S4@jRog8O2VKm85oqXNGgWf6k<;a<=>$ELH59*|o>(;k8|z+n>#9tZW+N z;vtv^nP|=?T(#^bSPb5Xk?vC%YCoFoa32Ut8ZXs7Xc$|ndJnvs5NaYI{L*Z{FOs!2 z@SuO+g=Iz8MgV-2J=U?HPxZRcGIOh^Sp7{J)hm-T8IF=UC~3M(uX9j}{v&TgipH(E z1Y}D~*SE9&0#8oP?XBu^J^>0fEGarNY5yC5c^2sXWY%uEGubD->!Wrhl%zvm>P zf#F23nFyu}v3z%0M%F)bhj<%^aXhVwB( z_&Ub7HbO+EYoEvSIoWv0nfu{E{~Nw#Mc0i0xSi=m;i2>DcCR*iSFk(5aXiAqS)DhW zcQC%;nxl`8Q6v%l@4f*=N*E8Zsio(!KjdoMoce>d`u81JR($PJ0L_Xlsfv=(McF(h zkRoB#wQ?q+prQuF^C$nzM${kQN~V1&I+Z?R~Xg_^mR@=Wao@k7S`qZgOk(p$)mYTuQT*lN`UDn zQg~D~!$`3Rcbm#W z&4iV6hhI-9fD%XPO*^|E+n3&COiVY|9VRgG%brYHsakfU8Z(hCR1Zo@<1rTMvq>RK zaeEiL;fYJ)EEZLvGYZQ~7g98Wa|i!C9$QHXzV=k-_D8es*Z_I^{99F*^HB<@UuBe| zVVr+&e@;TFIbKkob|xIKfe=Hp@bE4}U9fVaR!n zx;gB7E3&NkhL-`{&4eo^Rk0^db+jWKIALaf?Nb(G5QBczb7aDHte@w|ExS6l=k@5k zX%TPM*KfI%Q2L_w0bt9I#mSbL0$7{{DuZQ=mPtE|-xvkGl=31o6n>+vj002ovPDHLkV1kzk+4cYc literal 0 HcmV?d00001 diff --git a/LLama/Batched/Conversation.cs b/LLama/Batched/Conversation.cs index 01e8934a9..15a698618 100644 --- a/LLama/Batched/Conversation.cs +++ b/LLama/Batched/Conversation.cs @@ -23,9 +23,6 @@ public sealed class Conversation /// private bool _forked; private MtmdChunkSequence? _pendingMtmdSequence; - private readonly List _embed_inps = new(); - private readonly List _session_tokens = new(); - private int _consumedTokensCount; /// /// Stores the indices to sample from. Contains valid items. @@ -251,6 +248,12 @@ private void AssertCanBePrompted() throw new AlreadyPromptedConversationException(); } + /// + /// Add text to this conversation + /// + /// Text to tokenize and append + /// Whether to add a bos to the text. + /// Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. public void Prompt(string promptText, bool addBos = true, bool special = true) { var tokens = Executor.Context.Tokenize(promptText, addBos, special); @@ -498,18 +501,6 @@ internal void OnMtmdEvaluationCompleted(int newPast, MtmdChunkSequence sequence) _requiredEpoch = Executor.Epoch + 1; _forked = false; - if (sequence.TextTokens.Count > 0) - { - _embed_inps.AddRange(sequence.TextTokens); - _session_tokens.AddRange(sequence.TextTokens); - } - - var fillerToken = GetFillerToken(Executor.GetMtmdMarker()); - var fillerCount = Math.Max(0, sequence.TotalPositions - sequence.TotalTokens); - for (var i = 0; i < fillerCount; i++) - _embed_inps.Add(fillerToken); - - _consumedTokensCount = _embed_inps.Count; sequence.Dispose(); } diff --git a/LLama/LLamaInteractExecutor.cs b/LLama/LLamaInteractExecutor.cs index 104758dbd..3afb137fb 100644 --- a/LLama/LLamaInteractExecutor.cs +++ b/LLama/LLamaInteractExecutor.cs @@ -21,12 +21,7 @@ namespace LLama public class InteractiveExecutor : StatefulExecutorBase { // Indicates whether the executor is currently evaluating the initial prompt or a follow-up turn. - private bool _is_prompt_run = true; - - // MTMD multimodal state - private SafeMtmdInputChunks? _mtmdChunks; // Pending chunk collection produced by the multimodal tokenizer. - private string? _mtmdMarker; // Cached multimodal marker returned by the native helper. - + private bool _isPromptRun = true; /// /// Create an interactive executor for text-only inference. @@ -55,15 +50,15 @@ public override ExecutorBaseState GetStateData() InteractiveExecutorState state = new() { ConsumedSessionCount = _n_session_consumed, - EmbedInps = _embed_inps.ToArray(), - IsPromptRun = _is_prompt_run, + EmbedInps = [ .. _embed_inps ], + IsPromptRun = _isPromptRun, ConsumedTokensCount = _consumedTokensCount, - Embeds = _embeds.ToArray(), - LastTokens = _last_n_tokens.ToArray(), + Embeds = [ .. _embeds ], + LastTokens = [ .. _last_n_tokens ], MatchingSessionTokensCount = _n_matching_session_tokens, PastTokensCount = _pastTokensCount, SessionFilePath = _pathSession, - SessionTokens = _session_tokens.ToArray(), + SessionTokens = [ .. _session_tokens ], LastTokensCapacity = _last_n_tokens.Capacity, }; return state; @@ -76,15 +71,15 @@ public override Task LoadState(ExecutorBaseState data, CancellationToken cancell if (data is InteractiveExecutorState state) { _n_session_consumed = state.ConsumedSessionCount; - _embed_inps = state.EmbedInps!.ToList(); - _is_prompt_run = state.IsPromptRun; + _embed_inps = [ .. state.EmbedInps ]; + _isPromptRun = state.IsPromptRun; _consumedTokensCount = state.ConsumedTokensCount; - _embeds = state.Embeds!.ToList(); - _last_n_tokens = new FixedSizeQueue(state.LastTokensCapacity, state.LastTokens!); + _embeds = [ .. state.Embeds ]; + _last_n_tokens = new FixedSizeQueue(state.LastTokensCapacity, state.LastTokens); _n_matching_session_tokens = state.MatchingSessionTokensCount; _pastTokensCount = state.PastTokensCount; _pathSession = state.SessionFilePath; - _session_tokens = state.SessionTokens!.ToList(); + _session_tokens = [ .. state.SessionTokens ]; } else throw new ArgumentException("Invalid state data type."); @@ -119,7 +114,7 @@ public override async Task LoadState(string filename, CancellationToken cancella /// true to keep generating; otherwise false. protected override Task GetLoopCondition(InferStateArgs args, CancellationToken cancellationToken = default) { - return Task.FromResult(args.RemainedTokens != 0 && !args.WaitForInput || _is_prompt_run); + return Task.FromResult(args.RemainedTokens != 0 && !args.WaitForInput || _isPromptRun); } /// @@ -130,7 +125,7 @@ protected override Task GetLoopCondition(InferStateArgs args, Cancellation /// protected override Task PreprocessInputs(string? text, InferStateArgs args, CancellationToken cancellationToken = default) { - if (_is_prompt_run) + if (_isPromptRun) { // When running the first input (prompt) in interactive mode, we should specially process it. if (text == null) @@ -140,7 +135,7 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc if (!IsMultiModal) { - _embed_inps = Context.Tokenize(text, true, true).ToList(); + _embed_inps = [ .. Context.Tokenize(text, true, true) ]; } else { @@ -159,9 +154,9 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc if (!IsMultiModal) { - var line_inp = Context.Tokenize(text, false, true); - _embed_inps.AddRange(line_inp); - args.RemainedTokens -= line_inp.Length; + var lineInput = Context.Tokenize(text, false, true); + _embed_inps.AddRange(lineInput); + args.RemainedTokens -= lineInput.Length; } else { @@ -180,7 +175,7 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc /// Mutable inference state. /// /// Tuple describing whether to stop and any additional outputs to emit. - protected override Task<(bool, IReadOnlyList)> PostProcess(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken) + protected override Task<(bool, IReadOnlyList)> PostProcess(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken = default) { if (_embed_inps.Count <= _consumedTokensCount) { @@ -211,13 +206,13 @@ protected override Task PreprocessInputs(string? text, InferStateArgs args, Canc } /// - protected override async Task InferInternal(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken) + protected override async Task InferInternal(IInferenceParams inferenceParams, InferStateArgs args, CancellationToken cancellationToken = default) { var batch = new LLamaBatch(); if (_embeds.Count > 0) { - _is_prompt_run = false; + _isPromptRun = false; if (_pastTokensCount + _embeds.Count > Context.ContextSize) { // number of tokens to keep when resetting context @@ -248,7 +243,7 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In } else { - var result = await Context.DecodeAsync(_embeds, LLamaSeqId.Zero, batch, _pastTokensCount); + var result = await Context.DecodeAsync(_embeds, LLamaSeqId.Zero, batch, _pastTokensCount, cancellationToken); _pastTokensCount = result.Item3; if (result.Item1 != DecodeResult.Ok) throw new LLamaDecodeError(result.Item1); @@ -262,7 +257,7 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In } else if (IsMultiModal && MtmdChunks is not null) { - _is_prompt_run = false; + _isPromptRun = false; var nPast = _pastTokensCount; var previousConsumed = _consumedTokensCount; EvaluateMtmdChunks(ref nPast, previousConsumed, nameof(InteractiveExecutor)); @@ -297,8 +292,8 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In id = Context.NativeHandle.ModelHandle.Vocab.Newline!.Value; if (args.Antiprompts is not null && args.Antiprompts.Count > 0) { - var first_antiprompt = Context.Tokenize(args.Antiprompts[0], false, true); - _embed_inps.AddRange(first_antiprompt); + var firstAntiprompt = Context.Tokenize(args.Antiprompts[0], false, true); + _embed_inps.AddRange(firstAntiprompt); } } @@ -320,15 +315,13 @@ protected override async Task InferInternal(IInferenceParams inferenceParams, In } } } - - return; } /// /// Serializable state specific to the interactive executor. /// public class InteractiveExecutorState - : StatefulExecutorBase.ExecutorBaseState + : ExecutorBaseState { /// /// Whether the executor is running for the first time (running the prompt). diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj index 9acd98bfa..12d42ee44 100644 --- a/LLama/LLamaSharp.csproj +++ b/LLama/LLamaSharp.csproj @@ -14,7 +14,7 @@ MIT, SciSharp STACK $([System.DateTime]::UtcNow.ToString(yyyy)) https://github.com/SciSharp/LLamaSharp git - https://avatars3.githubusercontent.com/u/44989469?s=200&v=4 + https://avatars3.githubusercontent.com/u/44989469 LLama, LLM, GPT, ChatGPT, NLP, AI, Chat Bot, SciSharp LLamaSharp is a cross-platform library to run 🦙LLaMA/Mtmd model (and others) in your local device. diff --git a/LLama/MtmdWeights.cs b/LLama/MtmdWeights.cs index 32dd6b572..b25844a13 100644 --- a/LLama/MtmdWeights.cs +++ b/LLama/MtmdWeights.cs @@ -110,12 +110,6 @@ public int Tokenize(string text, bool addSpecial, bool parseSpecial, ReadOnlySpa public int EvaluateChunks(SafeMtmdInputChunks chunks, SafeLLamaContextHandle llamaContext, ref int nPast, int seqId, int nBatch, bool logitsLast) => NativeHandle.EvaluateChunks(chunks, llamaContext, ref nPast, seqId, nBatch, logitsLast); - public int EvaluateChunk(IntPtr chunkPtr, SafeLLamaContextHandle llamaContext, ref int nPast, int seqId, int nBatch, bool logitsLast) - => NativeHandle.EvaluateChunk(chunkPtr, llamaContext, ref nPast, seqId, nBatch, logitsLast); - - public int DecodeImageChunk(IntPtr chunkPtr, SafeLLamaContextHandle llamaContext, IntPtr encodedEmbeddings, ref int nPast, int seqId, int nBatch) - => NativeHandle.DecodeImageChunk(chunkPtr, llamaContext, encodedEmbeddings, ref nPast, seqId, nBatch); - /// /// Indicates whether the model supports vision inputs. /// diff --git a/LLama/Native/NativeApi.Mtmd.cs b/LLama/Native/NativeApi.Mtmd.cs index 685200bee..cdecfd9f2 100644 --- a/LLama/Native/NativeApi.Mtmd.cs +++ b/LLama/Native/NativeApi.Mtmd.cs @@ -244,7 +244,7 @@ internal struct mtmd_decoder_pos // tokenize ---------------------------------------------------------- /// - /// Native text structure consumed by . + /// Native text structure consumed by NativeApi.mtmd_tokenize /// internal unsafe struct mtmd_input_text_native {