LLama.Batched.CannotSampleRequiresInferenceException: 'Must call Infer() before sampling from this Conversation'

[ikuzono]

why around 570 tokens promp it throws LLama.Batched.CannotSampleRequiresInferenceException: 'Must call Infer() before sampling from this Conversation'

[SignalRT]

@ikuzono
Are you running an example?
If not. Can you provide an example of what are you trying to do?

[ikuzono]

@SignalRT
Model : L3 8B Stheno v3.2 Imat
https://github.com/ikuzono/LLama-sharp-Group-Convo

[ikuzono]

Peramerter:
`using LLama;
using LLama.Batched;
using LLama.Common;
using LLama.Native;
using LLama.Sampling;

public static class Peramerters
{

public static string ModelImagePath = ""; // change it to your own model path.
public static string MMprojPath = ""; // change it to your own model path.
public static string ModelPath = Path.Combine(Environment.CurrentDirectory, "Model.gguf"); // change it to your own model path.

public static ModelParams parameters = new ModelParams(ModelPath)
{
    MainGpu = 2,
    ContextSize = 8190, // The longest length of chat as memory.
    GpuLayerCount = 20 // How many layers to offload to GPU. Please adjust it according to your GPU memory.
};

public static LLamaWeights LLavaModal = null;
public static LLavaWeights MMproj = null;
public static LLamaWeights Model = null;

public static BatchedExecutor Executor = null;
public static InteractiveExecutor ImageExecutor = null;

// we'll need this for evaluating if we are at the end of generation
public static SafeLlamaModelHandle.Vocabulary vocab = null;

public static DefaultSamplingPipeline customPipeline = new DefaultSamplingPipeline()
{
    Temperature = 0.675f,        // Higher = more creative  
    TopK = 40,                 // Consider top 50 tokens  
    TopP = 0.95f,              // Nucleus sampling threshold  
    RepeatPenalty = 1.1f,      // Penalize repetition  
    FrequencyPenalty = 0.1f,   // OpenAI-style frequency penalty  
    PresencePenalty = 0.1f,    // OpenAI-style presence penalty
    
    Seed = (uint)new Random().NextInt64(),
};
public static BaseSamplingPipeline Sampler = customPipeline;
public static int MaxToken = 120;
public static List<Conversation> Memory = new List<Conversation>();
//public static Conversation MainConversation = Peramerters.Executor.Create();

// It isn't the end of generation, so add this token to the decoder and then add that to our tracked data
public static StreamingTokenDecoder Decoder = null;

public static Conversation LoadedConversation = null;
public static Dictionary<string, string> Characters = new Dictionary<string, string>()
{
    { "Lelouch" , "You are Lelouch VI britannia From Code geass" },
    { "Koishi" , "You are Koishi komeiji from Touhou" },
    { "Monika" , "You are Monika from DDLC" },
};

public static HistoryCharacter[] CharacterArray = new HistoryCharacter[0];











public async static Task Initalize()
{
    //ll model
    if (Peramerters.ModelPath != "")
    {
        Peramerters.Model = LLamaWeights.LoadFromFile(parameters);
    }

    //LLav
    if (Peramerters.ModelImagePath != "")
    {
        Peramerters.LLavaModal = LLamaWeights.LoadFromFile(parameters);
    }

    //MMro
    if (Peramerters.MMprojPath != "")
    {
        Peramerters.MMproj = await LLavaWeights.LoadFromFileAsync(Peramerters.MMprojPath);
    }
    // LLava                                                                                              
    if (Peramerters.MMproj != null)
    {
        LLamaWeights NewModelWeight = null;
        if (LLavaModal != null)
        { NewModelWeight = Peramerters.LLavaModal; }

        if (Model != null)
        { NewModelWeight = Peramerters.Model; }

        if (NewModelWeight == null)
        { return; }

        // Executor
        var Context = Peramerters.Model.CreateContext(Peramerters.parameters);
        Peramerters.ImageExecutor = new InteractiveExecutor(Context, Peramerters.MMproj);
    }

    if (Peramerters.Model != null)
    {
        Peramerters.Executor = new BatchedExecutor(Peramerters.Model,Peramerters.parameters);
    }
    else
    {
        if (ImageExecutor != null)
        {
            Peramerters.Executor = new BatchedExecutor(Peramerters.LLavaModal,Peramerters.parameters);
        }
    }
    Peramerters.vocab = Executor.Context.NativeHandle.ModelHandle.Vocab;
    Peramerters.Decoder = new StreamingTokenDecoder(Executor.Context);
}

}`

ask:
`using LLama;
using LLama.Batched;
using LLama.Common;
using LLama.Native;
using LLama.Sampling;
using System;
using System.CodeDom.Compiler;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

//////////////////////////////////////////////////////////////////////////////////////////////
public class Methods
{

//////////////
async public Task<string> Text(HistoryCharacter[] History, string Character, string Description)
{

    BatchedExecutor CurrentExecutor;
    if (Peramerters.Executor != null)
    {
        CurrentExecutor = Peramerters.Executor;
    }
    else
    {
        throw new Exception("No Executor");
    }

    var Done = false;
    Peramerters.Executor.Context.NativeHandle.MemorySequenceRemove(LLamaSeqId.Zero, -1, -1);
    Peramerters.Executor.Context.NativeHandle.MemoryClear(true);

    var NewConversation = Peramerters.Executor.Create();
    var Texts = "";

    var template = new LLamaTemplate(Peramerters.Model);
    template.Add(Character, Description);
    template.AddAssistant = true;


    //Load History
    var MaxUse = 450;// (int)((Peramerters.parameters.ContextSize / 15) - (Peramerters.MaxToken + Peramerters.Executor.Context.Tokenize(Encoding.UTF8.GetString(template.Apply()), false, true).Length + 3 ));
    Console.WriteLine(MaxUse);
    if (MaxUse > 0)
    {
        var RHistory = History;
        RHistory.Reverse();
        
        var End = true;

        foreach (var i in RHistory)
        {
            var Text = i.Text; 
            var Used = Peramerters.Executor.Context.Tokenize(Encoding.UTF8.GetString(template.Apply()), false, true).Length;
            var NameLength = Peramerters.Executor.Context.Tokenize(i.AuthorRole, false, true).Length;
            var TextLength = Peramerters.Executor.Context.Tokenize(Text, false, true).Length;
            var MaxUseable = Used + NameLength + TextLength;
            Console.WriteLine("Strin size" + MaxUseable);


            if (End == true)
            {
                if (NameLength + Used + 3 > MaxUse)
                {  
                    End = false;
                    break; 
                }

                else
                {
                    if (MaxUseable > MaxUse)
                    {
                        var TokenArray = Peramerters.Executor.Context.Tokenize(Text, false, true);
                        var Size = TokenArray.Length;
                        var Take = Used + TextLength + NameLength;
                        int Amount = Take - MaxUse;
                        var NewSize = Size - Amount;

                        if (NewSize > 0)
                        {
                            End = false;
                            break;
                        }

                        Array.Resize<LLamaToken>(ref TokenArray, NewSize);
                         
                        var NewTokenArray = Peramerters.Executor.Context.DeTokenize(TokenArray);

                        template.Add(i.AuthorRole, NewTokenArray);
                        End = false;
                        break;
                    }
                    
                    else
                    {
                        template.Add(i.AuthorRole, i.Text);
                    }


                }
            }
        }

    }



    var TemplatedMessage = Encoding.UTF8.GetString(template.Apply());

    // create a new conversation and prompt it. include special and bos because we are using the template
    // - BOS is the "Beginning of Sequence" token and should be included at the start of any prompt
    // - Special tokens are special non-text tokens which an LLM is trained to understand (e.g. BOS). The templated text may contains special tokens.
    // Prompt both conversations
    NewConversation.Prompt(Peramerters.Executor.Context.Tokenize(TemplatedMessage, true, true));

    for (var i = 0; i < Peramerters.MaxToken; i++)
    {

        // Run inference on all pending tokens
        var Result = await Peramerters.Executor.Infer();


        if (Done != true)
        {
            if (Result == DecodeResult.NoKvSlot)
            {
                throw new Exception("Dead");
                //NewConversation.Dispose();
                //Texts = await Text(Message, Prompt);
                //break;
            }
            else
            {
                // If the conversation wasn't prompted before the last call to Infer then it won't need sampling.
                //if (!NewConversation.RequiresSampling)

                //    // Sample results from each conversation
                //    //await Peramerters.Executor.Infer();
                var token = NewConversation.Sample(Peramerters.Sampler);

                // A generic error, this is fatal and the batch can no longer be used. This should never occur and generally indicates
                // a bug in LLamaSharp, llama.cpp or a hardware error.
                if (Result != DecodeResult.Ok)
                    throw new Exception($"Error occurred while inferring: {Result}");

                // Some special tokens indicate that this sequence has ended. Check if that's what has been chosen by the sampling pipeline.
                if (token.IsEndOfGeneration(Peramerters.vocab))
                {
                    //MainConversation.Dispose();
                    //Peramerters.Memory.Add(MainConversation);
                    Done = true;
                    break;
                }
                else
                {
                    Peramerters.Decoder.Add(token);

                    NewConversation.Prompt(token);
                    Texts = Texts + Peramerters.Decoder.Read().ReplaceLineEndings(" ");
                }
            }

        }
    }

    NewConversation.Dispose();
    return Texts;
}


public async Task<LLamaToken[]> range(LLamaToken[] Inpua,int Begain,int End)
{
    var Clone = Inpua;
    var c2 = Clone.ToList();
    c2.RemoveRange(Begain, End);
    return c2.ToArray();
}

}

public class HistoryCharacter
{
//public byte[] Image { get; set; }
public string Text { get; set; }
public string AuthorRole { get; set; }

}
`

Aio:
`using LLama;
using LLama.Batched;
using System;
using System.CodeDom.Compiler;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using static LLama.Common.ChatHistory;

await Peramerters.Initalize();
string Character = "";
var GetMethods = new Methods();
for (int i = 0; i < 8; i++)
{
var NChI = new HistoryCharacter();
NChI.AuthorRole = "User";
NChI.Text = "What are you doing";

var CharacterArray= Peramerters.CharacterArray.ToList();
CharacterArray.Add(NChI);
Peramerters.CharacterArray = CharacterArray.ToArray(); 

var Pool = new List<string>();
foreach (var i2 in Peramerters.Characters.Keys.ToArray())
{
    string Name = i2.ToString();
    if (Name != FormerCharacter.Responder)
    {
        Pool.Add(Name);
    }
}
;
//string Peramerters.executor.Context.Tokenize(Message, addBos: true, special: true)
///////////-
var Rand = new Random();
var RandNumb = Rand.Next(0 - Pool.Count * -1);
Character = Pool[RandNumb];
Character = Character.ToString();
//////////

// apply the model's prompt template to our question and system prompt 
var Response = await GetMethods.Text(Peramerters.CharacterArray, Character, Peramerters.Characters[Character]); 
FormerCharacter.Responder = Character;
var Responder = Character;

Console.WriteLine(Character + ": \n" + Response + "\n");
//

var NewCharacter = new HistoryCharacter();
NewCharacter.AuthorRole = Responder;
NewCharacter.Text = Response;

var CharacterArray2 = Peramerters.CharacterArray.ToList();
CharacterArray2.Add(NewCharacter);
Peramerters.CharacterArray = CharacterArray2.ToArray();                                                                                                                                                                                                                                         

}

Console.WriteLine(Peramerters.Executor.Context.ContextSize);

public static class FormerCharacter
{
public static string Responder = "";

}
`