generate method

List<int> generate(
  1. List<int> prompt,
  2. int maxNewTokens,
  3. Sampler sampler, {
  4. void onToken(
    1. int token,
    2. int step
    )?,
})

Generate token ids using sampler and optional callback per step.

Implementation

List<int> generate(List<int> prompt, int maxNewTokens, Sampler sampler,
    {void Function(int token, int step)? onToken}) {
  final ids = List<int>.from(prompt);

  // Prefill with prompt (build KV cache)
  if (ids.isNotEmpty) {
    _prefill(ids);
  }

  for (var step = 0; step < maxNewTokens; step++) {
    final next = _decodeStep(ids);
    final chosen = sampler.sampleNext(next, ids);
    ids.add(chosen);
    onToken?.call(chosen, step);
    // Update KV with last token context already handled in _decodeStep
  }
  return ids;
}