LlmInferenceOptions.gpu constructor
LlmInferenceOptions.gpu({})
{@macro LlmInferenceOptions}
Constructor for inference models using the GPU.
Implementation
factory LlmInferenceOptions.gpu({
required String modelPath,
required int sequenceBatchSize,
required int maxTokens,
required double temperature,
required int topK,
int decodeStepsPerSync = 3,
int? randomSeed,
}) =>
throw UnimplementedError();