completion method

Future<CactusCompletionResult> completion(CactusCompletionParams params, { List<String> mediaPaths = const [], })

Implementation

Future<CactusCompletionResult> completion(CactusCompletionParams params, {List<String> mediaPaths = const []}) async {
  _checkDisposed();
  final replyPort = ReceivePort();
  final completer = Completer<CactusCompletionResult>();

  final startTime = DateTime.now();
  DateTime? firstTokenTime;

  replyPort.listen((message) {
    if (message is String && params.onNewToken != null) {
      firstTokenTime ??= DateTime.now();
      params.onNewToken!(message);
    } else if (message is Map) {
      message['error'] != null ? completer.completeError(message['error']) : completer.complete(message['result']);
      replyPort.close();
    }
  });

  _isolateSendPort!.send(['completion', _SendableCompletionParams.fromOriginal(params), mediaPaths, replyPort.sendPort]);
  try{
    final result = await completer.future;
    final endTime = DateTime.now();
    final totalTime = endTime.difference(startTime).inMilliseconds;
    final tokPerSec = totalTime > 0 ? (result.tokensPredicted * 1000.0) / totalTime : null;
    final ttft = firstTokenTime?.difference(startTime).inMilliseconds;
    CactusTelemetry.track({
      'event': 'completion',
      'tok_per_sec': tokPerSec,
      'toks_generated': result.tokensPredicted,
      'ttft': ttft,
      'num_images': mediaPaths.length,
    }, initParams);
    return result;
  } catch (e) {
    CactusTelemetry.error(e, initParams);
    rethrow;
  }
}