countMemoryFileTokens method

Future<({List<MemoryFile> details, int neomageMdTokens})> countMemoryFileTokens(List<({String content, String path, String type})> memoryFiles)

Count memory file tokens.

Implementation

Future<({int neomageMdTokens, List<MemoryFile> details})>
countMemoryFileTokens(
  List<({String path, String type, String content})> memoryFiles,
) async {
  if (memoryFiles.isEmpty) {
    return (neomageMdTokens: 0, details: <MemoryFile>[]);
  }

  final results = await Future.wait(
    memoryFiles.map(
      (f) => _countTokensWithFallback([
        {'role': 'user', 'content': f.content},
      ], []),
    ),
  );

  int total = 0;
  final details = <MemoryFile>[];

  for (int i = 0; i < memoryFiles.length; i++) {
    final tokens = results[i] ?? 0;
    total += tokens;
    details.add(
      MemoryFile(
        path: memoryFiles[i].path,
        type: memoryFiles[i].type,
        tokens: tokens,
      ),
    );
  }

  return (neomageMdTokens: total, details: details);
}