getTokenCountForString method

dynamic getTokenCountForString(String inputString)

Implementation

getTokenCountForString(String inputString) async {
  /// A function that takes an input string and tokenizes it using the encodeString function of the SPAiSimpleBpeTokenizer class.
  /// It returns the number of tokens in the resulting SPTokenContainer object.
  /// If the operation is successful, it emits a BpeTokenizerLoaded event with the resulting token container.
  /// If an error occurs, it emits a BpeTokenizerError event with the error message and stack trace.
  /// The emit function is a part of the flutter_bloc library and is used to emit state changes in a Bloc or Cubit.
  /// The provided code seems to be a part of a Flutter application that uses flutter_bloc for state management.
  emit(BpeTokenizerLoading(state.mainBpeTokenizerState
      .copyWith(message: 'Tokenizing input string...', errorMessage: '')));
  try {
    /// The encodeString function of the SPAiSimpleBpeTokenizer class takes an input string and returns a SPTokenContainer object.
    SPTokenContainer tokenContainer =
        await SPAiSimpleBpeTokenizer().encodeString(inputString);
    emit(BpeTokenizerLoaded(state.mainBpeTokenizerState.copyWith(
        message: 'Successfully tokenized input string!',
        tokenContainer: tokenContainer)));
  } catch (error, stackTrace) {
    emit(BpeTokenizerError(
        state.mainBpeTokenizerState.copyWith(
            message: 'Error tokenizing input string',
            errorMessage: error.toString()),
        stackTrace: stackTrace.toString()));
  }
}