wordsDetect function

WordCountResult wordsDetect(String? text, [ WordCountConfig config = const WordCountConfig() ])

Detects and counts words in text, returning both count and word array.

This is the core function that performs word detection and counting for 85+ languages including CJK (Chinese, Japanese, Korean), European, South Asian, African, and Middle Eastern languages.

The function handles different writing systems appropriately:

• CJK languages: Character-level tokenization
• European languages: Space-separated word tokenization
• Mixed text: Proper handling of multilingual content

Returns emptyResult for null, empty, or whitespace-only text.

Example:

// Basic usage
WordCountResult result = wordsDetect('Hello World');
print('${result.count} words: ${result.words}'); // 2 words: [Hello, World]

// Chinese text
WordCountResult chinese = wordsDetect('你好世界');
print('${chinese.count} words: ${chinese.words}'); // 4 words: [你, 好, 世, 界]

// With configuration
const config = WordCountConfig(punctuationAsBreaker: true);
WordCountResult result2 = wordsDetect("don't", config);
print('${result2.count} words: ${result2.words}'); // 2 words: [don, t]

Implementation

WordCountResult wordsDetect(
  String? text, [
  WordCountConfig config = const WordCountConfig(),
]) {
  if (text == null || text.isEmpty) return emptyResult;
  if (text.trim().isEmpty) return emptyResult;

  // Initialize bitmap if not already done
  _initializeBitmap();

  // Pre-compute punctuation lookup for custom punctuation
  final customPunctuationSet = <int>{};
  final defaultPunctuationSet = <int>{};

  if (!config.disableDefaultPunctuation) {
    for (final punct in defaultPunctuation) {
      if (punct.isNotEmpty) {
        defaultPunctuationSet.add(punct.codeUnitAt(0));
      }
    }
  }

  for (final punct in config.punctuation) {
    if (punct.isNotEmpty) {
      customPunctuationSet.add(punct.codeUnitAt(0));
    }
  }

  // Check for ASCII-only fast path optimization
  bool isAsciiOnly = true;
  for (int i = 0; i < text.length && isAsciiOnly; i++) {
    if (text.codeUnitAt(i) > 127) {
      isAsciiOnly = false;
    }
  }

  if (isAsciiOnly &&
      config.disableDefaultPunctuation == false &&
      config.punctuation.isEmpty) {
    // Fast path for ASCII-only text with default configuration
    return _fastAsciiWordCount(text, config);
  }

  // Single-pass algorithm with state machine
  final words = <String>[];
  final wordBuffer = StringBuffer();
  int wordCount = 0;
  bool inWord = false;

  for (int i = 0; i < text.length; i++) {
    final charCode = text.codeUnitAt(i);
    bool isBoundary = false;
    bool isCjk = false;

    // Check if character is a boundary using bitmap
    if (charCode < _maxCodePoint) {
      isBoundary = _isBoundaryChar(charCode);
      isCjk = _isCjkChar(charCode);
    }

    // Check custom punctuation
    final isCustomPunctuation =
        customPunctuationSet.contains(charCode) ||
        defaultPunctuationSet.contains(charCode);

    if (isCustomPunctuation) {
      if (config.punctuationAsBreaker) {
        // Treat punctuation as word boundary
        if (inWord && wordBuffer.isNotEmpty) {
          words.add(wordBuffer.toString());
          wordBuffer.clear();
          wordCount++;
          inWord = false;
        }
      }
      // Skip punctuation character (either remove it or it acts as boundary)
      continue;
    }

    // Handle Unicode symbols (similar to original logic)
    if (charCode >= 0xFF00 && charCode <= 0xFFEF ||
        charCode >= 0x2000 && charCode <= 0x206F) {
      if (inWord && wordBuffer.isNotEmpty) {
        words.add(wordBuffer.toString());
        wordBuffer.clear();
        wordCount++;
        inWord = false;
      }
      continue;
    }

    // Handle CJK characters (each character is a word)
    if (isCjk) {
      // End current word if any
      if (inWord && wordBuffer.isNotEmpty) {
        words.add(wordBuffer.toString());
        wordBuffer.clear();
        wordCount++;
      }
      // Add CJK character as individual word
      words.add(String.fromCharCode(charCode));
      wordCount++;
      inWord = false;
      continue;
    }

    // Handle regular word boundaries (spaces, tabs, etc.)
    if (isBoundary) {
      if (inWord && wordBuffer.isNotEmpty) {
        words.add(wordBuffer.toString());
        wordBuffer.clear();
        wordCount++;
        inWord = false;
      }
      continue;
    }

    // For characters that are not CJK, not configured punctuation, and not basic boundaries,
    // we need to determine if they should be word characters or boundaries
    // This handles the case where only specific punctuation is configured
    if (!isCjk && !isCustomPunctuation && !isBoundary) {
      // Check if this character would be treated as punctuation in the original RegExp approach
      final isNonWordChar = _isNonWordCharacter(charCode);

      if (isNonWordChar && !_isLetterOrDigit(charCode)) {
        // This is a character that would split words in the original implementation
        if (inWord && wordBuffer.isNotEmpty) {
          words.add(wordBuffer.toString());
          wordBuffer.clear();
          wordCount++;
          inWord = false;
        }
        continue;
      }
    }

    // Regular character - add to current word
    wordBuffer.writeCharCode(charCode);
    inWord = true;
  }

  // Handle final word
  if (inWord && wordBuffer.isNotEmpty) {
    words.add(wordBuffer.toString());
    wordCount++;
  }

  return WordCountResult(words: words, count: wordCount);
}