parse method

List<int> parse(List<String> tokenizedWords, String inputTextbuffer)

Parses the tokenizedWords and looks for matches in this Dictionary. inputTextBuffer is supplied by op codes that receive

Specification Reference

Section 13.6 - 13.6.3

The parse table format is specified (buried) in op code 228 definition ("read" op code)

Implementation

List<int> parse(List<String> tokenizedWords, String inputTextbuffer) {
  log.fine("parse() Got line: $inputTextbuffer with tokens: $tokenizedWords");

  final parseTable = <int>[];

  // Total tokenized words is byte 1 of the parse table.
  parseTable.add(tokenizedWords.length);

  // Used to index into the inputTextBuffer during parsing
  // and then add to the parseTable
  int textBufferIndex = 0;

  for (final tokenizedWord in tokenizedWords) {
    var searchWord = tokenizedWord;

    if (searchWord.length > entryCharacterLimit) {
      searchWord = searchWord.substring(0, entryCharacterLimit);
      log.fine("parse() is truncating word $tokenizedWord to $searchWord.");
    }

    final wordMatchIndex = _entries.indexOf(searchWord);

    // final searchWord = _entries.reversed.firstWhere(
    //     (entry) => word == entry || (word.length > entry.length)
    //         ? word.startsWith(entry)
    //         : entry.startsWith(word),
    //     orElse: () => "");
    // final indexOfDictionaryWord =
    //     searchWord.isEmpty ? -1 : _entries.indexOf(searchWord);

    if (wordMatchIndex != -1) {
      final addr = _wordAddress(wordMatchIndex);
      log.fine(
          'parse() (found word: "$tokenizedWord ($searchWord)" in dictionary as "${_entries[wordMatchIndex]}"'
          ' at address 0x${addr.toRadixString(16)}) ${_entries.where((e) => e.startsWith(tokenizedWord[0])).toList()}');

      // byte address of the word in the dictionary
      parseTable.add((addr >> 8) & 0xff);

      parseTable.add(addr & 0xff);

      parseTable.add(tokenizedWord.length);
    } else {
      log.fine(
          'parse() (word: $tokenizedWord ($searchWord) not found in dictionary'
          ' ${_entries.where((e) => e.startsWith(tokenizedWord[0])).toList()})');
      log.fine(
          "parse() entryLength: $_entryLength, word length: ${searchWord.length}");
      //log.warning('(word: ${t} not found in dictionary ${entries})');

      // byte address of the word in the dictionary (0 if not found)
      parseTable.add(0);

      parseTable.add(0);

      // number of characters in the word
      parseTable.add(tokenizedWord.length);
    }

    //location in text buffer
    textBufferIndex = inputTextbuffer.indexOf(tokenizedWord, textBufferIndex);

    // Add the location to the parse tabel with the offset
    // base on the engine version (see [textBufferOffset] comments).
    parseTable.add(textBufferIndex + textBufferOffset);

    // Offset the buffer search position for the next go around.
    // Ensures that repeated words in the text buffer are not
    // given the same index.
    textBufferIndex += tokenizedWord.length;
  }

  return parseTable;
}