tokenize method

dynamic tokenize()

Implementation

tokenize() {
  var match = regexGrouping.firstMatch(buffer.substring(currentPos));

  while (match != null) {
    if (match.groupCount == 0) {
      throw Exception(
          "Invalid token type found at ${buffer.substring(currentPos)}");
    }
    for (int i = 1; i <= match.groupCount; i++) {
      var group = match.group(i);
      if (group != null) {
        var tokenType = delimiters.keys.toList()[i - 1];

        if (tokenType == 'string') {
          // range of characters inside the string supposed to be in range of 32 to 126
          // https://tezos.gitlab.io/whitedoc/michelson.html#string
          if (group.codeUnits.any((e) => e < 32 || e > 126)) {
            throw Exception(
                "Invalid string found at ${buffer.substring(currentPos)}");
          }
        }

        tokens.add({
          "type": tokenType,
          "value": group,
          "text": group,
          "col": columnNumber,
          "offset": currentPos,
          "line": '1',
        });
        break;
      }
    }
    currentPos += match.end;
    columnNumber += match.end;
    if (currentPos >= buffer.length) {
      break;
    }
    match = regexGrouping.firstMatch(buffer.substring(currentPos));
    if (match == null) {
      throw Exception(
          "Invalid token type found at ${buffer.substring(currentPos)}");
    }
  }

  if (tokens.map((e) => e['value']).join('') != buffer) {
    throw Exception("Invalid code");
  }
}