tokenize method
dynamic
tokenize()
Implementation
tokenize() {
var match = regexGrouping.firstMatch(buffer.substring(currentPos));
while (match != null) {
if (match.groupCount == 0) {
throw Exception(
"Invalid token type found at ${buffer.substring(currentPos)}");
}
for (int i = 1; i <= match.groupCount; i++) {
var group = match.group(i);
if (group != null) {
var tokenType = delimiters.keys.toList()[i - 1];
if (tokenType == 'string') {
// range of characters inside the string supposed to be in range of 32 to 126
// https://tezos.gitlab.io/whitedoc/michelson.html#string
if (group.codeUnits.any((e) => e < 32 || e > 126)) {
throw Exception(
"Invalid string found at ${buffer.substring(currentPos)}");
}
}
tokens.add({
"type": tokenType,
"value": group,
"text": group,
"col": columnNumber,
"offset": currentPos,
"line": '1',
});
break;
}
}
currentPos += match.end;
columnNumber += match.end;
if (currentPos >= buffer.length) {
break;
}
match = regexGrouping.firstMatch(buffer.substring(currentPos));
if (match == null) {
throw Exception(
"Invalid token type found at ${buffer.substring(currentPos)}");
}
}
if (tokens.map((e) => e['value']).join('') != buffer) {
throw Exception("Invalid code");
}
}