dequeue method

Token dequeue()

Implementation

Token dequeue() {
  if (pending.isNotEmpty) return pending.removeFirst();

  int oldPos = position;
  skipWhitespaceAndComment();

  if (atEnd) return Token.eol;

  Token result = Token();
  result.afterSpace = (position > oldPos);
  int startPos = position;
  String c = input[position++];

  // Handle two-character operators first.
  if (!atEnd) {
    String c2 = input[position];
    if (c2 == '=') {
      if (c == '=') {
        result.type = TokenType.opEqual;
      } else if (c == '+') {
        result.type = TokenType.opAssignPlus;
      } else if (c == '-') {
        result.type = TokenType.opAssignMinus;
      } else if (c == '*') {
        result.type = TokenType.opAssignTimes;
      } else if (c == '/') {
        result.type = TokenType.opAssignDivide;
      } else if (c == '%') {
        result.type = TokenType.opAssignMod;
      } else if (c == '^') {
        result.type = TokenType.opAssignPower;
      }
    }
    if (c == '!' && c2 == '=') result.type = TokenType.opNotEqual;
    if (c == '>' && c2 == '=') result.type = TokenType.opGreatEqual;
    if (c == '<' && c2 == '=') result.type = TokenType.opLessEqual;

    if (result.type != TokenType.unknown) {
      position++;
      return result;
    }
  }

  // Handle one-char operators next.
  if (c == '+') {
    result.type = TokenType.opPlus;
  } else if (c == '-') {
    result.type = TokenType.opMinus;
  } else if (c == '*') {
    result.type = TokenType.opTimes;
  } else if (c == '/') {
    result.type = TokenType.opDivide;
  } else if (c == '%') {
    result.type = TokenType.opMod;
  } else if (c == '^') {
    result.type = TokenType.opPower;
  } else if (c == '(') {
    result.type = TokenType.lParen;
  } else if (c == ')') {
    result.type = TokenType.rParen;
  } else if (c == '[') {
    result.type = TokenType.lSquare;
  } else if (c == ']') {
    result.type = TokenType.rSquare;
  } else if (c == '{') {
    result.type = TokenType.lCurly;
  } else if (c == '}') {
    result.type = TokenType.rCurly;
  } else if (c == ',') {
    result.type = TokenType.comma;
  } else if (c == ':') {
    result.type = TokenType.colon;
  } else if (c == '=') {
    result.type = TokenType.opAssign;
  } else if (c == '<') {
    result.type = TokenType.opLesser;
  } else if (c == '>') {
    result.type = TokenType.opGreater;
  } else if (c == '@') {
    result.type = TokenType.addressOf;
  } else if (c == ';' || c == '\n') {
    result.type = TokenType.eol;
    result.text = c == ';' ? ";" : "\n";
    if (c != ';') lineNum++;
  }
  if (c == '\r') {
    // Careful; DOS may use \r\n, so we need to check for that too.
    result.type = TokenType.eol;
    if (position < inputLength && input[position] == '\n') {
      position++;
      result.text = "\r\n";
    } else {
      result.text = "\r";
    }
    lineNum++;
  }
  if (result.type != TokenType.unknown) return result;

  // Then, handle more extended tokens.

  if (c == '.') {
    // A token that starts with a dot is just Type.Dot, UNLESS
    // it is followed by a number, in which case it's a decimal number.
    if (position >= inputLength || !isNumeric(input[position])) {
      result.type = TokenType.dot;
      return result;
    }
  }

  if (c == '.' || isNumeric(c)) {
    result.type = TokenType.number;
    while (position < inputLength) {
      String lastc = c;
      c = input[position];
      if (isNumeric(c) ||
          c == '.' ||
          c == 'E' ||
          c == 'e' ||
          ((c == '-' || c == '+') && (lastc == 'E' || lastc == 'e'))) {
        position++;
      } else {
        break;
      }
    }
  } else if (isIdentifier(c)) {
    while (position < inputLength) {
      if (isIdentifier(input[position])) {
        position++;
      } else {
        break;
      }
    }
    result.text = input.substring(startPos, position);
    result.type = (Keywords.isKeyword(result.text!)
        ? TokenType.keyword
        : TokenType.identifier);
    if (result.text == "end") {
      // As a special case: when we see "end", grab the next keyword (after whitespace)
      // too, and conjoin it, so our token is "end if", "end function", etc.
      Token nextWord = dequeue();
      if (nextWord.type == TokenType.keyword) {
        result.text = "${result.text!} ${nextWord.text ?? ""}";
      } else {
        // Oops, didn't find another keyword.  User error.
        throw LexerException(
          "'end' without following keyword ('if', 'function', etc.)",
        );
      }
    } else if (result.text == "else") {
      // And similarly, conjoin an "if" after "else" (to make "else if").
      // (Note we can't use Peek or Dequeue/Enqueue for these, because we are probably
      // inside a Peek call already, and that would end up swapping the order of these tokens.)
      var p = position;
      while (p < inputLength && (input[p] == ' ' || input[p] == '\t')) {
        p++;
      }
      if (p + 1 < inputLength &&
          input.substring(p, p + 2) == "if" &&
          (p + 2 >= inputLength || !isIdentifier(input[p + 2]))) {
        result.text = "else if";
        position = p + 2;
      }
    }
    return result;
  } else if (c == '"') {
    // Lex a string... to the closing ", but skipping (and singling) a doubled double quote ("")
    result.type = TokenType.string;
    bool haveDoubledQuotes = false;
    startPos = position;
    bool gotEndQuote = false;
    while (position < inputLength) {
      c = input[position++];
      if (c == '"') {
        if (position < inputLength && input[position] == '"') {
          // This is just a doubled quote.
          haveDoubledQuotes = true;
          position++;
        } else {
          // This is the closing quote, marking the end of the string.
          gotEndQuote = true;
          break;
        }
      } else if (c == '\n' || c == '\r') {
        // Break at end of line (string literals should not contain a line break).
        break;
      }
    }
    if (!gotEndQuote) throw LexerException("missing closing quote (\")");
    result.text = input.substring(startPos, position - 1);
    if (haveDoubledQuotes) {
      result.text = result.text!.replaceAll("\"\"", "\"");
    }
    return result;
  } else {
    result.type = TokenType.unknown;
  }

  result.text = input.substring(startPos, position);
  return result;
}