You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

40 lines
1.1 KiB

from misc.util import isChar, isNumber
2 years ago
def tokenize(text: str) -> list:
buffer_num = ""
buffer_func = ""
tokens = []
for char in text:
if isChar(char):
2 years ago
buffer_func += char
elif isNumber(char) or char == ".":
2 years ago
buffer_num += char
else:
if buffer_num != "":
tokens.append(Token(Token.NUMBER, buffer_num))
buffer_num = ""
if buffer_func != "":
tokens.append(Token(Token.FUNCTION, buffer_func))
buffer_func = ""
if char == "(" or char == ")":
tokens.append(Token(Token.PARENTHESE, char))
elif char != " ":
tokens.append(Token(Token.OPERATOR, char))
if buffer_num != "":
tokens.append(Token(Token.NUMBER, buffer_num))
if buffer_func != "":
tokens.append(Token(Token.FUNCTION, buffer_func))
return tokens
class Token:
NUMBER = 0
OPERATOR = 1
PARENTHESE = 2
FUNCTION = 3
def __init__(self, type, val):
self.type = type
self.val = val
def __str__(self):
return self.val