You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
40 lines
1.1 KiB
40 lines
1.1 KiB
from misc.util import isChar, isNumber
|
|
|
|
def tokenize(text: str) -> list:
|
|
buffer_num = ""
|
|
buffer_func = ""
|
|
tokens = []
|
|
for char in text:
|
|
if isChar(char):
|
|
buffer_func += char
|
|
elif isNumber(char) or char == ".":
|
|
buffer_num += char
|
|
else:
|
|
if buffer_num != "":
|
|
tokens.append(Token(Token.NUMBER, buffer_num))
|
|
buffer_num = ""
|
|
if buffer_func != "":
|
|
tokens.append(Token(Token.FUNCTION, buffer_func))
|
|
buffer_func = ""
|
|
if char == "(" or char == ")":
|
|
tokens.append(Token(Token.PARENTHESE, char))
|
|
elif char != " ":
|
|
tokens.append(Token(Token.OPERATOR, char))
|
|
if buffer_num != "":
|
|
tokens.append(Token(Token.NUMBER, buffer_num))
|
|
if buffer_func != "":
|
|
tokens.append(Token(Token.FUNCTION, buffer_func))
|
|
return tokens
|
|
|
|
class Token:
|
|
NUMBER = 0
|
|
OPERATOR = 1
|
|
PARENTHESE = 2
|
|
FUNCTION = 3
|
|
|
|
def __init__(self, type, val):
|
|
self.type = type
|
|
self.val = val
|
|
|
|
def __str__(self):
|
|
return self.val
|