calculatrice_aboulin/misc/tokenizer.py

from misc.util import isChar, isNumber

def tokenize(text: str) -> list:
    buffer_num = ""
    buffer_func = ""
    tokens = []
    for char in text:
        if isChar(char):
            buffer_func += char
        elif isNumber(char) or char == ".":
            buffer_num += char
        else:
            if buffer_num != "":
                tokens.append(Token(Token.NUMBER, buffer_num))
                buffer_num = ""
            if buffer_func != "":
                tokens.append(Token(Token.FUNCTION, buffer_func))
                buffer_func = ""
            if char == "(" or char == ")":
                tokens.append(Token(Token.PARENTHESE, char))
            elif char != " ":
                tokens.append(Token(Token.OPERATOR, char))
    if buffer_num != "":
        tokens.append(Token(Token.NUMBER, buffer_num))
    if buffer_func != "":
        tokens.append(Token(Token.FUNCTION, buffer_func))
    return tokens

class Token:
    NUMBER = 0
    OPERATOR = 1
    PARENTHESE = 2
    FUNCTION = 3

    def __init__(self, type, val):
        self.type = type
        self.val = val

    def __str__(self):
        return self.val