CompilerXArabic/tokenizer.py at master · TryOmar/CompilerXArabic · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
import re


class Token:
    def __init__(self, token_type, token_value, line=None):
        self.type = token_type
        self.value = token_value
        self.line = line

    def __str__(self):
        return f"{self.type}: {self.value} at line {self.line}"

    def __repr__(self):
        return f"{self.type}: {self.value} at line {self.line}"

    def is_token(self, token_type):
        return self.type == token_type or self.value == token_type


class Tokenizer:
    def __init__(self):
        self.token_patterns = {
            'specifier_type': r'صحيح(?![^\s])|حقيقي(?![^\s])|خالى(?![^\s])|خالي(?![^\s])',
            'if_stmt': r'اذا(?=\s|\()',
            'iteration': r'بينما(?=\s|\()',
            #  'if_stmt': r'اذا(?![^\s])',
            #  'iteration': r'بينما(?![^\s])',
            'else_stmt': r'اخر(?![^\s])',
            'return': r'ارجع(?![^\s])',

            # 'if_stmt': r'اذا',
            # 'else_stmt': r'اخر',
            # 'return': r'ارجع',
            # 'iteration': r'بينما',
            # 'keyword': r'ارجع|بينما|اخر|اذا|خالي|حقيقي|صحيح',
            'relOp': r'==|=!|=<|=>|<|>',
            'assign': r'=',
            'addOp': r'\+|-',
            'mulOp': r'\*|\\',
            'openBracket': r'[\(\[\{]',
            'closeBracket': r'[\)\]\}]',
            'ID': r'[_اإبتثجحخدذرزسشصضطظعغفقكلمنهويى]+[\da-zA-Z]*|[\da-zA-Z]+[ايإبتثجحخدذرزسشصضطظعغفقكلمنهوى_]+',
            'Num': r'(-?\d+(?:,\d+)?|-?\d*\.\d+|-?[٠-٩]+(?:٫[٠-٩]+)?|-?[٠-٩]*٫[٠-٩]+)',
            'semicolon': r'؛',
            'comma': r',',
        }

    def tokenize(self, text):
        output_tokens = []
        # split on  \n to get the line number
        lines = text.split('\n')
        for i, line in enumerate(lines):
            for match in re.finditer('|'.join('(?P<%s>%s)' % pair for pair in self.token_patterns.items()), line):
                token_type = match.lastgroup
                token_value = match.group(token_type)
                output_tokens.append(Token(token_type, token_value, i + 1))
        return output_tokens