CCF_100/relational
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Rework the tokenizing function

Browse Source
This commit is contained in:
Salvo 'LtWorf' Tomaselli 2020-08-12 11:05:47 +02:00
parent 40a15178fe
commit 786a9d61f5
No known key found for this signature in database
GPG Key ID: B3A7CF0C801886CF
1 changed files with 13 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
relational/optimizations.py
View File

@ -30,7 +30,7 @@
from io import StringIO from io import StringIO
from tokenize import generate_tokens from tokenize import generate_tokens
from typing import Tuple, Dict from typing import Tuple, Dict, List
from relational.relation import Relation from relational.relation import Relation
from relational import parser from relational import parser
@ -263,11 +263,11 @@ def subsequent_renames(n: parser.Node) -> Tuple[parser.Node, int]:
return n, 0 return n, 0
class level_string(str): class LevelString(str):
level = 0 level = 0
def tokenize_select(expression): def tokenize_select(expression: str) -> List[LevelString]:
'''This function returns the list of tokens present in a '''This function returns the list of tokens present in a
selection. The expression can contain parenthesis. selection. The expression can contain parenthesis.
It will use a subclass of str with the attribute level, which It will use a subclass of str with the attribute level, which
@ -275,8 +275,6 @@ def tokenize_select(expression):
g = generate_tokens(StringIO(str(expression)).readline) g = generate_tokens(StringIO(str(expression)).readline)
l = list(token[1] for token in g) l = list(token[1] for token in g)
l.remove('')
# Changes the 'a','.','method' token group into a single 'a.method' token # Changes the 'a','.','method' token group into a single 'a.method' token
try: try:
while True: while True:
@ -287,17 +285,21 @@ def tokenize_select(expression):
except: except:
pass pass
r = []
level = 0 level = 0
for i in range(len(l)): for i in l:
l[i] = level_string(l[i]) if not i:
l[i].level = level continue
value = LevelString(i)
value.level = level
if l[i] == '(': if value == '(':
level += 1 level += 1
elif l[i] == ')': elif value == ')':
level -= 1 level -= 1
r.append(value)
return l return r
def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: