CCF_100/relational
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Switch core module to Python3

Browse Source 
The core module (the relational algebra, not the interface) is now
using Python3

Tests are passing, so it should be ok.
This commit is contained in:
Salvo 'LtWorf' Tomaselli 2015-02-19 12:42:41 +01:00
parent 4019fbadb0
commit a355762fc4
9 changed files with 115 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
driver.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# coding=UTF-8 # coding=UTF-8
# Relational # Relational
@ -31,7 +31,7 @@ COLOR_GREEN = 0x00ff00
COLOR_MAGENTA = 0xff00ff COLOR_MAGENTA = 0xff00ff
COLOR_CYAN = 0x00ffff COLOR_CYAN = 0x00ffff
print relation print(relation)
rels = {} rels = {}
examples_path = 'samples/' examples_path = 'samples/'
@ -49,17 +49,17 @@ def readfile(fname):
def load_relations(): def load_relations():
'''Loads all the relations present in the directory indicated in the '''Loads all the relations present in the directory indicated in the
examples_path variable and stores them in the rels dictionary''' examples_path variable and stores them in the rels dictionary'''
print "Loading relations" print("Loading relations")
for i in os.listdir(examples_path): for i in os.listdir(examples_path):
if i.endswith('.csv'): # It's a relation, loading it if i.endswith('.csv'): # It's a relation, loading it
# Naming the relation # Naming the relation
relname = i[:-4] relname = i[:-4]
print ("Loading relation %s with name %s..." % (i, relname)), print ("Loading relation %s with name %s..." % (i, relname))
rels[relname] = relation.relation('%s%s' % (examples_path, i)) rels[relname] = relation.relation('%s%s' % (examples_path, i))
print 'done' print('done')
def execute_tests(): def execute_tests():
@ -93,49 +93,50 @@ def execute_tests():
ex_good += 1 ex_good += 1
else: else:
ex_bad += 1 ex_bad += 1
print colorize("Resume of the results", COLOR_CYAN) print (colorize("Resume of the results", COLOR_CYAN))
print colorize("Query tests", COLOR_MAGENTA) print (colorize("Query tests", COLOR_MAGENTA))
print "Total test count: %d" % q_tot print ("Total test count: %d" % q_tot)
print "Passed tests: %d" % q_good print ("Passed tests: %d" % q_good)
if q_bad > 0: if q_bad > 0:
print colorize("Failed tests count: %d" % q_bad, COLOR_RED) print (colorize("Failed tests count: %d" % q_bad, COLOR_RED))
print colorize("Python tests", COLOR_MAGENTA) print (colorize("Python tests", COLOR_MAGENTA))
print "Total test count: %d" % py_tot print ("Total test count: %d" % py_tot)
print "Passed tests: %d" % py_good print ("Passed tests: %d" % py_good)
if py_bad > 0: if py_bad > 0:
print colorize("Failed tests count: %d" % py_bad, COLOR_RED) print (colorize("Failed tests count: %d" % py_bad, COLOR_RED))
print colorize("Execute Python tests", COLOR_MAGENTA) print (colorize("Execute Python tests", COLOR_MAGENTA))
print "Total test count: %d" % ex_tot print ("Total test count: %d" % ex_tot)
print "Passed tests: %d" % ex_good print ("Passed tests: %d" % ex_good)
if ex_bad > 0: if ex_bad > 0:
print colorize("Failed tests count: %d" % ex_bad, COLOR_RED) print (colorize("Failed tests count: %d" % ex_bad, COLOR_RED))
print colorize("Total results", COLOR_CYAN) print (colorize("Total results", COLOR_CYAN))
if q_bad + py_bad + ex_bad == 0: if q_bad + py_bad + ex_bad == 0:
print colorize("No failed tests", COLOR_GREEN) print (colorize("No failed tests", COLOR_GREEN))
return 0 return 0
else: else:
print colorize("There are %d failed tests" % (py_bad + q_bad + ex_bad), COLOR_RED) print (colorize("There are %d failed tests" %
(py_bad + q_bad + ex_bad), COLOR_RED))
return 1 return 1
def run_exec_test(testname): def run_exec_test(testname):
'''Runs a python test, which executes code directly rather than queries''' '''Runs a python test, which executes code directly rather than queries'''
print "Running python test: " + colorize(testname, COLOR_MAGENTA) print ("Running python test: " + colorize(testname, COLOR_MAGENTA))
glob = rels.copy() glob = rels.copy()
exp_result = {} exp_result = {}
try: try:
expr = readfile('%s%s.exec' % (tests_path, testname))
exec(expr, glob) # Evaluating the expression
expr = readfile('%s%s.exec' % (tests_path, testname)) expr = readfile('%s%s.exec' % (tests_path, testname))
try:
exec(expr, glob) # Evaluating the expression exec(expr, glob) # Evaluating the expression
except Exception as e:
print (e)
raise Exception("")
expr = readfile('%s%s.result' % (tests_path, testname)) expr = readfile('%s%s.result' % (tests_path, testname))
exp_result = eval(expr, rels) # Evaluating the expression exp_result = eval(expr, rels) # Evaluating the expression
@ -147,21 +148,22 @@ def run_exec_test(testname):
fields_ok = fields_ok and glob[i] == exp_result[i] fields_ok = fields_ok and glob[i] == exp_result[i]
if fields_ok: if fields_ok:
print colorize('Test passed', COLOR_GREEN) print (colorize('Test passed', COLOR_GREEN))
return True return True
except: except:
pass pass
print colorize('ERROR', COLOR_RED) print (colorize('ERROR', COLOR_RED))
print colorize('=====================================', COLOR_RED) print (colorize('=====================================', COLOR_RED))
print "Expected %s" % exp_result print ("Expected %s" % exp_result)
# print "Got %s" % result # print ("Got %s" % glob)
print colorize('=====================================', COLOR_RED) print (colorize('=====================================', COLOR_RED))
return False return False
def run_py_test(testname): def run_py_test(testname):
'''Runs a python test, which evaluates expressions directly rather than queries''' '''Runs a python test, which evaluates expressions directly rather than queries'''
print "Running expression python test: " + colorize(testname, COLOR_MAGENTA) print ("Running expression python test: " +
colorize(testname, COLOR_MAGENTA))
try: try:
@ -172,16 +174,16 @@ def run_py_test(testname):
exp_result = eval(expr, rels) # Evaluating the expression exp_result = eval(expr, rels) # Evaluating the expression
if result == exp_result: if result == exp_result:
print colorize('Test passed', COLOR_GREEN) print (colorize('Test passed', COLOR_GREEN))
return True return True
except: except:
pass pass
print colorize('ERROR', COLOR_RED) print (colorize('ERROR', COLOR_RED))
print colorize('=====================================', COLOR_RED) print (colorize('=====================================', COLOR_RED))
print "Expected %s" % exp_result print ("Expected %s" % exp_result)
print "Got %s" % result print ("Got %s" % result)
print colorize('=====================================', COLOR_RED) print (colorize('=====================================', COLOR_RED))
return False return False
@ -192,7 +194,7 @@ def run_test(testname):
testname.result testname.result
The query will be executed both unoptimized and The query will be executed both unoptimized and
optimized''' optimized'''
print "Running test: " + colorize(testname, COLOR_MAGENTA) print ("Running test: " + colorize(testname, COLOR_MAGENTA))
query = None query = None
expr = None expr = None
@ -205,8 +207,7 @@ def run_test(testname):
try: try:
result_rel = relation.relation('%s%s.result' % (tests_path, testname)) result_rel = relation.relation('%s%s.result' % (tests_path, testname))
query = unicode( query = readfile('%s%s.query' % (tests_path, testname)).strip()
readfile('%s%s.query' % (tests_path, testname)).strip(), 'utf8')
o_query = optimizer.optimize_all(query, rels) o_query = optimizer.optimize_all(query, rels)
expr = parser.parse(query) # Converting expression to python string expr = parser.parse(query) # Converting expression to python string
@ -220,29 +221,31 @@ def run_test(testname):
c_result = eval(c_expr, rels) c_result = eval(c_expr, rels)
if (o_result == result_rel) and (result == result_rel) and (c_result == result_rel): if (o_result == result_rel) and (result == result_rel) and (c_result == result_rel):
print colorize('Test passed', COLOR_GREEN) print (colorize('Test passed', COLOR_GREEN))
return True return True
except Exception as inst: except Exception as inst:
print inst print (inst)
pass pass
print colorize('ERROR', COLOR_RED) print (colorize('ERROR', COLOR_RED))
print "Query: %s -> %s" % (query, expr) print ("Query: %s -> %s" % (query, expr))
print "Optimized query: %s -> %s" % (o_query, o_expr) print ("Optimized query: %s -> %s" % (o_query, o_expr))
print colorize('=====================================', COLOR_RED) print (colorize('=====================================', COLOR_RED))
print colorize("Expected result", COLOR_GREEN) print (colorize("Expected result", COLOR_GREEN))
print result_rel print (result_rel)
print colorize("Result", COLOR_RED) print (colorize("Result", COLOR_RED))
print result print (result)
print colorize("Optimized result", COLOR_RED) print (colorize("Optimized result", COLOR_RED))
print o_result print (o_result)
print colorize("optimized result match %s" % str(result_rel == o_result), COLOR_MAGENTA) print (colorize("optimized result match %s" %
print colorize("result match %s" % str(result == result_rel), COLOR_MAGENTA) str(result_rel == o_result), COLOR_MAGENTA))
print colorize('=====================================', COLOR_RED) print (colorize("result match %s" %
str(result == result_rel), COLOR_MAGENTA))
print (colorize('=====================================', COLOR_RED))
return False return False
if __name__ == '__main__': if __name__ == '__main__':
print "-> Starting testsuite for relational" print ("-> Starting testsuite for relational")
load_relations() load_relations()
print "-> Starting tests" print ("-> Starting tests")
exit(execute_tests()) exit(execute_tests())

9
relational/optimizations.py
View File

@ -29,11 +29,12 @@
# The class used is defined in optimizer module. # The class used is defined in optimizer module.
# A function will have to return the number of changes performed on the tree. # A function will have to return the number of changes performed on the tree.
import parser from io import StringIO
from cStringIO import StringIO
from tokenize import generate_tokens from tokenize import generate_tokens
from relational import parser
sel_op = ( sel_op = (
'//=', '**=', 'and', 'not', 'in', '//', '**', '<<', '>>', '==', '!=', '>=', '<=', '+=', '-=', '//=', '**=', 'and', 'not', 'in', '//', '**', '<<', '>>', '==', '!=', '>=', '<=', '+=', '-=',
'*=', '/=', '%=', 'or', '+', '-', '*', '/', '&', '|', '^', '~', '<', '>', '%', '=', '(', ')', ',', '[', ']') '*=', '/=', '%=', 'or', '+', '-', '*', '/', '&', '|', '^', '~', '<', '>', '%', '=', '(', ')', ',', '[', ']')
@ -620,4 +621,4 @@ general_optimizations = [
specific_optimizations = [selection_and_product] specific_optimizations = [selection_and_product]
if __name__ == "__main__": if __name__ == "__main__":
print tokenize_select("skill == 'C' and id % 2 == 0") print (tokenize_select("skill == 'C' and id % 2 == 0"))

12
relational/optimizer.py
View File

@ -25,8 +25,8 @@
# The functions will always return a string with the optimized query, but if a parse tree was provided, # The functions will always return a string with the optimized query, but if a parse tree was provided,
# the parse tree itself will be modified accordingly. # the parse tree itself will be modified accordingly.
import optimizations from relational import optimizations
import parser from relational import parser
# Stuff that was here before, keeping it for compatibility # Stuff that was here before, keeping it for compatibility
@ -56,12 +56,12 @@ def optimize_all(expression, rels, specific=True, general=True, debug=None):
steps. steps.
Return value: this will return an optimized version of the expression''' Return value: this will return an optimized version of the expression'''
if isinstance(expression, unicode): if isinstance(expression, str):
n = tree(expression) # Gets the tree n = tree(expression) # Gets the tree
elif isinstance(expression, node): elif isinstance(expression, node):
n = expression n = expression
else: else:
raise (TypeError("expression must be a unicode string or a node")) raise (TypeError("expression must be a string or a node"))
if isinstance(debug, list): if isinstance(debug, list):
dbg = True dbg = True
@ -128,8 +128,8 @@ if __name__ == "__main__":
print rels''' print rels'''
n = tree(u"π indice,qq,name (ρ age➡qq,id➡indice (P1-P2))") n = tree(u"π indice,qq,name (ρ age➡qq,id➡indice (P1-P2))")
# n=tree("σ id==3 and indice==2 and name==5 or name<2(P1 * S1)") # n=tree("σ id==3 and indice==2 and name==5 or name<2(P1 * S1)")
print n print (n)
print n.toPython() print (n.toPython())
# print optimizations.selection_and_product(n,rels) # print optimizations.selection_and_product(n,rels)

20
relational/parser.py
View File

@ -41,7 +41,8 @@
# Language definition here: # Language definition here:
# https://github.com/ltworf/relational/wiki/Grammar-and-language # https://github.com/ltworf/relational/wiki/Grammar-and-language
import re import re
import rtypes
from relational import rtypes
RELATION = 0 RELATION = 0
UNARY = 1 UNARY = 1
@ -107,7 +108,7 @@ class node (object):
expression = expression[0] expression = expression[0]
# The list contains only 1 string. Means it is the name of a relation # The list contains only 1 string. Means it is the name of a relation
if len(expression) == 1 and isinstance(expression[0], unicode): if len(expression) == 1:
self.kind = RELATION self.kind = RELATION
self.name = expression[0] self.name = expression[0]
if not rtypes.is_valid_relation_name(self.name): if not rtypes.is_valid_relation_name(self.name):
@ -125,7 +126,7 @@ class node (object):
Since it searches for strings, and expressions into parenthesis are Since it searches for strings, and expressions into parenthesis are
within sub-lists, they won't be found here, ensuring that they will within sub-lists, they won't be found here, ensuring that they will
have highest priority.''' have highest priority.'''
for i in xrange(len(expression) - 1, -1, -1): for i in range(len(expression) - 1, -1, -1):
if expression[i] in b_operators: # Binary operator if expression[i] in b_operators: # Binary operator
self.kind = BINARY self.kind = BINARY
self.name = expression[i] self.name = expression[i]
@ -142,7 +143,7 @@ class node (object):
self.right = node(expression[i + 1:]) self.right = node(expression[i + 1:])
return return
'''Searches for unary operators, parsing from right to left''' '''Searches for unary operators, parsing from right to left'''
for i in xrange(len(expression) - 1, -1, -1): for i in range(len(expression) - 1, -1, -1):
if expression[i] in u_operators: # Unary operator if expression[i] in u_operators: # Unary operator
self.kind = UNARY self.kind = UNARY
self.name = expression[i] self.name = expression[i]
@ -293,9 +294,6 @@ def tokenize(expression):
'''This function converts an expression into a list where '''This function converts an expression into a list where
every token of the expression is an item of a list. Expressions into every token of the expression is an item of a list. Expressions into
parenthesis will be converted into sublists.''' parenthesis will be converted into sublists.'''
if not isinstance(expression, unicode):
raise TokenizerException('expected unicode')
items = [] # List for the tokens items = [] # List for the tokens
'''This is a state machine. Initial status is determined by the starting of the '''This is a state machine. Initial status is determined by the starting of the
@ -334,7 +332,7 @@ def tokenize(expression):
elif expression.startswith((u"σ", u"π", u"ρ")): # Unary 2 bytes elif expression.startswith((u"σ", u"π", u"ρ")): # Unary 2 bytes
items.append(expression[0:1]) items.append(expression[0:1])
#Adding operator in the top of the list # Adding operator in the top of the list
expression = expression[ expression = expression[
1:].strip() # Removing operator from the expression 1:].strip() # Removing operator from the expression
@ -345,7 +343,7 @@ def tokenize(expression):
par = expression.find('(') par = expression.find('(')
items.append(expression[:par].strip()) items.append(expression[:par].strip())
#Inserting parameter of the operator # Inserting parameter of the operator
expression = expression[ expression = expression[
par:].strip() # Removing parameter from the expression par:].strip() # Removing parameter from the expression
elif expression.startswith((u"÷", u"ᑎ", u"ᑌ", u"*", u"-")): elif expression.startswith((u"÷", u"ᑎ", u"ᑌ", u"*", u"-")):
@ -417,8 +415,8 @@ def parse(expr):
if __name__ == "__main__": if __name__ == "__main__":
while True: while True:
e = unicode(raw_input("Expression: "), 'utf-8') e = str(raw_input("Expression: "))
print parse(e) print (parse(e))
# b=u"σ age>1 and skill=='C' (peopleᐅᐊskills)" # b=u"σ age>1 and skill=='C' (peopleᐅᐊskills)"
# print b[0] # print b[0]

6
relational/query.py
View File

@ -19,7 +19,7 @@
# #
# This module provides a classes to represent queries # This module provides a classes to represent queries
import parser from relational import
class TypeException(Exception): class TypeException(Exception):
@ -29,10 +29,6 @@ class TypeException(Exception):
class Query(object): class Query(object):
def __init__(self, query): def __init__(self, query):
if not isinstance(query, unicode):
raise TypeException('Expected unicode')
self.query = query self.query = query
self.tree = parser.tree(query) self.tree = parser.tree(query)
# TODO self.query_code = parser # TODO self.query_code = parser

13
relational/relation.py
View File

@ -20,9 +20,10 @@
# This module provides a classes to represent relations and to perform # This module provides a classes to represent relations and to perform
# relational operations on them. # relational operations on them.
from rtypes import *
import csv import csv
from relational.rtypes import *
class relation (object): class relation (object):
@ -46,13 +47,13 @@ class relation (object):
self.header = header([]) self.header = header([])
return return
# Opening file # Opening file
fp = file(filename) fp = open(filename)
reader = csv.reader(fp) # Creating a csv reader reader = csv.reader(fp) # Creating a csv reader
self.header = header(reader.next()) # read 1st line self.header = header(next(reader)) # read 1st line
self.content = set() self.content = set()
for i in reader.__iter__(): # Iterating rows for i in reader: # Iterating rows
self.content.add(tuple(i)) self.content.add(tuple(i))
# Closing file # Closing file
@ -123,7 +124,7 @@ class relation (object):
try: try:
if eval(expr, attributes): if eval(expr, attributes):
newt.content.add(i) newt.content.add(i)
except Exception, e: except Exception as e:
raise Exception( raise Exception(
"Failed to evaluate %s\n%s" % (expr, e.__str__())) "Failed to evaluate %s\n%s" % (expr, e.__str__()))
return newt return newt
@ -190,7 +191,7 @@ class relation (object):
newt = relation() newt = relation()
newt.header = header(list(self.header.attributes)) newt.header = header(list(self.header.attributes))
for old, new in params.iteritems(): for old, new in params.items():
if (newt.header.rename(old, new)) == False: if (newt.header.rename(old, new)) == False:
raise Exception('Unable to find attribute: %s' % old) raise Exception('Unable to find attribute: %s' % old)

46
relational_gui.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# coding=UTF-8 # coding=UTF-8
# Relational # Relational
@ -28,35 +28,35 @@ version = "1.3"
def printver(exit=True): def printver(exit=True):
print "Relational %s" % version print ("Relational %s" % version)
print "Copyright (C) 2008 Salvo 'LtWorf' Tomaselli." print ("Copyright (C) 2008 Salvo 'LtWorf' Tomaselli.")
print print ()
print "This program comes with ABSOLUTELY NO WARRANTY." print ("This program comes with ABSOLUTELY NO WARRANTY.")
print "This is free software, and you are welcome to redistribute it" print ("This is free software, and you are welcome to redistribute it")
print "under certain conditions." print ("under certain conditions.")
print "For details see the GPLv3 Licese." print ("For details see the GPLv3 Licese.")
print print ()
print "Written by Salvo 'LtWorf' Tomaselli <tiposchi@tiscali.it>" print ("Written by Salvo 'LtWorf' Tomaselli <tiposchi@tiscali.it>")
print print ()
print "http://ltworf.github.io/relational/" print ("http://ltworf.github.io/relational/")
if exit: if exit:
sys.exit(0) sys.exit(0)
def printhelp(code=0): def printhelp(code=0):
print "Relational" print ("Relational")
print print ()
print "Usage: %s [options] [files]" % sys.argv[0] print ("Usage: %s [options] [files]" % sys.argv[0])
print print ()
print " -v Print version and exits" print (" -v Print version and exits")
print " -h Print this help and exits" print (" -h Print this help and exits")
if sys.argv[0].endswith('relational-cli'): if sys.argv[0].endswith('relational-cli'):
print " -q Uses QT user interface" print (" -q Uses QT user interface")
print " -r Uses readline user interface (default)" print (" -r Uses readline user interface (default)")
else: else:
print " -q Uses QT user interface (default)" print (" -q Uses QT user interface (default)")
print " -r Uses readline user interface" print (" -r Uses readline user interface")
sys.exit(code) sys.exit(code)
if __name__ == "__main__": if __name__ == "__main__":
@ -116,7 +116,7 @@ if __name__ == "__main__":
defname = f[len(f) - 1].lower() defname = f[len(f) - 1].lower()
if defname.endswith(".csv"): # removes the extension if defname.endswith(".csv"): # removes the extension
defname = defname[:-4] defname = defname[:-4]
print 'Loading file "%s" with name "%s"' % (files[i], defname) print ('Loading file "%s" with name "%s"' % (files[i], defname))
Form.loadRelation(files[i], defname) Form.loadRelation(files[i], defname)
Form.show() Form.show()

2
test/rename_insert1.exec
View File

@ -1,7 +1,5 @@
p1=people.rename({"id":"ido"}) p1=people.rename({"id":"ido"})
people.insert((123,"lala",0,31)) people.insert((123,"lala",0,31))
print p1
print people
retval=people==p1 retval=people==p1
people.delete("id==123") people.delete("id==123")

2
test/rename_insert2.exec
View File

@ -1,7 +1,5 @@
p1=people.rename({"id":"ido"}) p1=people.rename({"id":"ido"})
p1.insert((123,"lala",0,31)) p1.insert((123,"lala",0,31))
print p1
print people
retval=people==p1 retval=people==p1
people.delete("id==123") people.delete("id==123")