diff --git a/README.md b/README.md index 7e276754..ac0fd69b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +#george's,iraklis',nikos' repo

mdblogo

diff --git a/mdb.py b/mdb.py index a981e5be..caa2b8d4 100644 --- a/mdb.py +++ b/mdb.py @@ -1,3 +1,5 @@ +from miniDB.table import Table +from miniDB.database import Database import os import re from pprint import pprint @@ -7,8 +9,6 @@ import shutil sys.path.append('miniDB') -from database import Database -from table import Table # art font is "big" art = ''' _ _ _____ ____ @@ -17,7 +17,7 @@ | '_ ` _ \ | || '_ \ | || | | || _ < | | | | | || || | | || || |__| || |_) | |_| |_| |_||_||_| |_||_||_____/ |____/ 2022 -''' +''' def search_between(s, first, last): @@ -25,17 +25,18 @@ def search_between(s, first, last): Search in 's' for the substring that is between 'first' and 'last' ''' try: - start = s.index( first ) + len( first ) - end = s.index( last, start ) + start = s.index(first) + len(first) + end = s.index(last, start) except: return return s[start:end].strip() + def in_paren(qsplit, ind): ''' Split string on space and return whether the item in index 'ind' is inside a parentheses ''' - return qsplit[:ind].count('(')>qsplit[:ind].count(')') + return qsplit[:ind].count('(') > qsplit[:ind].count(')') def create_query_plan(query, keywords, action): @@ -45,37 +46,35 @@ def create_query_plan(query, keywords, action): This can and will be used recursively ''' - dic = {val: None for val in keywords if val!=';'} + dic = {val: None for val in keywords if val != ';'} - ql = [val for val in query.split(' ') if val !=''] + ql = [val for val in query.split(' ') if val != ''] kw_in_query = [] kw_positions = [] - i=0 - while i ', auto_suggest=AutoSuggestFromHistory()).lower() - if line[-1]!=';': - line+=';' + line = session.prompt( + f'({db._name})> ', auto_suggest=AutoSuggestFromHistory()).lower() + if line[-1] != ';': + line += ';' except (KeyboardInterrupt, EOFError): print('\nbye!') break try: - if line=='exit': + if line == 'exit': break if line.split(' ')[0].removesuffix(';') in ['lsdb', 'lstb', 'cdb', 'rmdb']: interpret_meta(line) @@ -292,7 +309,7 @@ def remove_db(db_name): else: dic = interpret(line) result = execute_dic(dic) - if isinstance(result,Table): + if isinstance(result, Table): result.show() except Exception: print(traceback.format_exc()) diff --git a/miniDB/database.py b/miniDB/database.py index a3ac6be7..29e599cd 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -1,21 +1,22 @@ from __future__ import annotations +from table import Table +from misc import split_condition +from btree import Btree +from joins import Inlj, Smj +from miniDB import table import pickle from time import sleep, localtime, strftime -import os,sys +import os +import sys import logging import warnings import readline from tabulate import tabulate -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') -from miniDB import table +sys.path.append( + f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') sys.modules['table'] = table -from joins import Inlj, Smj -from btree import Btree -from misc import split_condition -from table import Table - # readline.clear_history() @@ -24,7 +25,7 @@ class Database: Main Database class, containing tables. ''' - def __init__(self, name, load=True, verbose = True): + def __init__(self, name, load=True, verbose=True): self.tables = {} self._name = name self.verbose = verbose @@ -38,7 +39,8 @@ def __init__(self, name, load=True, verbose = True): return except: if verbose: - warnings.warn(f'Database "{name}" does not exist. Creating new.') + warnings.warn( + f'Database "{name}" does not exist. Creating new.') # create dbdata directory if it doesnt exist if not os.path.exists('dbdata'): @@ -53,8 +55,11 @@ def __init__(self, name, load=True, verbose = True): # create all the meta tables self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') - self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') - self.create_table('meta_indexes', 'table_name,index_name', 'str,str') + self.create_table('meta_insert_stack', + 'table_name,indexes', 'str,list') + # Προσθηκη στηλης στον πινακα για το column_name του index + self.create_table( + 'meta_indexes', 'table_name,index_name,column_name', 'str,str,str') self.save_database() def save_database(self): @@ -82,7 +87,7 @@ def load_database(self): path = f'dbdata/{self._name}_db' for file in os.listdir(path): - if file[-3:]!='pkl': # if used to load only pkl files + if file[-3:] != 'pkl': # if used to load only pkl files continue f = open(path+'/'+file, 'rb') tmp_dict = pickle.load(f) @@ -100,7 +105,6 @@ def _update(self): self._update_meta_length() self._update_meta_insert_stack() - def create_table(self, name, column_names, column_types, primary_key=None, load=None): ''' This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name @@ -113,7 +117,8 @@ def create_table(self, name, column_names, column_types, primary_key=None, load= load: boolean. Defines table object parameters as the name of the table and the column names. ''' # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) + self.tables.update({name: Table(name=name, column_names=column_names.split( + ','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) # check that new dynamic var doesnt exist already # self.no_of_tables += 1 @@ -123,7 +128,6 @@ def create_table(self, name, column_names, column_types, primary_key=None, load= if self.verbose: print(f'Created table "{name}".') - def drop_table(self, table_name): ''' Drop table from current database. @@ -159,7 +163,6 @@ def drop_table(self, table_name): # self._update() self.save_database() - def import_table(self, table_name, filename, column_types=None, primary_key=None): ''' Creates table from CSV file. @@ -171,24 +174,25 @@ def import_table(self, table_name, filename, column_types=None, primary_key=None ''' file = open(filename, 'r') - first_line=True + first_line = True for line in file.readlines(): if first_line: colnames = line.strip('\n') if column_types is None: - column_types = ",".join(['str' for _ in colnames.split(',')]) - self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key) + column_types = ",".join( + ['str' for _ in colnames.split(',')]) + self.create_table(name=table_name, column_names=colnames, + column_types=column_types, primary_key=primary_key) lock_ownership = self.lock_table(table_name, mode='x') first_line = False continue self.tables[table_name]._insert(line.strip('\n').split(',')) if lock_ownership: - self.unlock_table(table_name) + self.unlock_table(table_name) self._update() self.save_database() - def export(self, table_name, filename=None): ''' Transform table to CSV. @@ -199,13 +203,14 @@ def export(self, table_name, filename=None): ''' res = '' for row in [self.tables[table_name].column_names]+self.tables[table_name].data: - res+=str(row)[1:-1].replace('\'', '').replace('"','').replace(' ','')+'\n' + res += str(row)[1:-1].replace('\'', + '').replace('"', '').replace(' ', '')+'\n' if filename is None: filename = f'{table_name}.csv' with open(filename, 'w') as file: - file.write(res) + file.write(res) def table_from_object(self, new_table): ''' @@ -219,12 +224,11 @@ def table_from_object(self, new_table): if new_table._name not in self.__dir__(): setattr(self, new_table._name, new_table) else: - raise Exception(f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') + raise Exception( + f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') self._update() self.save_database() - - ##### table functions ##### # In every table function a load command is executed to fetch the most recent table. @@ -246,7 +250,7 @@ def cast(self, column_name, table_name, cast_type): cast_type: type. Cast type (do not encapsulate in quotes). ''' self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') self.tables[table_name]._cast_column(column_name, eval(cast_type)) if lock_ownership: @@ -281,7 +285,6 @@ def insert_into(self, table_name, row_str): self._update() self.save_database() - def update_table(self, table_name, set_args, condition): ''' Update the value of a column where a condition is met. @@ -293,12 +296,12 @@ def update_table(self, table_name, set_args, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) ''' - set_column, set_value = set_args.replace(' ','').split('=') + set_column, set_value = set_args.replace(' ', '').split('=') self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') self.tables[table_name]._update_rows(set_value, set_column, condition) if lock_ownership: @@ -315,11 +318,11 @@ def delete_from(self, table_name, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) ''' self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') deleted = self.tables[table_name]._delete_where(condition) if lock_ownership: @@ -327,11 +330,11 @@ def delete_from(self, table_name, condition): self._update() self.save_database() # we need the save above to avoid loading the old database that still contains the deleted elements - if table_name[:4]!='meta': + if table_name[:4] != 'meta': self._add_to_insert_stack(table_name, deleted) self.save_database() - def select(self, columns, table_name, condition, distinct=None, order_by=None, \ + def select(self, columns, table_name, condition, distinct=None, order_by=None, limit=True, desc=None, save_as=None, return_object=True): ''' Selects and outputs a table's data where condtion is met. @@ -342,7 +345,7 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). desc: boolean. If True, order_by will return results in descending order (True by default). @@ -352,26 +355,38 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ distinct: boolean. If True, the resulting table will contain only unique rows. ''' - # print(table_name) - self.load_database() - if isinstance(table_name,Table): - return table_name._select_where(columns, condition, distinct, order_by, desc, limit) - + # check for operators if condition is not None: - condition_column = split_condition(condition)[0] + # elegxos an mesa sto condition yparxei and,or,not,between + # An nai tote eisagoyme to prwto stoixeio ths syntikis meta to split sto condition column + + if "BETWEEN" in condition.split() or "NOT" in condition.split() or "AND" in condition.split() or "OR" in condition.split(): + condition_column = condition.split(" ")[0] + elif "between" in condition.split() or "not" in condition.split() or "and" in condition.split() or "or" in condition.split(): + condition_column = condition.split(" ")[0] + else: + condition_column = split_condition(" ")[0] + else: condition_column = '' - # self.lock_table(table_name, mode='x') if self.is_locked(table_name): return - if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] + if self._has_index(table_name) and condition_column == self.tables[table_name].column_names[self.tables[table_name].pk_idx]: + index_name = self.select( + '*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + try: + table = self.tables[table_name]._select_where_with_btree( + columns, bt, condition, distinct, order_by, desc, limit) + # hash index in case first fails + except: + table = self.tables[table_name]._select_where_with_hash( + columns, bt, condition, distinct, order_by, desc, limit) else: - table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + table = self.tables[table_name]._select_where( + columns, condition, distinct, order_by, desc, limit) # self.unlock_table(table_name) if save_as is not None: table._name = save_as @@ -382,7 +397,6 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ else: return table.show() - def show_table(self, table_name, no_of_rows=None): ''' Print table in a readable tabular design (using tabulate). @@ -391,9 +405,8 @@ def show_table(self, table_name, no_of_rows=None): table_name: string. Name of table (must be part of database). ''' self.load_database() - - self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) + self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) def sort(self, table_name, column_name, asc=False): ''' @@ -406,7 +419,7 @@ def sort(self, table_name, column_name, asc=False): ''' self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') self.tables[table_name]._sort(column_name, asc=asc) if lock_ownership: @@ -435,7 +448,7 @@ def join(self, mode, left_table, right_table, condition, save_as=None, return_ob condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) save_as: string. The output filename that will be used to save the resulting table in the database (won't save if None). return_object: boolean. If True, the result will be a table object (useful for internal usage - the result will be printed by default). @@ -444,38 +457,44 @@ def join(self, mode, left_table, right_table, condition, save_as=None, return_ob if self.is_locked(left_table) or self.is_locked(right_table): return - left_table = left_table if isinstance(left_table, Table) else self.tables[left_table] - right_table = right_table if isinstance(right_table, Table) else self.tables[right_table] - + left_table = left_table if isinstance( + left_table, Table) else self.tables[left_table] + right_table = right_table if isinstance( + right_table, Table) else self.tables[right_table] - if mode=='inner': + if mode == 'inner': res = left_table._inner_join(right_table, condition) - - elif mode=='left': + + elif mode == 'left': res = left_table._left_join(right_table, condition) - - elif mode=='right': + + elif mode == 'right': res = left_table._right_join(right_table, condition) - - elif mode=='full': + + elif mode == 'full': res = left_table._full_join(right_table, condition) - elif mode=='inl': + elif mode == 'inl': # Check if there is an index of either of the two tables available, as if there isn't we can't use inlj leftIndexExists = self._has_index(left_table._name) rightIndexExists = self._has_index(right_table._name) if not leftIndexExists and not rightIndexExists: res = None - raise Exception('Index-nested-loop join cannot be executed. Use inner join instead.\n') + raise Exception( + 'Index-nested-loop join cannot be executed. Use inner join instead.\n') elif rightIndexExists: - index_name = self.select('*', 'meta_indexes', f'table_name={right_table._name}', return_object=True).column_by_name('index_name')[0] - res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'right').join() + index_name = self.select( + '*', 'meta_indexes', f'table_name={right_table._name}', return_object=True).column_by_name('index_name')[0] + res = Inlj(condition, left_table, right_table, + self._load_idx(index_name), 'right').join() elif leftIndexExists: - index_name = self.select('*', 'meta_indexes', f'table_name={left_table._name}', return_object=True).column_by_name('index_name')[0] - res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'left').join() + index_name = self.select( + '*', 'meta_indexes', f'table_name={left_table._name}', return_object=True).column_by_name('index_name')[0] + res = Inlj(condition, left_table, right_table, + self._load_idx(index_name), 'left').join() - elif mode=='sm': + elif mode == 'sm': res = Smj(condition, left_table, right_table).join() else: @@ -502,23 +521,25 @@ def lock_table(self, table_name, mode='x'): Args: table_name: string. Table name (must be part of database). ''' - if table_name[:4]=='meta' or table_name not in self.tables.keys() or isinstance(table_name,Table): + if table_name[:4] == 'meta' or table_name not in self.tables.keys() or isinstance(table_name, Table): return with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: self.tables.update({'meta_locks': pickle.load(f)}) try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by process with pid={pid}') + pid = self.tables['meta_locks']._select_where( + 'pid', f'table_name={table_name}').data[0][0] + if pid != os.getpid(): + raise Exception( + f'Table "{table_name}" is locked by process with pid={pid}') else: return False except IndexError: pass - if mode=='x': + if mode == 'x': self.tables['meta_locks']._insert([table_name, os.getpid(), mode]) else: raise NotImplementedError @@ -539,9 +560,11 @@ def unlock_table(self, table_name, force=False): if not force: try: # pid = self.select('*','meta_locks', f'table_name={table_name}', return_object=True).data[0][1] - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + pid = self.tables['meta_locks']._select_where( + 'pid', f'table_name={table_name}').data[0][0] + if pid != os.getpid(): + raise Exception( + f'Table "{table_name}" is locked by the process with pid={pid}') except IndexError: pass self.tables['meta_locks']._delete_where(f'table_name={table_name}') @@ -555,22 +578,23 @@ def is_locked(self, table_name): Args: table_name: string. Table name (must be part of database). ''' - if isinstance(table_name,Table) or table_name[:4]=='meta': # meta tables will never be locked (they are internal) + if isinstance(table_name, Table) or table_name[:4] == 'meta': # meta tables will never be locked (they are internal) return False with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: self.tables.update({'meta_locks': pickle.load(f)}) try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + pid = self.tables['meta_locks']._select_where( + 'pid', f'table_name={table_name}').data[0][0] + if pid != os.getpid(): + raise Exception( + f'Table "{table_name}" is locked by the process with pid={pid}') except IndexError: pass return False - #### META #### # The following functions are used to update, alter, load and save the meta tables. @@ -582,15 +606,17 @@ def _update_meta_length(self): Updates the meta_length table. ''' for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables + if table._name[:4] == 'meta': # skip meta tables continue - if table._name not in self.tables['meta_length'].column_by_name('table_name'): # if new table, add record with 0 no. of rows + # if new table, add record with 0 no. of rows + if table._name not in self.tables['meta_length'].column_by_name('table_name'): self.tables['meta_length']._insert([table._name, 0]) # the result needs to represent the rows that contain data. Since we use an insert_stack # some rows are filled with Nones. We skip these rows. non_none_rows = len([row for row in table.data if any(row)]) - self.tables['meta_length']._update_rows(non_none_rows, 'no_of_rows', f'table_name={table._name}') + self.tables['meta_length']._update_rows( + non_none_rows, 'no_of_rows', f'table_name={table._name}') # self.update_row('meta_length', len(table.data), 'no_of_rows', 'table_name', '==', table._name) def _update_meta_locks(self): @@ -598,7 +624,7 @@ def _update_meta_locks(self): Updates the meta_locks table. ''' for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables + if table._name[:4] == 'meta': # skip meta tables continue if table._name not in self.tables['meta_locks'].column_by_name('table_name'): @@ -610,12 +636,11 @@ def _update_meta_insert_stack(self): Updates the meta_insert_stack table. ''' for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables + if table._name[:4] == 'meta': # skip meta tables continue if table._name not in self.tables['meta_insert_stack'].column_by_name('table_name'): self.tables['meta_insert_stack']._insert([table._name, []]) - def _add_to_insert_stack(self, table_name, indexes): ''' Adds provided indices to the insert stack of the specified table. @@ -646,11 +671,13 @@ def _update_meta_insert_stack_for_tb(self, table_name, new_stack): table_name: string. Table name (must be part of database). new_stack: string. The stack that will be used to replace the existing one. ''' - self.tables['meta_insert_stack']._update_rows(new_stack, 'indexes', f'table_name={table_name}') - + self.tables['meta_insert_stack']._update_rows( + new_stack, 'indexes', f'table_name={table_name}') # indexes - def create_index(self, index_name, table_name, index_type='btree'): + # Προσθηκη argument που αφορα column του πινακα + + def create_index(self, index_name, table_name, column_name, index_type): ''' Creates an index on a specified table with a given name. Important: An index can only be created on a primary key (the user does not specify the column). @@ -659,21 +686,39 @@ def create_index(self, index_name, table_name, index_type='btree'): table_name: string. Table name (must be part of database). index_name: string. Name of the created index. ''' - if self.tables[table_name].pk_idx is None: # if no primary key, no index - raise Exception('Cannot create index. Table has no primary key.') + # Αφαιρεση exception για δυνατοτητα index σε ολες τις στηλες + if table_name not in self.tables: # πινακας δεν υπαρχει + raise Exception('Index creation stopped. Table name invalid.') + + # στηλη δεν υπαρχει + if column_name not in self.tables[table_name].column_names: + raise Exception('Index creation stopped. Column name invalid.') + + # ηδη υπαρχει index_name if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): # currently only btree is supported. This can be changed by adding another if. - if index_type=='btree': + if index_type == 'btree' or index_type == 'BTREE': logging.info('Creating Btree index.') # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name]) + self.tables['meta_indexes']._insert( + [table_name, index_name, column_name]) # crate the actual index - self._construct_index(table_name, index_name) + self._construct_index(table_name, index_name, column_name) + self.save_database() + elif index_type == 'hash' or index_type == 'HASH': + logging.info('Creating Hash index') + # insert a record with the name of the index and the table on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert( + [table_name, index_name, column_name]) + # crate the actual index + self._construct_index_hash(table_name, index_name, column_name) self.save_database() else: - raise Exception('Cannot create index. Another index with the same name already exists.') + raise Exception( + 'Index creation stopped. This index name is already in use.') - def _construct_index(self, table_name, index_name): + # προσθηκη argument του column του index + def _construct_index(self, table_name, index_name, column_name): ''' Construct a btree on a table and save. @@ -681,16 +726,48 @@ def _construct_index(self, table_name, index_name): table_name: string. Table name (must be part of database). index_name: string. Name of the created index. ''' - bt = Btree(3) # 3 is arbitrary + bt = Btree(3) # 3 is arbitrary # for each record in the primary key of the table, insert its value and index to the btree - for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): + for idx, key in enumerate(self.tables[table_name].column_by_name(column_name)): if key is None: continue bt.insert(key, idx) # save the btree self._save_index(index_name, bt) + return + + def _construct_index_hash(self, table_name, index_name, column_name): + r_length = len(self.tables[table_name].data) + hash_map = {} + hash_map[0] = {} + hash_map[0][0] = [str(r_length)] # store the number of the rows + for idx, key in enumerate(self.tables[table_name].column_by_name(column_name)): + if key is None: + continue + hash_sum = 0 + for letter in key: + hash_sum += ord(letter) + + hash_index = hash_sum % r_length + + sub_hash_index = hash_index + if not (hash_index in hash_map): + hash_map[hash_index] = {} + else: + while True: + if not (sub_hash_index in hash_map[hash_index]): + break + + if (sub_hash_index == r_length): + sub_hash_index = 0 + else: + sub_hash_index += 1 + + hash_map[hash_index][sub_hash_index] = [idx, key] + + self._save_index(index_name, hash_map) def _has_index(self, table_name): ''' @@ -740,9 +817,13 @@ def drop_index(self, index_name): self.delete_from('meta_indexes', f'index_name = {index_name}') if os.path.isfile(f'{self.savedir}/indexes/meta_{index_name}_index.pkl'): - os.remove(f'{self.savedir}/indexes/meta_{index_name}_index.pkl') + os.remove( + f'{self.savedir}/indexes/meta_{index_name}_index.pkl') else: - warnings.warn(f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') + warnings.warn( + f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') self.save_database() - \ No newline at end of file + else: + # ειδικη περιπτωση για την αδυναμια ευρεσης του index + raise Exception('Cannot find index') diff --git a/miniDB/table.py b/miniDB/table.py index f5c7d937..6a56cfcb 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -1,12 +1,12 @@ from __future__ import annotations +from miniDB.misc import get_op, split_condition, reverse_op from tabulate import tabulate import pickle import os import sys -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') - -from misc import get_op, split_condition +sys.path.append( + f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') class Table: @@ -26,6 +26,7 @@ class Table: - a dictionary that includes the appropriate info (all the attributes in __init__) ''' + def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): if load is not None: @@ -42,7 +43,7 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= self._name = name - if len(column_names)!=len(column_types): + if len(column_names) != len(column_types): raise ValueError('Need same number of column names and types.') self.column_names = column_names @@ -56,10 +57,12 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= setattr(self, col, []) self.columns.append([]) else: - raise Exception(f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') + raise Exception( + f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') - self.column_types = [eval(ct) if not isinstance(ct, type) else ct for ct in column_types] - self.data = [] # data is a list of lists, a list of rows that is. + self.column_types = [eval(ct) if not isinstance( + ct, type) else ct for ct in column_types] + self.data = [] # data is a list of lists, a list of rows that is. # if primary key is set, keep its index as an attribute if primary_key is not None: @@ -75,12 +78,12 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= def column_by_name(self, column_name): return [row[self.column_names.index(column_name)] for row in self.data] - def _update(self): ''' Update all the available columns with the appended rows. ''' - self.columns = [[row[i] for row in self.data] for i in range(len(self.column_names))] + self.columns = [[row[i] for row in self.data] + for i in range(len(self.column_names))] for ind, col in enumerate(self.column_names): setattr(self, col, self.columns[ind]) @@ -101,7 +104,6 @@ def _cast_column(self, column_name, cast_type): self.column_types[column_idx] = cast_type # self._update() - def _insert(self, row, insert_stack=[]): ''' Insert row to table. @@ -110,8 +112,9 @@ def _insert(self, row, insert_stack=[]): row: list. A list of values to be inserted (will be casted to a predifined type automatically). insert_stack: list. The insert stack (empty by default). ''' - if len(row)!=len(self.column_names): - raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') + if len(row) != len(self.column_names): + raise ValueError( + f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') for i in range(len(row)): # for each value, cast and replace it in row. @@ -119,21 +122,24 @@ def _insert(self, row, insert_stack=[]): row[i] = self.column_types[i](row[i]) except ValueError: if row[i] != 'NULL': - raise ValueError(f'ERROR -> Value {row[i]} of type {type(row[i])} is not of type {self.column_types[i]}.') + raise ValueError( + f'ERROR -> Value {row[i]} of type {type(row[i])} is not of type {self.column_types[i]}.') except TypeError as exc: if row[i] != None: print(exc) # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) - if i==self.pk_idx and row[i] in self.column_by_name(self.pk): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') - elif i==self.pk_idx and row[i] is None: - raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + if i == self.pk_idx and row[i] in self.column_by_name(self.pk): + raise ValueError( + f'## ERROR -> Value {row[i]} already exists in primary key column.') + elif i == self.pk_idx and row[i] is None: + raise ValueError( + f'ERROR -> The value of the primary key cannot be None.') # if insert_stack is not empty, append to its last index if insert_stack != []: self.data[insert_stack[-1]] = row - else: # else append to the end + else: # else append to the end self.data.append(row) # self._update() @@ -147,7 +153,7 @@ def _update_rows(self, set_value, set_column, condition): condition: string. A condition using the following format: 'column[<,<=,=,>=,>]value' or 'value[<,<=,=,>=,>]column'. - + Operatores supported: (<,<=,=,>=,>) ''' # parse the condition @@ -167,7 +173,6 @@ def _update_rows(self, set_value, set_column, condition): # self._update() # print(f"Updated {len(indexes_to_del)} rows") - def _delete_where(self, condition): ''' Deletes rows where condition is met. @@ -179,7 +184,7 @@ def _delete_where(self, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) ''' column_name, operator, value = self._parse_condition(condition) @@ -198,7 +203,8 @@ def _delete_where(self, condition): for index in sorted(indexes_to_del, reverse=True): if self._name[:4] != 'meta': # if the table is not a metatable, replace the row with a row of nones - self.data[index] = [None for _ in range(len(self.column_names))] + self.data[index] = [ + None for _ in range(len(self.column_names))] else: self.data.pop(index) @@ -206,7 +212,6 @@ def _delete_where(self, condition): # we have to return the deleted indexes, since they will be appended to the insert_stack return indexes_to_del - def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None): ''' Select and return a table containing specified columns and rows where condition is met. @@ -216,7 +221,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) distinct: boolean. If True, the resulting table will contain only unique rows (False by default). order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). @@ -228,28 +233,88 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by if return_columns == '*': return_cols = [i for i in range(len(self.column_names))] else: - return_cols = [self.column_names.index(col.strip()) for col in return_columns.split(',')] + return_cols = [self.column_names.index( + col.strip()) for col in return_columns.split(',')] # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + # not + if "NOT" in condition.split() or "not" in condition.split(): + con_lst = condition.split("NOT") + con_lst = con_lst[0].split("not") + column_name, operator, value = self._parse_condition( + con_lst[1]) + column = self.column_by_name(column_name) + sec_op = reverse_op(operator) + rows = [ind for ind, x in enumerate( + column) if get_op(sec_op, x, value)] + # between + elif "BETWEEN" in condition.split() or "between" in condition.split(): + con_split = condition.split() + # Tsekarw an yparxei "and" meta tin prwth synthiki gia na nai swstos o kwdikas + if (con_split[3] != 'and'): + print('Bale "and" anamesa stoys arithmoys') + exit() + else: + # H synthiki meta anamesa sto "between" kai to "and" + first_value = con_split[4] + second_value = con_split[2] # H synthiki meta to "and" + column_name = con_split[0] + column = self.column_by_name(column_name) + rows = [] + # Elegxw an oi times einai arithmoi + if (first_value.isdigit() and second_value.isdigit()): + for i, j in enumerate(column): + if int(j) >= int(first_value) and int(j) <= int(second_value): + rows.append(i) + else: + print("Not allowed strings") + exit() + # and + elif "AND" in condition.split() or "and" in condition.split(): + con_lst = condition.split("AND") + con_lst = con_lst[0].spit("and") + lists_for_rows = [] + for i in con_lst: + column_name, operator, value = self._parse_condition(i) + column = self.column_by_name(column_name) + lists_for_rows.append([ind for ind, x in enumerate( + column) if get_op(operator, x, value)]) + rows = set(lists_for_rows[0].intersection(*lists_for_rows)) + # or + elif "OR" in condition.split() or "or" in condition.split(): + con_lst = condition.split("OR") + con_lst = con_lst[0].split("or") + + lists_for_rows = [] + for i in con_lst: # gia kathe synthiki ksexwrista + column_name, operator, value = self._parse_condition(i) + column = self.column_by_name(column_name) + lists_for_rows.append([ind for ind, x in enumerate( + column) if get_op(operator, x, value)]) + + rows = [] + for l in lists_for_rows: # metakinhse ola ta rows sthn 1h lista + for row in l: + if not (row in rows): + rows.append(row) else: rows = [i for i in range(len(self.data))] # copy the old dict, but only the rows and columns of data with index in rows/columns (the indexes that we want returned) - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + dict = {(key): ([[self.data[i][j] for j in return_cols] for i in rows] + if key == "data" else value) for key, value in self.__dict__.items()} # we need to set the new column names/types and no of columns, since we might # only return some columns dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] s_table = Table(load=dict) - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + s_table.data = list( + set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data if order_by: s_table.order_by(order_by, desc) @@ -259,25 +324,25 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # k = int(limit) # except ValueError: # raise Exception("The value following 'top' in the query should be a number.") - + # # Remove from the table's data all the None-filled rows, as they are not shown by default - # # Then, show the first k rows + # # Then, show the first k rows # s_table.data.remove(len(s_table.column_names) * [None]) # s_table.data = s_table.data[:k] - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if any(row)][:int(limit)] + if isinstance(limit, str): + s_table.data = [ + row for row in s_table.data if any(row)][:int(limit)] return s_table - def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): # if * return all columns, else find the column indexes for the columns specified if return_columns == '*': return_cols = [i for i in range(len(self.column_names))] else: - return_cols = [self.column_names.index(colname) for colname in return_columns] - + return_cols = [self.column_names.index( + colname) for colname in return_columns] column_name, operator, value = self._parse_condition(condition) @@ -291,9 +356,9 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False # sequential rows1 = [] - opsseq = 0 + ops = 0 for ind, x in enumerate(column): - opsseq+=1 + ops += 1 if get_op(operator, x, value): rows1.append(ind) @@ -307,20 +372,92 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False # same as simple select from now on rows = rows[:k] # TODO: this needs to be dumbed down - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + dict = {(key): ([[self.data[i][j] for j in return_cols] for i in rows] + if key == "data" else value) for key, value in self.__dict__.items()} + + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + + s_table.data = list( + set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + + if order_by: + s_table.order_by(order_by, desc) + + if isinstance(limit, str): + s_table.data = [ + row for row in s_table.data if row is not None][:int(limit)] + + return s_table + + # select where hash + def _select_where_with_hash(self, return_columns, hash_map, condition, distinct=False, order_by=None, desc=True, limit=None): + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index( + colname) for colname in return_columns] + + column_name, operator, value = self._parse_condition(condition) + + # if column_name != self.column_names[self.pk_idx]: #DEN Yparxei pleon autos o periorismos + # print('Column is not PK. Aborting') + + rows = [] + if (operator == '<' or operator == '>'): + column = self.column_by_name(column_name) + + # sequential + + ops = 0 + for ind, x in enumerate(column): + ops += 1 + if get_op(operator, x, value): + rows.append(ind) + + else: + + # hash value + h_sum = 0 + for letter in value: + h_sum += ord(letter) + + h_index = h_sum % int(hash_map[0][0][0]) + + # find in dictionary with h_index + for item in hash_map[h_index]: + if hash_map[h_index][item][0] == hash_map[0][0][0]: + continue + + if hash_map[h_index][item][1] == value: + rows.append(hash_map[h_index][item][0]) + + try: + k = int(limit) + except TypeError: + k = None + # same as simple select from now on + rows = rows[:k] + # TODO: this needs to be dumbed down + dict = {(key): ([[self.data[i][j] for j in return_cols] for i in rows] + if key == "data" else value) for key, value in self.__dict__.items()} dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] s_table = Table(load=dict) - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + s_table.data = list( + set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data if order_by: s_table.order_by(order_by, desc) - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if row is not None][:int(limit)] + if isinstance(limit, str): + s_table.data = [ + row for row in s_table.data if row is not None][:int(limit)] return s_table @@ -332,14 +469,14 @@ def order_by(self, column_name, desc=True): column_name: string. Name of column. desc: boolean. If True, order_by will return results in descending order (False by default). ''' - column = [val if val is not None else 0 for val in self.column_by_name(column_name)] + column = [ + val if val is not None else 0 for val in self.column_by_name(column_name)] idx = sorted(range(len(column)), key=lambda k: column[k], reverse=desc) # print(idx) self.data = [self.data[i] for i in idx] # self._update() - - def _general_join_processing(self, table_right:Table, condition, join_type): + def _general_join_processing(self, table_right: Table, condition, join_type): ''' Performs the processes all the join operations need (regardless of type) so that there is no code repetition. @@ -347,42 +484,49 @@ def _general_join_processing(self, table_right:Table, condition, join_type): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' # get columns and operator - column_name_left, operator, column_name_right = self._parse_condition(condition, join=True) + column_name_left, operator, column_name_right = self._parse_condition( + condition, join=True) # try to find both columns, if you fail raise error - if(operator != '=' and join_type in ['left','right','full']): + if (operator != '=' and join_type in ['left', 'right', 'full']): class CustomFailException(Exception): pass - raise CustomFailException('Outer Joins can only be used if the condition operator is "=".\n') + raise CustomFailException( + 'Outer Joins can only be used if the condition operator is "=".\n') try: column_index_left = self.column_names.index(column_name_left) except: - raise Exception(f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') + raise Exception( + f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') try: - column_index_right = table_right.column_names.index(column_name_right) + column_index_right = table_right.column_names.index( + column_name_right) except: - raise Exception(f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') + raise Exception( + f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') # get the column names of both tables with the table name in front # ex. for left -> name becomes left_table_name_name etc - left_names = [f'{self._name}.{colname}' if self._name!='' else colname for colname in self.column_names] - right_names = [f'{table_right._name}.{colname}' if table_right._name!='' else colname for colname in table_right.column_names] + left_names = [f'{self._name}.{colname}' if self._name != + '' else colname for colname in self.column_names] + right_names = [f'{table_right._name}.{colname}' if table_right._name != + '' else colname for colname in table_right.column_names] # define the new tables name, its column names and types join_table_name = '' join_table_colnames = left_names+right_names join_table_coltypes = self.column_types+table_right.column_types - join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) + join_table = Table( + name=join_table_name, column_names=join_table_colnames, column_types=join_table_coltypes) return join_table, column_index_left, column_index_right, operator - def _inner_join(self, table_right: Table, condition): ''' Join table (left) with a supplied table (right) where condition is met. @@ -391,10 +535,11 @@ def _inner_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'inner') + join_table, column_index_left, column_index_right, operator = self._general_join_processing( + table_right, condition, 'inner') # count the number of operations (<,> etc) no_of_ops = 0 @@ -404,14 +549,14 @@ def _inner_join(self, table_right: Table, condition): left_value = row_left[column_index_left] for row_right in table_right.data: right_value = row_right[column_index_right] - if(left_value is None and right_value is None): + if (left_value is None and right_value is None): continue - no_of_ops+=1 - if get_op(operator, left_value, right_value): #EQ_OP + no_of_ops += 1 + if get_op(operator, left_value, right_value): # EQ_OP join_table._insert(row_left+row_right) return join_table - + def _left_join(self, table_right: Table, condition): ''' Perform a left join on the table with the supplied table (right). @@ -420,12 +565,14 @@ def _left_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'left') + join_table, column_index_left, column_index_right, operator = self._general_join_processing( + table_right, condition, 'left') - right_column = table_right.column_by_name(table_right.column_names[column_index_right]) + right_column = table_right.column_by_name( + table_right.column_names[column_index_right]) right_table_row_length = len(table_right.column_names) for row_left in self.data: @@ -450,10 +597,11 @@ def _right_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'right') + join_table, column_index_left, column_index_right, operator = self._general_join_processing( + table_right, condition, 'right') left_column = self.column_by_name(self.column_names[column_index_left]) left_table_row_length = len(self.column_names) @@ -471,7 +619,7 @@ def _right_join(self, table_right: Table, condition): join_table._insert(row_left + row_right) return join_table - + def _full_join(self, table_right: Table, condition): ''' Perform a full join on the table with the supplied table (right). @@ -480,17 +628,19 @@ def _full_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operators supported: (<,<=,==,>=,>) ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'full') + join_table, column_index_left, column_index_right, operator = self._general_join_processing( + table_right, condition, 'full') - right_column = table_right.column_by_name(table_right.column_names[column_index_right]) + right_column = table_right.column_by_name( + table_right.column_names[column_index_right]) left_column = self.column_by_name(self.column_names[column_index_left]) right_table_row_length = len(table_right.column_names) left_table_row_length = len(self.column_names) - + for row_left in self.data: left_value = row_left[column_index_left] if left_value is None: @@ -529,7 +679,8 @@ def show(self, no_of_rows=None, is_locked=False): output += f"\n## {self._name} ##\n" # headers -> "column name (column type)" - headers = [f'{col} ({tp.__name__})' for col, tp in zip(self.column_names, self.column_types)] + headers = [f'{col} ({tp.__name__})' for col, tp in zip( + self.column_names, self.column_types)] if self.pk_idx is not None: # table has a primary key, add PK next to the appropriate column headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' @@ -539,7 +690,6 @@ def show(self, no_of_rows=None, is_locked=False): # print using tabulate print(tabulate(non_none_rows[:no_of_rows], headers=headers)+'\n') - def _parse_condition(self, condition, join=False): ''' Parse the single string condition and return the value of the column and the operator. @@ -548,7 +698,7 @@ def _parse_condition(self, condition, join=False): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) join: boolean. Whether to join or not (False by default). ''' @@ -564,7 +714,6 @@ def _parse_condition(self, condition, join=False): return left, op, coltype(right) - def _load_from_file(self, filename): ''' Load table from a pkl file (not used currently).