diff --git a/miniDB/database.py b/miniDB/database.py index ffd57a04..5fd2a983 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -2,7 +2,8 @@ import pickle from table import Table from time import sleep, localtime, strftime -import os,sys +import os +import sys from btree import Btree import shutil from misc import split_condition @@ -17,6 +18,7 @@ # Clear command cache (journal) readline.clear_history() + class Database: ''' Main Database class, containing tables. @@ -34,7 +36,8 @@ def __init__(self, name, load=True): logging.info(f'Loaded "{name}".') return except: - warnings.warn(f'Database "{name}" does not exist. Creating new.') + warnings.warn( + f'Database "{name}" does not exist. Creating new.') # create dbdata directory if it doesnt exist if not os.path.exists('dbdata'): @@ -49,7 +52,8 @@ def __init__(self, name, load=True): # create all the meta tables self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') - self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') + self.create_table('meta_insert_stack', + 'table_name,indexes', 'str,list') self.create_table('meta_indexes', 'table_name,index_name', 'str,str') self.save_database() @@ -78,7 +82,7 @@ def load_database(self): path = f'dbdata/{self._name}_db' for file in os.listdir(path): - if file[-3:]!='pkl': # if used to load only pkl files + if file[-3:] != 'pkl': # if used to load only pkl files continue f = open(path+'/'+file, 'rb') tmp_dict = pickle.load(f) @@ -96,7 +100,6 @@ def _update(self): self._update_meta_length() self._update_meta_insert_stack() - def create_table(self, name, column_names, column_types, primary_key=None, load=None): ''' This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name @@ -109,7 +112,8 @@ def create_table(self, name, column_names, column_types, primary_key=None, load= load: boolean. Defines table object parameters as the name of the table and the column names. ''' # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) + self.tables.update({name: Table(name=name, column_names=column_names.split( + ','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) # check that new dynamic var doesnt exist already # self.no_of_tables += 1 @@ -118,7 +122,6 @@ def create_table(self, name, column_names, column_types, primary_key=None, load= # (self.tables[name]) print(f'Created table "{name}".') - def drop_table(self, table_name): ''' Drop table from current database. @@ -141,7 +144,6 @@ def drop_table(self, table_name): # self._update() self.save_database() - def import_table(self, table_name, filename, column_types=None, primary_key=None): ''' Creates table from CSV file. @@ -153,24 +155,25 @@ def import_table(self, table_name, filename, column_types=None, primary_key=None ''' file = open(filename, 'r') - first_line=True + first_line = True for line in file.readlines(): if first_line: colnames = line.strip('\n') if column_types is None: - column_types = ",".join(['str' for _ in colnames.split(',')]) - self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key) + column_types = ",".join( + ['str' for _ in colnames.split(',')]) + self.create_table(name=table_name, column_names=colnames, + column_types=column_types, primary_key=primary_key) lock_ownership = self.lock_table(table_name, mode='x') first_line = False continue self.tables[table_name]._insert(line.strip('\n').split(',')) if lock_ownership: - self.unlock_table(table_name) + self.unlock_table(table_name) self._update() self.save_database() - def export(self, table_name, filename=None): ''' Transform table to CSV. @@ -181,13 +184,14 @@ def export(self, table_name, filename=None): ''' res = '' for row in [self.tables[table_name].column_names]+self.tables[table_name].data: - res+=str(row)[1:-1].replace('\'', '').replace('"','').replace(' ','')+'\n' + res += str(row)[1:-1].replace('\'', + '').replace('"', '').replace(' ', '')+'\n' if filename is None: filename = f'{table_name}.csv' with open(filename, 'w') as file: - file.write(res) + file.write(res) def table_from_object(self, new_table): ''' @@ -201,12 +205,11 @@ def table_from_object(self, new_table): if new_table._name not in self.__dir__(): setattr(self, new_table._name, new_table) else: - raise Exception(f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') + raise Exception( + f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') self._update() self.save_database() - - ##### table functions ##### # In every table function a load command is executed to fetch the most recent table. @@ -228,7 +231,7 @@ def cast(self, column_name, table_name, cast_type): cast_type: type. Cast type (do not encapsulate in quotes). ''' self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') self.tables[table_name]._cast_column(column_name, eval(cast_type)) if lock_ownership: @@ -236,7 +239,7 @@ def cast(self, column_name, table_name, cast_type): self._update() self.save_database() - def insert_into(self, table_name, row_str): + def insert_into(self, table_name, row_str,columns=None,table_nam=None,condition=None): ''' Inserts data to given table. @@ -245,24 +248,48 @@ def insert_into(self, table_name, row_str): row: list. A list of values to be inserted (will be casted to a predifined type automatically). lock_load_save: boolean. If False, user needs to load, lock and save the states of the database (CAUTION). Useful for bulk-loading. ''' - row = row_str.strip().split(',') - self.load_database() - # fetch the insert_stack. For more info on the insert_stack - # check the insert_stack meta table - lock_ownership = self.lock_table(table_name, mode='x') - insert_stack = self._get_insert_stack_for_table(table_name) - try: - self.tables[table_name]._insert(row, insert_stack) - except Exception as e: - logging.info(e) - logging.info('ABORTED') - self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) - - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - + if table_nam is not None: + if isinstance(table_nam, Table): + table = table_nam._select_where(columns,condition) + else : + table = self.tables[table_nam]._select_where(columns,condition) + + if columns is None and row_str is not None: + row = row_str.strip().split(',') + self.load_database() + # fetch the insert_stack. For more info on the insert_stack + # check the insert_stack meta table + lock_ownership = self.lock_table(table_name, mode='x') + insert_stack = self._get_insert_stack_for_table(table_name) + try: + self.tables[table_name]._insert(row, insert_stack) + except Exception as e: + logging.info(e) + logging.info('ABORTED') + self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) + + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + elif columns is not None: + for row in table.data: + self.load_database() + # fetch the insert_stack. For more info on the insert_stack + # check the insert_stack meta table + lock_ownership = self.lock_table(table_name, mode='x') + insert_stack = self._get_insert_stack_for_table(table_name) + try: + self.tables[table_name]._insert(row, insert_stack) + except Exception as e: + logging.info(e) + logging.info('ABORTED') + self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) + + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() def update_table(self, table_name, set_args, condition): ''' @@ -275,12 +302,12 @@ def update_table(self, table_name, set_args, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) ''' - set_column, set_value = set_args.replace(' ','').split('=') + set_column, set_value = set_args.replace(' ', '').split('=') self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') self.tables[table_name]._update_rows(set_value, set_column, condition) if lock_ownership: @@ -297,11 +324,11 @@ def delete_from(self, table_name, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) ''' self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') deleted = self.tables[table_name]._delete_where(condition) if lock_ownership: @@ -309,11 +336,11 @@ def delete_from(self, table_name, condition): self._update() self.save_database() # we need the save above to avoid loading the old database that still contains the deleted elements - if table_name[:4]!='meta': + if table_name[:4] != 'meta': self._add_to_insert_stack(table_name, deleted) self.save_database() - def select(self, columns, table_name, condition, order_by=None, top_k=True,\ + def select(self, columns, table_name, condition, in_args=None, between=None, like=None, order_by=None, top_k=True, desc=None, save_as=None, return_object=True): ''' Selects and outputs a table's data where condtion is met. @@ -324,7 +351,7 @@ def select(self, columns, table_name, condition, order_by=None, top_k=True,\ condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). desc: boolean. If True, order_by will return results in descending order (True by default). @@ -332,26 +359,36 @@ def select(self, columns, table_name, condition, order_by=None, top_k=True,\ save_as: string. The name that will be used to save the resulting table into the database (no save if None). return_object: boolean. If True, the result will be a table object (useful for internal use - the result will be printed by default). ''' - # print(table_name) + self.load_database() - if isinstance(table_name,Table): - return table_name._select_where(columns, condition, order_by, desc, top_k) + if isinstance(table_name, Table): + return table_name._select_where(columns, condition, in_args, between, like, order_by, desc, top_k) if condition is not None: - condition_column = split_condition(condition)[0] + if in_args is not None: + condition_column = condition + in_args = in_args.strip('(').strip(')').split(',') + elif between is not None: + condition_column = condition + elif like is not None: + condition_column = condition + else: + condition_column = split_condition(condition)[0] else: condition_column = '' - # self.lock_table(table_name, mode='x') if self.is_locked(table_name): return - if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] + if self._has_index(table_name) and condition_column == self.tables[table_name].column_names[self.tables[table_name].pk_idx]: + index_name = self.select( + '*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, order_by, desc, top_k) + table = self.tables[table_name]._select_where_with_btree( + columns, bt, condition, in_args, between, like, order_by, desc, top_k) else: - table = self.tables[table_name]._select_where(columns, condition, order_by, desc, top_k) + table = self.tables[table_name]._select_where( + columns, condition, in_args, between, like, order_by, desc, top_k) # self.unlock_table(table_name) if save_as is not None: table._name = save_as @@ -362,7 +399,6 @@ def select(self, columns, table_name, condition, order_by=None, top_k=True,\ else: return table.show() - def show_table(self, table_name, no_of_rows=None): ''' Print table in a readable tabular design (using tabulate). @@ -371,9 +407,8 @@ def show_table(self, table_name, no_of_rows=None): table_name: string. Name of table (must be part of database). ''' self.load_database() - - self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) + self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) def sort(self, table_name, column_name, asc=False): ''' @@ -386,7 +421,7 @@ def sort(self, table_name, column_name, asc=False): ''' self.load_database() - + lock_ownership = self.lock_table(table_name, mode='x') self.tables[table_name]._sort(column_name, asc=asc) if lock_ownership: @@ -404,7 +439,7 @@ def join(self, mode, left_table, right_table, condition, save_as=None, return_ob condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) save_as: string. The output filename that will be used to save the resulting table in the database (won't save if None). return_object: boolean. If True, the result will be a table object (useful for internal usage - the result will be printed by default). @@ -413,11 +448,12 @@ def join(self, mode, left_table, right_table, condition, save_as=None, return_ob if self.is_locked(left_table) or self.is_locked(right_table): return - left_table = left_table if isinstance(left_table, Table) else self.tables[left_table] - right_table = right_table if isinstance(right_table, Table) else self.tables[right_table] - + left_table = left_table if isinstance( + left_table, Table) else self.tables[left_table] + right_table = right_table if isinstance( + right_table, Table) else self.tables[right_table] - if mode=='inner': + if mode == 'inner': res = left_table._inner_join(right_table, condition) else: raise NotImplementedError @@ -438,23 +474,25 @@ def lock_table(self, table_name, mode='x'): Args: table_name: string. Table name (must be part of database). ''' - if table_name[:4]=='meta' or table_name not in self.tables.keys() or isinstance(table_name,Table): + if table_name[:4] == 'meta' or table_name not in self.tables.keys() or isinstance(table_name, Table): return with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: self.tables.update({'meta_locks': pickle.load(f)}) try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by process with pid={pid}') + pid = self.tables['meta_locks']._select_where( + 'pid', f'table_name={table_name}').data[0][0] + if pid != os.getpid(): + raise Exception( + f'Table "{table_name}" is locked by process with pid={pid}') else: return False except IndexError: pass - if mode=='x': + if mode == 'x': self.tables['meta_locks']._insert([table_name, os.getpid(), mode]) else: raise NotImplementedError @@ -475,9 +513,11 @@ def unlock_table(self, table_name, force=False): if not force: try: # pid = self.select('*','meta_locks', f'table_name={table_name}', return_object=True).data[0][1] - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + pid = self.tables['meta_locks']._select_where( + 'pid', f'table_name={table_name}').data[0][0] + if pid != os.getpid(): + raise Exception( + f'Table "{table_name}" is locked by the process with pid={pid}') except IndexError: pass self.tables['meta_locks']._delete_where(f'table_name={table_name}') @@ -491,31 +531,36 @@ def is_locked(self, table_name): Args: table_name: string. Table name (must be part of database). ''' - if isinstance(table_name,Table) or table_name[:4]=='meta': # meta tables will never be locked (they are internal) + if isinstance(table_name, Table) or table_name[:4] == 'meta': # meta tables will never be locked (they are internal) return False with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: self.tables.update({'meta_locks': pickle.load(f)}) try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + pid = self.tables['meta_locks']._select_where( + 'pid', f'table_name={table_name}').data[0][0] + if pid != os.getpid(): + raise Exception( + f'Table "{table_name}" is locked by the process with pid={pid}') except IndexError: pass return False - def journal(idx = None): + def journal(idx=None): if idx != None: - cache_list = '\n'.join([str(readline.get_history_item(i + 1)) for i in range(readline.get_current_history_length())]).split('\n')[int(idx)] - out = tabulate({"Command": cache_list.split('\n')}, headers=["Command"]) + cache_list = '\n'.join([str(readline.get_history_item( + i + 1)) for i in range(readline.get_current_history_length())]).split('\n')[int(idx)] + out = tabulate({"Command": cache_list.split('\n')}, + headers=["Command"]) else: - cache_list = '\n'.join([str(readline.get_history_item(i + 1)) for i in range(readline.get_current_history_length())]) - out = tabulate({"Command": cache_list.split('\n')}, headers=["Index","Command"], showindex="always") + cache_list = '\n'.join([str(readline.get_history_item( + i + 1)) for i in range(readline.get_current_history_length())]) + out = tabulate({"Command": cache_list.split('\n')}, headers=[ + "Index", "Command"], showindex="always") print('journal:', out) - #return out - + # return out #### META #### @@ -528,15 +573,17 @@ def _update_meta_length(self): Updates the meta_length table. ''' for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables + if table._name[:4] == 'meta': # skip meta tables continue - if table._name not in self.tables['meta_length'].column_by_name('table_name'): # if new table, add record with 0 no. of rows + # if new table, add record with 0 no. of rows + if table._name not in self.tables['meta_length'].column_by_name('table_name'): self.tables['meta_length']._insert([table._name, 0]) # the result needs to represent the rows that contain data. Since we use an insert_stack # some rows are filled with Nones. We skip these rows. non_none_rows = len([row for row in table.data if any(row)]) - self.tables['meta_length']._update_rows(non_none_rows, 'no_of_rows', f'table_name={table._name}') + self.tables['meta_length']._update_rows( + non_none_rows, 'no_of_rows', f'table_name={table._name}') # self.update_row('meta_length', len(table.data), 'no_of_rows', 'table_name', '==', table._name) def _update_meta_locks(self): @@ -544,7 +591,7 @@ def _update_meta_locks(self): Updates the meta_locks table. ''' for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables + if table._name[:4] == 'meta': # skip meta tables continue if table._name not in self.tables['meta_locks'].column_by_name('table_name'): @@ -556,12 +603,11 @@ def _update_meta_insert_stack(self): Updates the meta_insert_stack table. ''' for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables + if table._name[:4] == 'meta': # skip meta tables continue if table._name not in self.tables['meta_insert_stack'].column_by_name('table_name'): self.tables['meta_insert_stack']._insert([table._name, []]) - def _add_to_insert_stack(self, table_name, indexes): ''' Adds provided indices to the insert stack of the specified table. @@ -592,10 +638,11 @@ def _update_meta_insert_stack_for_tb(self, table_name, new_stack): table_name: string. Table name (must be part of database). new_stack: string. The stack that will be used to replace the existing one. ''' - self.tables['meta_insert_stack']._update_rows(new_stack, 'indexes', f'table_name={table_name}') - + self.tables['meta_insert_stack']._update_rows( + new_stack, 'indexes', f'table_name={table_name}') # indexes + def create_index(self, index_name, table_name, index_type='btree'): ''' Creates an index on a specified table with a given name. @@ -605,11 +652,11 @@ def create_index(self, index_name, table_name, index_type='btree'): table_name: string. Table name (must be part of database). index_name: string. Name of the created index. ''' - if self.tables[table_name].pk_idx is None: # if no primary key, no index + if self.tables[table_name].pk_idx is None: # if no primary key, no index raise Exception('Cannot create index. Table has no primary key.') if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): # currently only btree is supported. This can be changed by adding another if. - if index_type=='btree': + if index_type == 'btree': logging.info('Creating Btree index.') # insert a record with the name of the index and the table on which it's created to the meta_indexes table self.tables['meta_indexes']._insert([table_name, index_name]) @@ -617,7 +664,8 @@ def create_index(self, index_name, table_name, index_type='btree'): self._construct_index(table_name, index_name) self.save_database() else: - raise Exception('Cannot create index. Another index with the same name already exists.') + raise Exception( + 'Cannot create index. Another index with the same name already exists.') def _construct_index(self, table_name, index_name): ''' @@ -627,7 +675,7 @@ def _construct_index(self, table_name, index_name): table_name: string. Table name (must be part of database). index_name: string. Name of the created index. ''' - bt = Btree(3) # 3 is arbitrary + bt = Btree(3) # 3 is arbitrary # for each record in the primary key of the table, insert its value and index to the btree for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): @@ -635,7 +683,6 @@ def _construct_index(self, table_name, index_name): # save the btree self._save_index(index_name, bt) - def _has_index(self, table_name): ''' Check whether the specified table's primary key column is indexed. @@ -671,4 +718,4 @@ def _load_idx(self, index_name): f = open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'rb') index = pickle.load(f) f.close() - return index \ No newline at end of file + return index diff --git a/miniDB/table.py b/miniDB/table.py index 7bde8602..7d2b9693 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -1,8 +1,10 @@ from __future__ import annotations +from tkinter import N from tabulate import tabulate import pickle import os from misc import get_op, split_condition +import re class Table: @@ -22,6 +24,7 @@ class Table: - a dictionary that includes the appropriate info (all the attributes in __init__) ''' + def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): if load is not None: @@ -38,7 +41,7 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= self._name = name - if len(column_names)!=len(column_types): + if len(column_names) != len(column_types): raise ValueError('Need same number of column names and types.') self.column_names = column_names @@ -52,10 +55,12 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= setattr(self, col, []) self.columns.append([]) else: - raise Exception(f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') + raise Exception( + f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') - self.column_types = [eval(ct) if not isinstance(ct, type) else ct for ct in column_types] - self.data = [] # data is a list of lists, a list of rows that is. + self.column_types = [eval(ct) if not isinstance( + ct, type) else ct for ct in column_types] + self.data = [] # data is a list of lists, a list of rows that is. # if primary key is set, keep its index as an attribute if primary_key is not None: @@ -71,12 +76,12 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= def column_by_name(self, column_name): return [row[self.column_names.index(column_name)] for row in self.data] - def _update(self): ''' Update all the available columns with the appended rows. ''' - self.columns = [[row[i] for row in self.data] for i in range(len(self.column_names))] + self.columns = [[row[i] for row in self.data] + for i in range(len(self.column_names))] for ind, col in enumerate(self.column_names): setattr(self, col, self.columns[ind]) @@ -97,7 +102,6 @@ def _cast_column(self, column_name, cast_type): self.column_types[column_idx] = cast_type # self._update() - def _insert(self, row, insert_stack=[]): ''' Insert row to table. @@ -106,8 +110,9 @@ def _insert(self, row, insert_stack=[]): row: list. A list of values to be inserted (will be casted to a predifined type automatically). insert_stack: list. The insert stack (empty by default). ''' - if len(row)!=len(self.column_names): - raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') + if len(row) != len(self.column_names): + raise ValueError( + f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') for i in range(len(row)): # for each value, cast and replace it in row. @@ -117,13 +122,14 @@ def _insert(self, row, insert_stack=[]): # raise ValueError(f'ERROR -> Value {row[i]} of type {type(row[i])} is not of type {self.column_types[i]}.') # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) - if i==self.pk_idx and row[i] in self.column_by_name(self.pk): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') + if i == self.pk_idx and row[i] in self.column_by_name(self.pk): + raise ValueError( + f'## ERROR -> Value {row[i]} already exists in primary key column.') # if insert_stack is not empty, append to its last index if insert_stack != []: self.data[insert_stack[-1]] = row - else: # else append to the end + else: # else append to the end self.data.append(row) # self._update() @@ -137,7 +143,7 @@ def _update_rows(self, set_value, set_column, condition): condition: string. A condition using the following format: 'column[<,<=,=,>=,>]value' or 'value[<,<=,=,>=,>]column'. - + Operatores supported: (<,<=,=,>=,>) ''' # parse the condition @@ -157,8 +163,7 @@ def _update_rows(self, set_value, set_column, condition): # self._update() # print(f"Updated {len(indexes_to_del)} rows") - - def _delete_where(self, condition): + def _delete_where(self, condition,in_args=None,between=None,like=None): ''' Deletes rows where condition is met. @@ -169,7 +174,7 @@ def _delete_where(self, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) ''' column_name, operator, value = self._parse_condition(condition) @@ -188,7 +193,8 @@ def _delete_where(self, condition): for index in sorted(indexes_to_del, reverse=True): if self._name[:4] != 'meta': # if the table is not a metatable, replace the row with a row of nones - self.data[index] = [None for _ in range(len(self.column_names))] + self.data[index] = [ + None for _ in range(len(self.column_names))] else: self.data.pop(index) @@ -196,8 +202,7 @@ def _delete_where(self, condition): # we have to return the deleted indexes, since they will be appended to the insert_stack return indexes_to_del - - def _select_where(self, return_columns, condition=None, order_by=None, desc=True, top_k=None): + def _select_where(self, return_columns, condition=None, in_args=None, between=None, like=None, order_by=None, desc=True, top_k=None): ''' Select and return a table containing specified columns and rows where condition is met. @@ -206,7 +211,7 @@ def _select_where(self, return_columns, condition=None, order_by=None, desc=True condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). desc: boolean. If True, order_by will return results in descending order (False by default). @@ -217,46 +222,71 @@ def _select_where(self, return_columns, condition=None, order_by=None, desc=True if return_columns == '*': return_cols = [i for i in range(len(self.column_names))] else: - return_cols = [self.column_names.index(col.strip()) for col in return_columns.split(',')] + return_cols = [self.column_names.index( + col.strip()) for col in return_columns.split(',')] # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + if in_args is not None: + rows = [] + column = self.column_by_name(condition) + for value in in_args: + _, _, valu = self._parse_condition(condition+"="+value) + rows += [ind for ind, + x in enumerate(column) if get_op('=', x, valu)] + elif between is not None: + column = self.column_by_name(condition) + lower, upper = [between.split('and')[0].strip(),between.split('and')[1].strip()] + rows = [ind for ind, x in enumerate(column) if get_op('>=', x, self.column_types[self.column_names.index( + condition)](lower)) and get_op('<=', x, self.column_types[self.column_names.index(condition)](upper))] + elif like is not None: + column = self.column_by_name(condition) + like = like.replace('%', '.*').replace('_', '.').strip() + rows = [ind for ind, x in enumerate( + column) if re.findall(like, x)] + else: + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate( + column) if get_op(operator, x, value)] else: rows = [i for i in range(len(self.data))] # top k rows # rows = rows[:int(top_k)] if isinstance(top_k,str) else rows # copy the old dict, but only the rows and columns of data with index in rows/columns (the indexes that we want returned) - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + dict = {(key): ([[self.data[i][j] for j in return_cols] for i in rows] + if key == "data" else value) for key, value in self.__dict__.items()} # we need to set the new column names/types and no of columns, since we might # only return some columns dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] - s_table = Table(load=dict) + s_table = Table(load=dict) if order_by: s_table.order_by(order_by, desc) - s_table.data = s_table.data[:int(top_k)] if isinstance(top_k,str) else s_table.data + s_table.data = s_table.data[:int(top_k)] if isinstance( + top_k, str) else s_table.data return s_table - - def _select_where_with_btree(self, return_columns, bt, condition, order_by=None, desc=True, top_k=None): + def _select_where_with_btree(self, return_columns, bt, condition, in_args=None, order_by=None, desc=True, top_k=None): # if * return all columns, else find the column indexes for the columns specified if return_columns == '*': return_cols = [i for i in range(len(self.column_names))] else: - return_cols = [self.column_names.index(colname) for colname in return_columns] - + return_cols = [self.column_names.index( + colname) for colname in return_columns] - column_name, operator, value = self._parse_condition(condition) + if in_args is not None: + column_name = condition + operator = "=" + else: + column_name, operator, value = self._parse_condition(condition) # if the column in condition is not a primary key, abort the select if column_name != self.column_names[self.pk_idx]: @@ -269,27 +299,41 @@ def _select_where_with_btree(self, return_columns, bt, condition, order_by=None, # sequential rows1 = [] opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) + if in_args is not None: + for value in in_args: + for ind, x in enumerate(column): + opsseq += 1 + if get_op(operator, x, value): + rows1.append(ind) + else: + for ind, x in enumerate(column): + opsseq += 1 + if get_op(operator, x, value): + rows1.append(ind) # btree find - rows = bt.find(operator, value) + if in_args is not None: + rows = [] + for value in in_args: + rows += bt.find(operator, value) + else: + rows = bt.find(operator, value) # same as simple select from now on rows = rows[:top_k] # TODO: this needs to be dumbed down - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + dict = {(key): ([[self.data[i][j] for j in return_cols] for i in rows] + if key == "data" else value) for key, value in self.__dict__.items()} dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] - s_table = Table(load=dict) + s_table = Table(load=dict) if order_by: s_table.order_by(order_by, desc) - s_table.data = s_table.data[:int(top_k)] if isinstance(top_k,str) else s_table.data + s_table.data = s_table.data[:int(top_k)] if isinstance( + top_k, str) else s_table.data return s_table @@ -307,7 +351,6 @@ def order_by(self, column_name, desc=True): self.data = [self.data[i] for i in idx] # self._update() - def _inner_join(self, table_right: Table, condition): ''' Join table (left) with a supplied table (right) where condition is met. @@ -316,32 +359,39 @@ def _inner_join(self, table_right: Table, condition): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) ''' # get columns and operator - column_name_left, operator, column_name_right = self._parse_condition(condition, join=True) + column_name_left, operator, column_name_right = self._parse_condition( + condition, join=True) # try to find both columns, if you fail raise error try: column_index_left = self.column_names.index(column_name_left) except: - raise Exception(f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') + raise Exception( + f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') try: - column_index_right = table_right.column_names.index(column_name_right) + column_index_right = table_right.column_names.index( + column_name_right) except: - raise Exception(f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') + raise Exception( + f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') # get the column names of both tables with the table name in front # ex. for left -> name becomes left_table_name_name etc - left_names = [f'{self._name}.{colname}' if self._name!='' else colname for colname in self.column_names] - right_names = [f'{table_right._name}.{colname}' if table_right._name!='' else colname for colname in table_right.column_names] + left_names = [f'{self._name}.{colname}' if self._name != + '' else colname for colname in self.column_names] + right_names = [f'{table_right._name}.{colname}' if table_right._name != + '' else colname for colname in table_right.column_names] # define the new tables name, its column names and types join_table_name = '' join_table_colnames = left_names+right_names join_table_coltypes = self.column_types+table_right.column_types - join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) + join_table = Table( + name=join_table_name, column_names=join_table_colnames, column_types=join_table_coltypes) # count the number of operations (<,> etc) no_of_ops = 0 @@ -351,13 +401,12 @@ def _inner_join(self, table_right: Table, condition): left_value = row_left[column_index_left] for row_right in table_right.data: right_value = row_right[column_index_right] - no_of_ops+=1 - if get_op(operator, left_value, right_value): #EQ_OP + no_of_ops += 1 + if get_op(operator, left_value, right_value): # EQ_OP join_table._insert(row_left+row_right) return join_table - def show(self, no_of_rows=None, is_locked=False): ''' Print the table in a nice readable format. @@ -374,7 +423,8 @@ def show(self, no_of_rows=None, is_locked=False): output += f"\n## {self._name} ##\n" # headers -> "column name (column type)" - headers = [f'{col} ({tp.__name__})' for col, tp in zip(self.column_names, self.column_types)] + headers = [f'{col} ({tp.__name__})' for col, tp in zip( + self.column_names, self.column_types)] if self.pk_idx is not None: # table has a primary key, add PK next to the appropriate column headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' @@ -384,7 +434,6 @@ def show(self, no_of_rows=None, is_locked=False): # print using tabulate print(tabulate(non_none_rows[:no_of_rows], headers=headers)+'\n') - def _parse_condition(self, condition, join=False): ''' Parse the single string condition and return the value of the column and the operator. @@ -393,7 +442,7 @@ def _parse_condition(self, condition, join=False): condition: string. A condition using the following format: 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - + Operatores supported: (<,<=,==,>=,>) join: boolean. Whether to join or not (False by default). ''' @@ -409,7 +458,6 @@ def _parse_condition(self, condition, join=False): return left, op, coltype(right) - def _load_from_file(self, filename): ''' Load table from a pkl file (not used currently).