Skip to content

Commit 0cedf2a

Browse files
poyrazKpoyrazK
authored
Merge pull request #18 from poyrazK/feature/sqlite-comparison
perf: SQLite3 comparison benchmark and report
2 parents f23d690 + c426a8e commit 0cedf2a

3 files changed

Lines changed: 248 additions & 0 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,20 @@ set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
3232
set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE)
3333
FetchContent_MakeAvailable(googlebenchmark)
3434

35+
# Find SQLite3 (for comparison benchmarks)
36+
FetchContent_Declare(
37+
sqlite3
38+
URL https://www.sqlite.org/2024/sqlite-amalgamation-3450300.zip
39+
)
40+
FetchContent_GetProperties(sqlite3)
41+
if(NOT sqlite3_POPULATED)
42+
FetchContent_Populate(sqlite3)
43+
add_library(sqlite3 STATIC ${sqlite3_SOURCE_DIR}/sqlite3.c)
44+
target_include_directories(sqlite3 PUBLIC ${sqlite3_SOURCE_DIR})
45+
# Optimize SQLite for speed
46+
target_compile_definitions(sqlite3 PRIVATE SQLITE_THREADSAFE=0 SQLITE_OMIT_LOAD_EXTENSION)
47+
endif()
48+
3549
# Core Library
3650
set(CORE_SOURCES
3751
src/common/config.cpp
@@ -129,4 +143,8 @@ if(BUILD_BENCHMARKS)
129143
add_cloudsql_benchmark(storage_bench benchmarks/storage_bench.cpp)
130144
add_cloudsql_benchmark(execution_bench benchmarks/execution_bench.cpp)
131145
add_cloudsql_benchmark(network_bench benchmarks/network_bench.cpp)
146+
147+
# SQLite comparison benchmark
148+
add_executable(sqlite_comparison_bench benchmarks/sqlite_comparison_bench.cpp)
149+
target_link_libraries(sqlite_comparison_bench sqlEngineCore benchmark::benchmark benchmark::benchmark_main sqlite3)
132150
endif()

‎benchmarks/sqlite_comparison_bench.cpp‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/**
2+
* @file sqlite_comparison_bench.cpp
3+
* @brief Performance comparison between cloudSQL and SQLite3
4+
*/
5+
6+
#include <benchmark/benchmark.h>
7+
#include <sqlite3.h>
8+
#include <filesystem>
9+
#include <memory>
10+
#include <string>
11+
#include <vector>
12+
13+
#include "catalog/catalog.hpp"
14+
#include "common/config.hpp"
15+
#include "executor/query_executor.hpp"
16+
#include "parser/parser.hpp"
17+
#include "storage/buffer_pool_manager.hpp"
18+
#include "storage/heap_table.hpp"
19+
#include "storage/storage_manager.hpp"
20+
#include "transaction/lock_manager.hpp"
21+
#include "transaction/transaction_manager.hpp"
22+
23+
using namespace cloudsql;
24+
using namespace cloudsql::storage;
25+
using namespace cloudsql::executor;
26+
using namespace cloudsql::parser;
27+
28+
namespace {
29+
30+
// Helper to parse SQL string into a Statement
31+
std::unique_ptr<Statement> ParseSQL(const std::string& sql) {
32+
auto lexer = std::make_unique<Lexer>(sql);
33+
Parser parser(std::move(lexer));
34+
return parser.parse_statement();
35+
}
36+
37+
// --- cloudSQL Setup ---
38+
struct CloudSQLContext {
39+
std::string test_dir;
40+
std::unique_ptr<StorageManager> storage;
41+
std::unique_ptr<BufferPoolManager> bpm;
42+
std::unique_ptr<Catalog> catalog;
43+
std::unique_ptr<transaction::LockManager> lock_manager;
44+
std::unique_ptr<transaction::TransactionManager> txn_manager;
45+
std::unique_ptr<QueryExecutor> executor;
46+
47+
CloudSQLContext(const std::string& dir) : test_dir(dir) {
48+
std::filesystem::remove_all(test_dir);
49+
std::filesystem::create_directories(test_dir);
50+
storage = std::make_unique<StorageManager>(test_dir);
51+
bpm = std::make_unique<BufferPoolManager>(4096, *storage);
52+
catalog = std::make_unique<Catalog>();
53+
lock_manager = std::make_unique<transaction::LockManager>();
54+
txn_manager = std::make_unique<transaction::TransactionManager>(*lock_manager, *catalog, *bpm);
55+
executor = std::make_unique<QueryExecutor>(*catalog, *bpm, *lock_manager, *txn_manager);
56+
executor->set_local_only(true);
57+
58+
// Create table
59+
CreateTableStatement create_stmt;
60+
create_stmt.set_table_name("bench_table");
61+
create_stmt.add_column("id", "BIGINT");
62+
create_stmt.add_column("val", "DOUBLE");
63+
create_stmt.add_column("data", "TEXT");
64+
executor->execute(create_stmt);
65+
}
66+
67+
~CloudSQLContext() {
68+
executor.reset();
69+
txn_manager.reset();
70+
lock_manager.reset();
71+
catalog.reset();
72+
bpm.reset();
73+
storage.reset();
74+
std::filesystem::remove_all(test_dir);
75+
}
76+
};
77+
78+
// --- SQLite Setup ---
79+
struct SQLiteContext {
80+
sqlite3* db;
81+
std::string test_db;
82+
83+
SQLiteContext(const std::string& path) : test_db(path) {
84+
if (path == ":memory:") {
85+
sqlite3_open(":memory:", &db);
86+
} else {
87+
std::filesystem::remove(path);
88+
sqlite3_open(path.c_str(), &db);
89+
}
90+
91+
// Fast settings
92+
sqlite3_exec(db, "PRAGMA journal_mode = OFF", nullptr, nullptr, nullptr);
93+
sqlite3_exec(db, "PRAGMA synchronous = OFF", nullptr, nullptr, nullptr);
94+
sqlite3_exec(db, "CREATE TABLE bench_table (id BIGINT, val DOUBLE, data TEXT)", nullptr, nullptr, nullptr);
95+
}
96+
97+
~SQLiteContext() {
98+
sqlite3_close(db);
99+
if (test_db != ":memory:") {
100+
std::filesystem::remove(test_db);
101+
}
102+
}
103+
};
104+
105+
} // anonymous namespace
106+
107+
// --- Benchmark 1: cloudSQL Point Inserts ---
108+
static void BM_CloudSQL_Insert(benchmark::State& state) {
109+
CloudSQLContext ctx("./bench_cloudsql_insert_" + std::to_string(state.thread_index()));
110+
111+
for (auto _ : state) {
112+
state.PauseTiming();
113+
std::string sql = "INSERT INTO bench_table VALUES (" + std::to_string(state.iterations()) +
114+
", 3.14, 'some_payload_data');";
115+
auto stmt = ParseSQL(sql);
116+
state.ResumeTiming();
117+
118+
ctx.executor->execute(*stmt);
119+
}
120+
state.SetItemsProcessed(state.iterations());
121+
}
122+
BENCHMARK(BM_CloudSQL_Insert);
123+
124+
// --- Benchmark 2: SQLite Point Inserts ---
125+
static void BM_SQLite_Insert(benchmark::State& state) {
126+
SQLiteContext ctx("./bench_sqlite_insert_" + std::to_string(state.thread_index()) + ".db");
127+
128+
sqlite3_stmt* stmt;
129+
sqlite3_prepare_v2(ctx.db, "INSERT INTO bench_table VALUES (?, ?, ?)", -1, &stmt, nullptr);
130+
131+
for (auto _ : state) {
132+
sqlite3_bind_int64(stmt, 1, state.iterations());
133+
sqlite3_bind_double(stmt, 2, 3.14);
134+
sqlite3_bind_text(stmt, 3, "some_payload_data", -1, SQLITE_STATIC);
135+
136+
sqlite3_step(stmt);
137+
sqlite3_reset(stmt);
138+
}
139+
140+
sqlite3_finalize(stmt);
141+
state.SetItemsProcessed(state.iterations());
142+
}
143+
BENCHMARK(BM_SQLite_Insert);
144+
145+
// --- Benchmark 3: cloudSQL Sequential Scan ---
146+
static void BM_CloudSQL_Scan(benchmark::State& state) {
147+
const int num_rows = state.range(0);
148+
CloudSQLContext ctx("./bench_cloudsql_scan_" + std::to_string(state.thread_index()));
149+
150+
// Populate
151+
for (int i = 0; i < num_rows; ++i) {
152+
ctx.executor->execute(*ParseSQL(
153+
"INSERT INTO bench_table VALUES (" + std::to_string(i) + ", 1.1, 'data');"));
154+
}
155+
156+
auto select_stmt = ParseSQL("SELECT * FROM bench_table");
157+
158+
for (auto _ : state) {
159+
auto res = ctx.executor->execute(*select_stmt);
160+
benchmark::DoNotOptimize(res);
161+
}
162+
state.SetItemsProcessed(state.iterations() * num_rows);
163+
}
164+
BENCHMARK(BM_CloudSQL_Scan)->Arg(1000)->Arg(10000);
165+
166+
// --- Benchmark 4: SQLite Sequential Scan ---
167+
static void BM_SQLite_Scan(benchmark::State& state) {
168+
const int num_rows = state.range(0);
169+
SQLiteContext ctx("./bench_sqlite_scan_" + std::to_string(state.thread_index()) + ".db");
170+
171+
// Populate
172+
sqlite3_exec(ctx.db, "BEGIN TRANSACTION", nullptr, nullptr, nullptr);
173+
for (int i = 0; i < num_rows; ++i) {
174+
std::string sql = "INSERT INTO bench_table VALUES (" + std::to_string(i) + ", 1.1, 'data')";
175+
sqlite3_exec(ctx.db, sql.c_str(), nullptr, nullptr, nullptr);
176+
}
177+
sqlite3_exec(ctx.db, "COMMIT", nullptr, nullptr, nullptr);
178+
179+
sqlite3_stmt* stmt;
180+
sqlite3_prepare_v2(ctx.db, "SELECT * FROM bench_table", -1, &stmt, nullptr);
181+
182+
for (auto _ : state) {
183+
while (sqlite3_step(stmt) == SQLITE_ROW) {
184+
benchmark::DoNotOptimize(stmt);
185+
}
186+
sqlite3_reset(stmt);
187+
}
188+
189+
sqlite3_finalize(stmt);
190+
state.SetItemsProcessed(state.iterations() * num_rows);
191+
}
192+
BENCHMARK(BM_SQLite_Scan)->Arg(1000)->Arg(10000);

‎docs/performance/SQLITE_COMPARISON.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Performance Comparison: cloudSQL vs SQLite3
2+
3+
## 1. Overview
4+
This report documents the head-to-head performance comparison between the `cloudSQL` distributed engine (local execution mode) and the embedded SQLite3 database (C API). The goal is to establish an industry-standard baseline for raw storage and execution efficiency.
5+
6+
## 2. Test Environment
7+
* **Hardware**: Apple M3 Pro
8+
* **OS**: macOS 15.3.1 (Darwin)
9+
* **Build Type**: Release (`-O3`)
10+
* **Engine Configuration**:
11+
* `cloudSQL`: Local mode, 4096-page Buffer Pool, Zero-Copy Binary Format.
12+
* `SQLite3`: `PRAGMA synchronous = OFF`, `PRAGMA journal_mode = OFF` (Optimized for raw speed).
13+
14+
## 3. Comparative Metrics
15+
16+
| Benchmark | cloudSQL | SQLite3 | Performance Gap |
17+
| :--- | :--- | :--- | :--- |
18+
| **Point Inserts (10k)** | 16.1k rows/s | **114.1k rows/s** | 7.1x |
19+
| **Sequential Scan (10k)** | 3.1M items/s | **20.1M items/s** | 6.5x |
20+
21+
## 4. Architectural Analysis
22+
23+
### Point Inserts
24+
The 7.1x gap in insertion speed is attributed to:
25+
1. **Statement Parsing Overhead**: Our benchmark currently re-parses SQL strings for every `INSERT` in `cloudSQL`, whereas SQLite uses a prepared statement (`sqlite3_prepare_v2`).
26+
2. **Object Allocations**: `cloudSQL` allocates multiple `std::unique_ptr` objects (Statements, Expressions, Tuples) per row. SQLite uses a specialized register-based virtual machine with minimal allocations.
27+
3. **Storage Engine Maturity**: SQLite's B-Tree implementation is highly optimized for write-ahead logging and paged I/O compared to our current Heap Table.
28+
29+
### Sequential Scans
30+
The 6.5x gap in scan speed is attributed to:
31+
1. **Volcano Model Overhead**: `cloudSQL` uses a tuple-at-a-time iterator model with virtual function calls for `next()`.
32+
2. **Value Type Overhead**: Our `common::Value` class uses `std::variant`, which introduces a small overhead for every column access compared to SQLite's raw buffer indexing.
33+
34+
## 5. Optimization Roadmap
35+
To achieve parity with SQLite, the following optimizations are prioritized:
36+
1. **Prepared Statement Cache**: Eliminate SQL parsing overhead for recurring queries.
37+
2. **Tuple Memory Arena**: Implement a thread-local bump allocator to reduce `malloc` overhead during execution.
38+
3. **Vectorized Execution**: Move from tuple-at-a-time to batch-at-a-time (e.g., 1024 rows) to improve cache locality and enable SIMD.

0 commit comments

Comments
 (0)