Main

.gitattributes

@@ -0,0 +1,20 @@
+# Force LF line endings for all text files
+* text=auto eol=lf
+
+# Explicitly force LF for shell scripts (critical for Docker/Linux)
+*.sh text eol=lf
+
+# Force LF for other script types
+*.bash text eol=lf
+*.py text eol=lf
+
+# Let Git handle these as binary (never convert)
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.ico binary
+*.pdf binary
+*.zip binary
+*.tar.gz binary
+*.whl binary

.gitignore

@@ -8,6 +8,13 @@ build/*
 
 # Ignore tests build directory
 tests/build/
+tests/build*/
+
+# Test binaries
+tests/**/ndd_filter_test
+
+# macOS debug symbols
+*.dSYM/
 
 # Sometimes data files are created for tetsing
 data/
@@ -20,3 +27,22 @@ data/*
 # Frontend
 frontend/
 frontend/*
+
+# DS Store
+.DS_Store
+
+# Node
+node_modules/
+app/lov_frontend/node_modules/
+
+# Python
+app/.venv/
+__pycache__/
+*.pyc
+*.pyo
+
+# Docker persistent data volume (local only)
+endee-data/
+
+# Environment files (never commit real keys)
+app/.env

CMakeLists.txt

@@ -80,11 +80,25 @@ if(ND_DEBUG)
     add_definitions(-DND_DEBUG)
 endif()
 
+# Sparse instrumentation can be enabled independently from ND_DEBUG.
+option(ND_SPARSE_INSTRUMENT "Enable sparse index timing instrumentation" OFF)
+if(ND_SPARSE_INSTRUMENT)
+    add_definitions(-DND_SPARSE_INSTRUMENT)
+endif()
+
+# MDBX instrumentation can be enabled independently from ND_DEBUG.
+option(ND_MDBX_INSTRUMENT "Enable MDBX timing instrumentation" OFF)
+if(ND_MDBX_INSTRUMENT)
+    add_definitions(-DND_MDBX_INSTRUMENT)
+endif()
+
+
 # SIMD Optimization Options
 option(USE_AVX512 "Enable AVX512 (F, BW, VNNI, FP16)" OFF)
 option(USE_AVX2   "Enable AVX2 (FMA, F16C)" OFF)
 option(USE_SVE2   "Enable SVE2 (INT8/16, FP16)" OFF)
 option(USE_NEON   "Enable NEON (FP16, DotProd)" OFF)
+option(NDD_INV_IDX_STORE_FLOATS "Store raw float 32 values in sparse index (no quantization)" OFF)
 
 # Check if any SIMD option is selected
 if(NOT USE_AVX512 AND NOT USE_AVX2 AND NOT USE_SVE2 AND NOT USE_NEON)
@@ -236,17 +250,44 @@ endif()
 
 message(STATUS "Binary name: ${NDD_BINARY_NAME}")
 
+# Add new src/*.cpp files here when they should be compiled into ndd.
+set(NDD_CORE_SOURCES
+    src/sparse/inverted_index.cpp
+)
 
+# Build non-main project sources separately so they can be compiled in parallel
+# and linked into the final executable at the end.
+add_library(ndd_core OBJECT ${NDD_CORE_SOURCES})
 
-# Create the target
-add_executable(${NDD_BINARY_NAME} src/main.cpp ${LMDB_SOURCES} third_party/roaring_bitmap/roaring.c)
+# Create the final binary target.
+add_executable(${NDD_BINARY_NAME}
+    src/main.cpp
+    $<TARGET_OBJECTS:ndd_core>
+    ${LMDB_SOURCES}
+    third_party/roaring_bitmap/roaring.c
+)
 
 # Set MDBX-specific compile flags
 set_source_files_properties(${LMDB_SOURCES} PROPERTIES
     COMPILE_FLAGS "-DMDBX_BUILD_SHARED_LIBRARY=0 -DMDBX_BUILD_FLAGS=\\\"NDD_EMBEDDED\\\""
 )
 
 # Include directories
+target_include_directories(ndd_core PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/src
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/server
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/core
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/storage
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/utils
+    ${CMAKE_CURRENT_SOURCE_DIR}/third_party
+    ${CROW_INCLUDE_DIR}
+    ${MSGPACK_INCLUDE_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/roaring
+    ${LMDB_INCLUDE_DIR}
+    ${ASIO_INCLUDE_DIR}
+    ${OPENSSL_INCLUDE_DIR}
+    ${CURL_INCLUDE_DIRS}
+)
 target_include_directories(${NDD_BINARY_NAME} PRIVATE
     ${CMAKE_CURRENT_SOURCE_DIR}/src
     ${CMAKE_CURRENT_SOURCE_DIR}/src/server
@@ -265,33 +306,54 @@ target_include_directories(${NDD_BINARY_NAME} PRIVATE
 
 # Set compiler flags
 if(NOT DEBUG)
+    target_compile_options(ndd_core PRIVATE -O3 -ffast-math -fno-finite-math-only)
     target_compile_options(${NDD_BINARY_NAME} PRIVATE -O3 -ffast-math -fno-finite-math-only)
 endif()
 
 # Apply Flags based on selection
 if(USE_AVX512)
     message(STATUS "SIMD: AVX512 enabled (F, BW, VNNI, FP16)")
+    target_compile_options(ndd_core PRIVATE -mavx512f -mavx512bw -mavx512vnni -mavx512fp16 -mavx512vpopcntdq)
+    target_compile_definitions(ndd_core PRIVATE USE_AVX512)
     target_compile_options(${NDD_BINARY_NAME} PRIVATE -mavx512f -mavx512bw -mavx512vnni -mavx512fp16 -mavx512vpopcntdq)
     target_compile_definitions(${NDD_BINARY_NAME} PRIVATE USE_AVX512)
 elseif(USE_AVX2)
     message(STATUS "SIMD: AVX2 enabled")
+    target_compile_options(ndd_core PRIVATE -mavx2 -mfma -mf16c)
+    target_compile_definitions(ndd_core PRIVATE USE_AVX2)
     target_compile_options(${NDD_BINARY_NAME} PRIVATE -mavx2 -mfma -mf16c)
     target_compile_definitions(${NDD_BINARY_NAME} PRIVATE USE_AVX2)
 elseif(USE_SVE2)
-    message(STATUS "SIMD: SVE2 enabled (ARMv8.6-a + SVE2 + FP16)")
-    target_compile_options(${NDD_BINARY_NAME} PRIVATE -march=armv8.6-a+sve2+fp16)
+    message(STATUS "SIMD: SVE2 enabled (ARMv8.6-a + SVE2 + FP16 + DotProd)")
+    target_compile_options(ndd_core PRIVATE -march=armv8.6-a+sve2+fp16+dotprod)
+    target_compile_definitions(ndd_core PRIVATE USE_SVE2)
+    target_compile_options(${NDD_BINARY_NAME} PRIVATE -march=armv8.6-a+sve2+fp16+dotprod)
     target_compile_definitions(${NDD_BINARY_NAME} PRIVATE USE_SVE2)
 elseif(USE_NEON)
     message(STATUS "SIMD: NEON enabled")
     if(APPLE AND (CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64"))
+        target_compile_options(ndd_core PRIVATE -mcpu=native)
         target_compile_options(${NDD_BINARY_NAME} PRIVATE -mcpu=native)
     else()
+        target_compile_options(ndd_core PRIVATE -march=armv8.2-a+fp16+fp16fml+dotprod)
         target_compile_options(${NDD_BINARY_NAME} PRIVATE -march=armv8.2-a+fp16+fp16fml+dotprod)
     endif()
+    target_compile_definitions(ndd_core PRIVATE USE_NEON)
     target_compile_definitions(${NDD_BINARY_NAME} PRIVATE USE_NEON)
 endif()
 
+if(NDD_INV_IDX_STORE_FLOATS)
+    target_compile_definitions(ndd_core PRIVATE NDD_INV_IDX_STORE_FLOATS)
+    target_compile_definitions(${NDD_BINARY_NAME} PRIVATE NDD_INV_IDX_STORE_FLOATS)
+endif()
+
 # Add ASIO definitions
+target_compile_definitions(ndd_core PRIVATE
+    ASIO_STANDALONE
+    ASIO_HAS_STD_CHRONO
+    ASIO_HAS_STD_STRING_VIEW
+    MDB_MAXKEYSIZE=512
+)
 target_compile_definitions(${NDD_BINARY_NAME} PRIVATE 
     ASIO_STANDALONE 
     ASIO_HAS_STD_CHRONO 
@@ -311,9 +373,16 @@ target_link_libraries(${NDD_BINARY_NAME} PRIVATE
 # Installation rules
 install(TARGETS ${NDD_BINARY_NAME} RUNTIME DESTINATION bin)
 
-# Print configuration info
-message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
-message(STATUS "Debug mode: ${DEBUG}")
+
+# =======================
+# Testing
+# =======================
+option(ENABLE_TESTING "Enable building tests" OFF)
+if(ENABLE_TESTING)
+    enable_testing()
+    add_subdirectory(tests)
+endif()
+
 message(STATUS "Processor: ${CMAKE_SYSTEM_PROCESSOR}")
 if(USE_AVX512)
     message(STATUS "SIMD Mode: AVX512")
@@ -327,3 +396,11 @@ endif()
 message(STATUS "ASIO include dir: ${ASIO_INCLUDE_DIR}")
 message(STATUS "LMDB include dir: ${LMDB_INCLUDE_DIR}")
 message(STATUS "OpenSSL include dir: ${OPENSSL_INCLUDE_DIR}")
+
+# Create a symbolic link named 'ndd' pointing to the architecture-specific binary
+add_custom_command(TARGET ${NDD_BINARY_NAME} POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E create_symlink
+            $<TARGET_FILE_NAME:${NDD_BINARY_NAME}>
+            ${CMAKE_CURRENT_BINARY_DIR}/ndd
+    COMMENT "Creating softlink 'ndd' -> ${NDD_BINARY_NAME}"
+)

CONTRIBUTING.md

@@ -0,0 +1,29 @@
+# Contributing to Endee
+
+Thanks for your interest in contributing to Endee! We welcome community
+contributions and aim to keep the process simple and transparent.
+
+## License
+
+By contributing to this repository, you agree that your contributions will be
+licensed under the **Apache License 2.0**, the same license that governs the
+project.
+
+You confirm that:
+- You have the right to submit the contribution
+- The contribution does not knowingly infringe any third-party rights
+
+No additional contributor license agreement (CLA) is required.
+
+## How to Contribute
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes with clear commits
+4. Submit a pull request with a description of the change
+
+## Code of Conduct
+
+Be respectful and professional. Harassment or abusive behavior will not be
+tolerated.
+