fix: clean up metatable on row filter change and add edge-case test coverage (#38)

andinux · web-flow · commit b36ff575f957 · 2026-04-08T15:39:19.000-06:00
Previously, cloudsync_set_filter and cloudsync_clear_filter only updated
the filter setting and recreated triggers — they did not touch the
metatable. This left stale metadata for rows that no longer matched the
active filter, causing non-matching rows to sync to other nodes.

Now both functions DELETE all rows from the metatable and call
cloudsync_refill_metatable after changing the filter. This ensures the
metatable only contains entries for rows matching the current filter
(or all rows when the filter is cleared).

Core changes:

Add cloudsync_reset_metatable() which deletes metatable contents and
refills from scratch using the current filter setting
Add SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE constant with platform-specific
format specifiers (%w for SQLite, %s for PostgreSQL) to match how each
platform formats the meta_ref identifier
Call cloudsync_reset_metatable from set_filter/clear_filter in both
the SQLite and PostgreSQL implementations
Test coverage:

Add row filter edge-case tests: clear_filter lifecycle, complex filter
expressions (AND, IS NULL), row entering/exiting filter via UPDATE,
filter change after data, composite PK with multi-column filters,
multi-table roundtrip sync, and pre-existing data prefill
Add corresponding PostgreSQL test files (47, 48, 49)
Update test expectations to validate that the metatable reflects the
latest filter after set_filter and clear_filter calls
diff --git a/src/cloudsync.c b/src/cloudsync.c
@@ -2528,6 +2528,18 @@ char *cloudsync_filter_add_row_prefix (const char *filter, const char *prefix, c
     return result;
 }
 
+int cloudsync_reset_metatable (cloudsync_context *data, const char *table_name) {
+    cloudsync_table_context *table = table_lookup(data, table_name);
+    if (!table) return DBRES_ERROR;
+
+    char *sql = cloudsync_memory_mprintf(SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE, table->meta_ref);
+    int rc = database_exec(data, sql);
+    cloudsync_memory_free(sql);
+    if (rc != DBRES_OK) return rc;
+
+    return cloudsync_refill_metatable(data, table_name);
+}
+
 int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name) {
     cloudsync_table_context *table = table_lookup(data, table_name);
     if (!table) return DBRES_ERROR;
diff --git a/src/cloudsync.h b/src/cloudsync.h
@@ -18,7 +18,7 @@
 extern "C" {
 #endif
 
-#define CLOUDSYNC_VERSION                       "1.0.9"
+#define CLOUDSYNC_VERSION                       "1.0.10"
 #define CLOUDSYNC_MAX_TABLENAME_LEN             512
 
 #define CLOUDSYNC_VALUE_NOTSET                  -1
@@ -95,6 +95,8 @@ int    cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_si
 int    cloudsync_payload_save (cloudsync_context *data, const char *payload_path, int *blob_size); // available only on Desktop OS (no WASM, no mobile)
 
 // CloudSync table context
+int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name);
+int cloudsync_reset_metatable (cloudsync_context *data, const char *table_name);
 cloudsync_table_context *table_lookup (cloudsync_context *data, const char *table_name);
 void *table_column_lookup (cloudsync_table_context *table, const char *col_name, bool is_merge, int *index);
 bool table_enabled (cloudsync_table_context *table);
diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c
@@ -695,6 +695,13 @@ Datum cloudsync_set_filter (PG_FUNCTION_ARGS) {
             ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
                             errmsg("cloudsync_set_filter: error recreating triggers")));
         }
+
+        // Clean and refill metatable with the new filter
+        rc = cloudsync_reset_metatable(data, tbl);
+        if (rc != DBRES_OK) {
+            ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
+                            errmsg("cloudsync_set_filter: error resetting metatable")));
+        }
     }
     PG_CATCH();
     {
@@ -753,6 +760,13 @@ Datum cloudsync_clear_filter (PG_FUNCTION_ARGS) {
             ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
                             errmsg("cloudsync_clear_filter: error recreating triggers")));
         }
+
+        // Clean and refill metatable without filter (all rows)
+        rc = cloudsync_reset_metatable(data, tbl);
+        if (rc != DBRES_OK) {
+            ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),
+                            errmsg("cloudsync_clear_filter: error resetting metatable")));
+        }
     }
     PG_CATCH();
     {
diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c
@@ -354,6 +354,9 @@ const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID =
 const char * const SQL_DROP_CLOUDSYNC_TABLE =
     "DROP TABLE IF EXISTS %s CASCADE;";
 
+const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE =
+    "DELETE FROM %s;";
+
 const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL =
     "DELETE FROM %s WHERE col_name NOT IN ("
     "SELECT column_name FROM information_schema.columns WHERE table_name = '%s' "
diff --git a/src/sql.h b/src/sql.h
@@ -57,6 +57,7 @@ extern const char * const SQL_CLOUDSYNC_SELECT_COL_VERSION_BY_PK_COL;
 extern const char * const SQL_CLOUDSYNC_SELECT_SITE_ID_BY_PK_COL;
 extern const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID;
 extern const char * const SQL_DROP_CLOUDSYNC_TABLE;
+extern const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE;
 extern const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL;
 extern const char * const SQL_PRAGMA_TABLEINFO_PK_QUALIFIED_COLLIST_FMT;
 extern const char * const SQL_CLOUDSYNC_GC_DELETE_ORPHANED_PK;
diff --git a/src/sqlite/cloudsync_sqlite.c b/src/sqlite/cloudsync_sqlite.c
@@ -1259,6 +1259,14 @@ void dbsync_set_filter (sqlite3_context *context, int argc, sqlite3_value **argv
         return;
     }
 
+    // Clean and refill metatable with the new filter
+    rc = cloudsync_reset_metatable(data, tbl);
+    if (rc != DBRES_OK) {
+        dbsync_set_error(context, "cloudsync_set_filter: error resetting metatable");
+        sqlite3_result_error_code(context, rc);
+        return;
+    }
+
     sqlite3_result_int(context, 1);
 }
 
@@ -1289,6 +1297,14 @@ void dbsync_clear_filter (sqlite3_context *context, int argc, sqlite3_value **ar
         return;
     }
 
+    // Clean and refill metatable without filter (all rows)
+    rc = cloudsync_reset_metatable(data, tbl);
+    if (rc != DBRES_OK) {
+        dbsync_set_error(context, "cloudsync_clear_filter: error resetting metatable");
+        sqlite3_result_error_code(context, rc);
+        return;
+    }
+
     sqlite3_result_int(context, 1);
 }
 
diff --git a/src/sqlite/sql_sqlite.c b/src/sqlite/sql_sqlite.c
@@ -226,6 +226,9 @@ const char * const SQL_PRAGMA_TABLEINFO_LIST_NONPK_NAME_CID =
 const char * const SQL_DROP_CLOUDSYNC_TABLE =
     "DROP TABLE IF EXISTS \"%w\";";
 
+const char * const SQL_DELETE_ALL_FROM_CLOUDSYNC_TABLE =
+    "DELETE FROM \"%w\";";
+
 const char * const SQL_CLOUDSYNC_DELETE_COLS_NOT_IN_SCHEMA_OR_PKCOL =
     "DELETE FROM \"%w\" WHERE \"col_name\" NOT IN ("
     "SELECT name FROM pragma_table_info('%q') UNION SELECT '%s'"
diff --git a/test/postgresql/47_row_filter_advanced.sql b/test/postgresql/47_row_filter_advanced.sql
@@ -0,0 +1,194 @@
+-- 'Row-level filter advanced tests (clear, complex expressions, row transitions, filter change)'
+
+\set testid '47'
+\ir helper_test_init.sql
+
+-- Create database
+\connect postgres
+\ir helper_psql_conn_setup.sql
+DROP DATABASE IF EXISTS cloudsync_test_47_a;
+CREATE DATABASE cloudsync_test_47_a;
+
+\connect cloudsync_test_47_a
+\ir helper_psql_conn_setup.sql
+CREATE EXTENSION IF NOT EXISTS cloudsync;
+
+-- ============================================================
+-- Test 1: cloudsync_clear_filter lifecycle
+-- ============================================================
+CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER);
+SELECT cloudsync_init('tasks') AS _init \gset
+SELECT cloudsync_set_filter('tasks', 'user_id = 1') AS _sf \gset
+
+INSERT INTO tasks VALUES ('a', 'Task A', 1);
+INSERT INTO tasks VALUES ('b', 'Task B', 2);
+INSERT INTO tasks VALUES ('c', 'Task C', 1);
+
+-- Only matching rows tracked
+SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset
+SELECT (:meta_pk_count = 2) AS clear_t1a_ok \gset
+\if :clear_t1a_ok
+\echo [PASS] (:testid) clear_filter: 2 PKs tracked before clear
+\else
+\echo [FAIL] (:testid) clear_filter: expected 2 tracked PKs before clear, got :meta_pk_count
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- Clear filter
+SELECT cloudsync_clear_filter('tasks') AS _cf \gset
+
+-- Insert non-matching row — should now be tracked (no filter)
+-- clear_filter refilled metatable with all 3 existing rows (a, b, c) + insert d = 4
+INSERT INTO tasks VALUES ('d', 'Task D', 2);
+SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset
+SELECT (:meta_pk_count = 4) AS clear_t1b_ok \gset
+\if :clear_t1b_ok
+\echo [PASS] (:testid) clear_filter: non-matching row tracked after clear (4 PKs)
+\else
+\echo [FAIL] (:testid) clear_filter: expected 4 PKs after clear+insert, got :meta_pk_count
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- Update row 'b' — already tracked by clear_filter refill, meta count unchanged
+UPDATE tasks SET title = 'Task B Updated' WHERE id = 'b';
+SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM tasks_cloudsync \gset
+SELECT (:meta_pk_count = 4) AS clear_t1c_ok \gset
+\if :clear_t1c_ok
+\echo [PASS] (:testid) clear_filter: update on 'b' still 4 PKs
+\else
+\echo [FAIL] (:testid) clear_filter: expected 4 PKs after update on 'b', got :meta_pk_count
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- ============================================================
+-- Test 2: Complex filter — AND + comparison operators
+-- ============================================================
+DROP TABLE IF EXISTS items;
+CREATE TABLE items (id TEXT PRIMARY KEY NOT NULL, status TEXT, priority INTEGER, category TEXT, user_id INTEGER);
+SELECT cloudsync_init('items') AS _init \gset
+SELECT cloudsync_set_filter('items', 'user_id = 1 AND priority > 3') AS _sf \gset
+
+INSERT INTO items VALUES ('a', 'active', 5, 'work', 1);   -- matches
+INSERT INTO items VALUES ('b', 'active', 2, 'work', 1);   -- fails priority
+INSERT INTO items VALUES ('c', 'active', 5, 'work', 2);   -- fails user_id
+
+SELECT COUNT(DISTINCT pk) AS meta_pk_count FROM items_cloudsync \gset
+SELECT (:meta_pk_count = 1) AS complex_t2_ok \gset
+\if :complex_t2_ok
+\echo [PASS] (:testid) complex_filter: AND+comparison tracked 1 of 3 rows
+\else
+\echo [FAIL] (:testid) complex_filter: expected 1 tracked PK, got :meta_pk_count
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- ============================================================
+-- Test 3: IS NULL filter
+-- ============================================================
+SELECT cloudsync_clear_filter('items') AS _cf \gset
+SELECT cloudsync_set_filter('items', 'category IS NULL') AS _sf \gset
+
+SELECT COUNT(DISTINCT pk) AS meta_before FROM items_cloudsync \gset
+INSERT INTO items VALUES ('f', 'x', 1, NULL, 1);       -- matches
+INSERT INTO items VALUES ('g', 'x', 1, 'work', 1);     -- fails
+SELECT COUNT(DISTINCT pk) AS meta_after FROM items_cloudsync \gset
+SELECT ((:meta_after::int - :meta_before::int) = 1) AS null_t3_ok \gset
+\if :null_t3_ok
+\echo [PASS] (:testid) IS NULL filter: only NULL-category row tracked
+\else
+\echo [FAIL] (:testid) IS NULL filter: expected 1 new PK, got (:meta_after - :meta_before)
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- ============================================================
+-- Test 4: Row exits filter (matching -> non-matching via UPDATE)
+-- ============================================================
+DROP TABLE IF EXISTS trans;
+CREATE TABLE trans (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER);
+SELECT cloudsync_init('trans') AS _init \gset
+SELECT cloudsync_set_filter('trans', 'user_id = 1') AS _sf \gset
+
+INSERT INTO trans VALUES ('a', 'Task A', 1);
+
+SELECT COUNT(*) AS meta_before FROM trans_cloudsync \gset
+UPDATE trans SET user_id = 2 WHERE id = 'a';
+SELECT COUNT(*) AS meta_after FROM trans_cloudsync \gset
+SELECT (:meta_before = :meta_after) AS exit_t4_ok \gset
+\if :exit_t4_ok
+\echo [PASS] (:testid) row_exit: UPDATE out of filter did not change metadata
+\else
+\echo [FAIL] (:testid) row_exit: UPDATE out of filter changed metadata (:meta_before -> :meta_after)
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- ============================================================
+-- Test 5: Row enters filter (non-matching -> matching via UPDATE)
+-- ============================================================
+INSERT INTO trans VALUES ('b', 'Task B', 2);
+
+SELECT COUNT(DISTINCT pk) AS meta_before FROM trans_cloudsync \gset
+UPDATE trans SET user_id = 1 WHERE id = 'b';
+SELECT COUNT(DISTINCT pk) AS meta_after FROM trans_cloudsync \gset
+SELECT (:meta_after::int > :meta_before::int) AS enter_t5_ok \gset
+\if :enter_t5_ok
+\echo [PASS] (:testid) row_enter: UPDATE into filter created metadata
+\else
+\echo [FAIL] (:testid) row_enter: UPDATE into filter did not create metadata (:meta_before -> :meta_after)
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- ============================================================
+-- Test 6: Filter change after data
+-- ============================================================
+DROP TABLE IF EXISTS fchange;
+CREATE TABLE fchange (id TEXT PRIMARY KEY NOT NULL, title TEXT, user_id INTEGER);
+SELECT cloudsync_init('fchange') AS _init \gset
+SELECT cloudsync_set_filter('fchange', 'user_id = 1') AS _sf \gset
+
+INSERT INTO fchange VALUES ('a', 'A', 1);  -- matches
+INSERT INTO fchange VALUES ('b', 'B', 2);  -- non-matching
+INSERT INTO fchange VALUES ('c', 'C', 1);  -- matches
+
+SELECT COUNT(DISTINCT pk) AS meta_count FROM fchange_cloudsync \gset
+SELECT (:meta_count = 2) AS change_t6a_ok \gset
+\if :change_t6a_ok
+\echo [PASS] (:testid) filter_change: 2 PKs under initial filter
+\else
+\echo [FAIL] (:testid) filter_change: expected 2 PKs under initial filter, got :meta_count
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- Change filter — resets metatable to only rows matching new filter (user_id = 2)
+-- Only 'b' (user_id=2) matches new filter → 1 PK from refill, then insert d → 2
+SELECT cloudsync_set_filter('fchange', 'user_id = 2') AS _sf2 \gset
+
+INSERT INTO fchange VALUES ('d', 'D', 2);  -- matches new filter
+INSERT INTO fchange VALUES ('e', 'E', 1);  -- non-matching under new filter
+
+SELECT COUNT(DISTINCT pk) AS meta_count FROM fchange_cloudsync \gset
+SELECT (:meta_count = 2) AS change_t6b_ok \gset
+\if :change_t6b_ok
+\echo [PASS] (:testid) filter_change: 2 PKs after filter change (metatable reset)
+\else
+\echo [FAIL] (:testid) filter_change: expected 2 PKs after filter change, got :meta_count
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- Update 'a' (user_id=1) should NOT generate new metadata under new filter (user_id=2)
+SELECT COUNT(*) AS meta_before FROM fchange_cloudsync \gset
+UPDATE fchange SET title = 'A Updated' WHERE id = 'a';
+SELECT COUNT(*) AS meta_after FROM fchange_cloudsync \gset
+SELECT (:meta_before = :meta_after) AS change_t6c_ok \gset
+\if :change_t6c_ok
+\echo [PASS] (:testid) filter_change: update on 'a' not tracked under new filter
+\else
+\echo [FAIL] (:testid) filter_change: update on 'a' changed metadata (:meta_before -> :meta_after)
+SELECT (:fail::int + 1) AS fail \gset
+\endif
+
+-- Cleanup
+\ir helper_test_cleanup.sql
+\if :should_cleanup
+DROP DATABASE IF EXISTS cloudsync_test_47_a;
+\else
+\echo [INFO] !!!!!
+\endif
diff --git a/test/postgresql/48_row_filter_multi_table.sql b/test/postgresql/48_row_filter_multi_table.sql
diff --git a/test/postgresql/49_row_filter_prefill.sql b/test/postgresql/49_row_filter_prefill.sql
diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql
diff --git a/test/unit.c b/test/unit.c

Original file line number	Diff line number	Diff line change
`@@ -695,6 +695,13 @@ Datum cloudsync_set_filter (PG_FUNCTION_ARGS) {`
`695`	`695`	`ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),`
`696`	`696`	`errmsg("cloudsync_set_filter: error recreating triggers")));`
`697`	`697`	`}`
	`698`	`+`
	`699`	`+ // Clean and refill metatable with the new filter`
	`700`	`+ rc = cloudsync_reset_metatable(data, tbl);`
	`701`	`+ if (rc != DBRES_OK) {`
	`702`	`+ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),`
	`703`	`+ errmsg("cloudsync_set_filter: error resetting metatable")));`
	`704`	`+ }`
`698`	`705`	`}`
`699`	`706`	`PG_CATCH();`
`700`	`707`	`{`
`@@ -753,6 +760,13 @@ Datum cloudsync_clear_filter (PG_FUNCTION_ARGS) {`
`753`	`760`	`ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),`
`754`	`761`	`errmsg("cloudsync_clear_filter: error recreating triggers")));`
`755`	`762`	`}`
	`763`	`+`
	`764`	`+ // Clean and refill metatable without filter (all rows)`
	`765`	`+ rc = cloudsync_reset_metatable(data, tbl);`
	`766`	`+ if (rc != DBRES_OK) {`
	`767`	`+ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR),`
	`768`	`+ errmsg("cloudsync_clear_filter: error resetting metatable")));`
	`769`	`+ }`
`756`	`770`	`}`
`757`	`771`	`PG_CATCH();`
`758`	`772`	`{`