Build large test payloads lazily

.github/workflows/pr-tests.yml

@@ -53,6 +53,36 @@ jobs:
 
       - name: Run compatibility tests against ${{ matrix.target.name }}
         run: |
+          # Sample memory, load, and per-container usage to the live step log
+          # while the tests run. Writing to the live log means the samples
+          # survive even when the job is killed before it finishes, which is
+          # when a resource problem is most likely to show up. Every line begins
+          # with resource-monitor so the timeline can be read on its own by
+          # filtering the log for that word.
+          echo "resource-monitor samples follow. Each line shows the UTC time, host memory as used over total in megabytes, and swap in megabytes. It then shows the 1, 5, and 15 minute load average and the memory each container is using. Filter the log for resource-monitor to read them on their own."
+          sample_resources() {
+            set +e
+            while true; do
+              memory=$(free -m | awk '/^Mem:/{print $3"/"$2}')
+              swap=$(free -m | awk '/^Swap:/{print $3}')
+              load=$(cut -d' ' -f1-3 /proc/loadavg | tr ' ' ',')
+              containers=$(docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' 2>/dev/null \
+                    | awk '{print $1"="$2}' | paste -sd' ' -)
+              echo "resource-monitor $(date -u +%H:%M:%S) memory=${memory}MB swap=${swap}MB load=${load} ${containers}"
+              sleep 10
+            done
+          }
+          sample_resources &
+          # When the machine runs out of memory the kernel logs a line naming
+          # the process it kills. Follow the kernel log and forward that line so
+          # it lands in the step log, since it is the clearest evidence of an
+          # out of memory kill. The search terms are the exact phrases the
+          # kernel writes.
+          ( sudo -n dmesg --follow 2>/dev/null \
+            | grep --line-buffered -iE 'out of memory|killed process|oom-kill' \
+            | sed -u 's/^/resource-monitor out-of-memory /' ) &
+          trap 'kill %1 %2 2>/dev/null || true' EXIT
+
           pytest documentdb_tests/compatibility/tests \
             --connection-string "${{ matrix.target.connection_string }}" \
             --engine-name "${{ matrix.target.engine }}" \

documentdb_tests/compatibility/tests/core/aggregation/commands/aggregate/test_aggregate_allowdiskuse.py

@@ -27,6 +27,7 @@
     TYPE_MISMATCH_ERROR,
 )
 from documentdb_tests.framework.executor import execute_command
+from documentdb_tests.framework.lazy_payload import lazy
 from documentdb_tests.framework.parametrize import pytest_params
 from documentdb_tests.framework.property_checks import Eq, Exists
 from documentdb_tests.framework.target_collection import ViewCollection
@@ -120,7 +121,9 @@
     ),
     CommandTestCase(
         "allowdiskuse_true_memory_exceeded",
-        docs=[{"_id": i, "data": "x" * 100_000, "sort_key": 1050 - i} for i in range(1050)],
+        docs=lazy(
+            lambda: [{"_id": i, "data": "x" * 100_000, "sort_key": 1050 - i} for i in range(1050)]
+        ),
         command=lambda ctx: {
             "aggregate": ctx.collection,
             "pipeline": [{"$sort": {"sort_key": 1}}],
@@ -170,7 +173,9 @@
     ],
     CommandTestCase(
         "allowdiskuse_reject_memory_exceeded",
-        docs=[{"_id": i, "data": "x" * 100_000, "sort_key": 1050 - i} for i in range(1050)],
+        docs=lazy(
+            lambda: [{"_id": i, "data": "x" * 100_000, "sort_key": 1050 - i} for i in range(1050)]
+        ),
         command=lambda ctx: {
             "aggregate": ctx.collection,
             "pipeline": [{"$sort": {"sort_key": 1}}],

documentdb_tests/compatibility/tests/core/aggregation/commands/aggregate/test_aggregate_cursor_acceptance.py

@@ -11,6 +11,7 @@
 )
 from documentdb_tests.framework.assertions import assertResult
 from documentdb_tests.framework.executor import execute_command
+from documentdb_tests.framework.lazy_payload import lazy
 from documentdb_tests.framework.parametrize import pytest_params
 from documentdb_tests.framework.property_checks import Eq, Ne
 from documentdb_tests.framework.test_constants import (
@@ -492,7 +493,7 @@
 AGGREGATE_CURSOR_RESPONSE_LIMIT_TESTS: list[CommandTestCase] = [
     CommandTestCase(
         "cursor_batchsize_16mb_limit",
-        docs=[{"_id": i, "data": "x" * 1_000_000} for i in range(20)],
+        docs=lazy(lambda: [{"_id": i, "data": "x" * 1_000_000} for i in range(20)]),
         command=lambda ctx: {
             "aggregate": ctx.collection,
             "pipeline": [],

documentdb_tests/compatibility/tests/core/aggregation/commands/distinct/test_distinct_command_errors.py

@@ -24,6 +24,7 @@
     UNRECOGNIZED_COMMAND_FIELD_ERROR,
 )
 from documentdb_tests.framework.executor import execute_command
+from documentdb_tests.framework.lazy_payload import lazy
 from documentdb_tests.framework.parametrize import pytest_params
 from documentdb_tests.framework.test_constants import (
     DECIMAL128_INFINITY,
@@ -281,7 +282,7 @@
 DISTINCT_BSON_SIZE_LIMIT_TESTS: list[CommandTestCase] = [
     CommandTestCase(
         "bson_size_limit_exceeded",
-        docs=[{"_id": i, "x": f"v{i}" + "x" * 17_000} for i in range(1100)],
+        docs=lazy(lambda: [{"_id": i, "x": f"v{i}" + "x" * 17_000} for i in range(1100)]),
         command=lambda ctx: {"distinct": ctx.collection, "key": "x"},
         error_code=DISTINCT_TOO_BIG_ERROR,
         msg="distinct should produce an error when results exceed the 16MB BSON size limit",

documentdb_tests/compatibility/tests/core/operator/accumulators/sum/test_accumulator_sum_boundary.py

@@ -10,6 +10,7 @@
 )
 from documentdb_tests.framework.assertions import assertSuccess
 from documentdb_tests.framework.executor import execute_command
+from documentdb_tests.framework.lazy_payload import lazy, materialize
 from documentdb_tests.framework.parametrize import pytest_params
 from documentdb_tests.framework.test_constants import (
     DECIMAL128_NEGATIVE_ZERO,
@@ -158,7 +159,7 @@
 SUM_LARGE_GROUP_TESTS: list[AccumulatorTestCase] = [
     AccumulatorTestCase(
         "large_group_10k_int1",
-        docs=[{"v": 1} for _ in range(10_000)],
+        docs=lazy(lambda: [{"v": 1} for _ in range(10_000)]),
         pipeline=[
             {"$group": {"_id": None, "result": {"$sum": "$v"}}},
             {"$project": {"_id": 0, "value": "$result", "type": {"$type": "$result"}}},
@@ -175,7 +176,7 @@
 def test_accumulator_sum_boundary(collection, test_case: AccumulatorTestCase):
     """Test $sum integer boundary values and large group accumulation."""
     if test_case.docs:
-        collection.insert_many(test_case.docs)
+        collection.insert_many(materialize(test_case.docs))
     result = execute_command(
         collection,
         {"aggregate": collection.name, "pipeline": test_case.pipeline or [], "cursor": {}},
@@ -237,7 +238,7 @@ def test_accumulator_sum_boundary(collection, test_case: AccumulatorTestCase):
 def test_accumulator_sum_negative_zero(collection, test_case: AccumulatorTestCase):
     """Test $sum negative zero normalization."""
     if test_case.docs:
-        collection.insert_many(test_case.docs)
+        collection.insert_many(materialize(test_case.docs))
     result = execute_command(
         collection,
         {"aggregate": collection.name, "pipeline": test_case.pipeline or [], "cursor": {}},

documentdb_tests/compatibility/tests/core/operator/expressions/string/concat/test_concat_size_limit.py

@@ -9,6 +9,7 @@
 )
 from documentdb_tests.framework.assertions import assertFailureCode
 from documentdb_tests.framework.error_codes import STRING_SIZE_LIMIT_ERROR
+from documentdb_tests.framework.lazy_payload import lazy
 from documentdb_tests.framework.parametrize import pytest_params
 from documentdb_tests.framework.test_constants import STRING_SIZE_LIMIT_BYTES
 
@@ -22,38 +23,48 @@
     # Two large strings concatenated, just under the limit.
     ConcatTest(
         "size_two_args_one_under",
-        args=[
-            "a" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2),
-            "b" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2),
-        ],
-        expected="a" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2)
-        + "b" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2),
+        args=lazy(
+            lambda: [
+                "a" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2),
+                "b" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2),
+            ]
+        ),
+        expected=lazy(
+            lambda: "a" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2)
+            + "b" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2)
+        ),
         msg="$concat should handle two large strings concatenated together",
     ),
     ConcatTest(
         "size_one_under",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES - 1)],
-        expected="a" * (STRING_SIZE_LIMIT_BYTES - 1),
+        args=lazy(lambda: ["a" * (STRING_SIZE_LIMIT_BYTES - 1)]),
+        expected=lazy(lambda: "a" * (STRING_SIZE_LIMIT_BYTES - 1)),
         msg="$concat should succeed when result is one byte under the size limit",
     ),
     # 2-byte chars: one byte under the limit.
     ConcatTest(
         "size_one_under_2byte",
-        args=["é" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2) + "a"],
-        expected="é" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2) + "a",
+        args=lazy(lambda: ["é" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2) + "a"]),
+        expected=lazy(lambda: "é" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2) + "a"),
         msg="$concat should succeed with 2-byte chars one byte under the limit",
     ),
     # 4-byte chars: one byte under the limit.
     ConcatTest(
         "size_one_under_4byte",
-        args=["😀" * ((STRING_SIZE_LIMIT_BYTES - 1) // 4) + "abc"],
-        expected="😀" * ((STRING_SIZE_LIMIT_BYTES - 1) // 4) + "abc",
+        args=lazy(lambda: ["😀" * ((STRING_SIZE_LIMIT_BYTES - 1) // 4) + "abc"]),
+        expected=lazy(lambda: "😀" * ((STRING_SIZE_LIMIT_BYTES - 1) // 4) + "abc"),
         msg="$concat should succeed with 4-byte chars one byte under the limit",
     ),
     # Null propagation wins when individual args are under the limit.
     ConcatTest(
         "size_null_precedence",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES // 2), None, "b" * (STRING_SIZE_LIMIT_BYTES // 2)],
+        args=lazy(
+            lambda: [
+                "a" * (STRING_SIZE_LIMIT_BYTES // 2),
+                None,
+                "b" * (STRING_SIZE_LIMIT_BYTES // 2),
+            ]
+        ),
         expected=None,
         msg="$concat should return null when null appears among large strings under the limit",
     ),
@@ -63,45 +74,49 @@
 STRING_SIZE_LIMIT_ERROR_TESTS: list[ConcatTest] = [
     ConcatTest(
         "size_at_limit",
-        args=["a" * STRING_SIZE_LIMIT_BYTES],
+        args=lazy(lambda: ["a" * STRING_SIZE_LIMIT_BYTES]),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$concat should reject result at the size limit",
     ),
     # Two halves summing to exactly the limit.
     ConcatTest(
         "size_two_halves",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES // 2), "b" * (STRING_SIZE_LIMIT_BYTES // 2)],
+        args=lazy(
+            lambda: ["a" * (STRING_SIZE_LIMIT_BYTES // 2), "b" * (STRING_SIZE_LIMIT_BYTES // 2)]
+        ),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$concat should reject two strings summing to the size limit",
     ),
     # 2-byte chars totaling exactly the limit.
     ConcatTest(
         "size_at_limit_2byte",
-        args=["é" * (STRING_SIZE_LIMIT_BYTES // 2)],
+        args=lazy(lambda: ["é" * (STRING_SIZE_LIMIT_BYTES // 2)]),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$concat should reject 2-byte chars totaling the size limit",
     ),
     # 4-byte chars totaling exactly the limit.
     ConcatTest(
         "size_at_limit_4byte",
-        args=["😀" * (STRING_SIZE_LIMIT_BYTES // 4)],
+        args=lazy(lambda: ["😀" * (STRING_SIZE_LIMIT_BYTES // 4)]),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$concat should reject 4-byte chars totaling the size limit",
     ),
     # Many small operands summing to exactly the limit.
     ConcatTest(
         "size_many_small",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES // 1024)] * 1024,
+        args=lazy(lambda: ["a" * (STRING_SIZE_LIMIT_BYTES // 1024)] * 1024),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$concat should reject many small operands summing to the size limit",
     ),
     # Operand produced by a nested expression rather than a literal.
     ConcatTest(
         "size_nested",
-        args=[
-            {"$concat": ["a" * (STRING_SIZE_LIMIT_BYTES // 2)]},
-            "b" * (STRING_SIZE_LIMIT_BYTES // 2),
-        ],
+        args=lazy(
+            lambda: [
+                {"$concat": ["a" * (STRING_SIZE_LIMIT_BYTES // 2)]},
+                "b" * (STRING_SIZE_LIMIT_BYTES // 2),
+            ]
+        ),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$concat should reject nested expression result exceeding the size limit",
     ),
@@ -123,8 +138,8 @@ def test_concat_size_limit_stored_field(collection):
     """Test $concat size limit is enforced when stored fields contribute to the result."""
     result = execute_project_with_insert(
         collection,
-        {"s": "a" * (STRING_SIZE_LIMIT_BYTES // 2)},
-        {"result": {"$concat": ["$s", "b" * (STRING_SIZE_LIMIT_BYTES // 2)]}},
+        lazy(lambda: {"s": "a" * (STRING_SIZE_LIMIT_BYTES // 2)}),
+        lazy(lambda: {"result": {"$concat": ["$s", "b" * (STRING_SIZE_LIMIT_BYTES // 2)]}}),
     )
     assertFailureCode(
         result, STRING_SIZE_LIMIT_ERROR, msg="$concat should enforce size limit with stored fields"

documentdb_tests/compatibility/tests/core/operator/expressions/string/indexOfBytes/test_indexOfBytes_size_limit.py

@@ -7,6 +7,7 @@
     execute_expression,
 )
 from documentdb_tests.framework.error_codes import STRING_SIZE_LIMIT_ERROR
+from documentdb_tests.framework.lazy_payload import lazy
 from documentdb_tests.framework.parametrize import pytest_params
 from documentdb_tests.framework.test_constants import STRING_SIZE_LIMIT_BYTES
 
@@ -19,34 +20,34 @@
 INDEXOFBYTES_SIZE_LIMIT_SUCCESS_TESTS: list[IndexOfBytesTest] = [
     IndexOfBytesTest(
         "size_string_one_under",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES - 1), "a"],
+        args=lazy(lambda: ["a" * (STRING_SIZE_LIMIT_BYTES - 1), "a"]),
         expected=0,
         msg="$indexOfBytes should accept string one byte under the size limit",
     ),
     IndexOfBytesTest(
         "size_substring_one_under",
-        args=["hello", "a" * (STRING_SIZE_LIMIT_BYTES - 1)],
+        args=lazy(lambda: ["hello", "a" * (STRING_SIZE_LIMIT_BYTES - 1)]),
         expected=-1,
         msg="$indexOfBytes should accept substring one byte under the size limit",
     ),
     # 2-byte chars: one byte under the limit. Limit is byte-based, not codepoint-based.
     IndexOfBytesTest(
         "size_string_one_under_2byte",
-        args=["é" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2) + "a", "a"],
+        args=lazy(lambda: ["é" * ((STRING_SIZE_LIMIT_BYTES - 1) // 2) + "a", "a"]),
         expected=STRING_SIZE_LIMIT_BYTES - 2,
         msg="$indexOfBytes should accept 2-byte char string one byte under limit",
     ),
     # Found at end of a large string.
     IndexOfBytesTest(
         "size_found_at_end",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES - 2) + "b", "b"],
+        args=lazy(lambda: ["a" * (STRING_SIZE_LIMIT_BYTES - 2) + "b", "b"]),
         expected=STRING_SIZE_LIMIT_BYTES - 2,
         msg="$indexOfBytes should find match at end of a large string",
     ),
     # Not found in a large string.
     IndexOfBytesTest(
         "size_not_found",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES - 1), "b"],
+        args=lazy(lambda: ["a" * (STRING_SIZE_LIMIT_BYTES - 1), "b"]),
         expected=-1,
         msg="$indexOfBytes should return -1 for no match in a large string",
     ),
@@ -57,20 +58,20 @@
 INDEXOFBYTES_SIZE_LIMIT_ERROR_TESTS: list[IndexOfBytesTest] = [
     IndexOfBytesTest(
         "size_string_at_limit",
-        args=["a" * STRING_SIZE_LIMIT_BYTES, "a"],
+        args=lazy(lambda: ["a" * STRING_SIZE_LIMIT_BYTES, "a"]),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$indexOfBytes should reject string at the size limit",
     ),
     IndexOfBytesTest(
         "size_substring_at_limit",
-        args=["hello", "a" * STRING_SIZE_LIMIT_BYTES],
+        args=lazy(lambda: ["hello", "a" * STRING_SIZE_LIMIT_BYTES]),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$indexOfBytes should reject substring at the size limit",
     ),
     # 2-byte chars: exactly at the limit. Limit is byte-based, not codepoint-based.
     IndexOfBytesTest(
         "size_string_at_limit_2byte",
-        args=["é" * (STRING_SIZE_LIMIT_BYTES // 2), "a"],
+        args=lazy(lambda: ["é" * (STRING_SIZE_LIMIT_BYTES // 2), "a"]),
         error_code=STRING_SIZE_LIMIT_ERROR,
         msg="$indexOfBytes should reject 2-byte char string at the byte size limit",
     ),

documentdb_tests/compatibility/tests/core/operator/expressions/string/indexOfBytes/test_indexOfBytes_usage.py

@@ -8,6 +8,7 @@
     execute_project_with_insert,
 )
 from documentdb_tests.framework.assertions import assertSuccess
+from documentdb_tests.framework.lazy_payload import lazy
 from documentdb_tests.framework.parametrize import pytest_params
 from documentdb_tests.framework.test_constants import STRING_SIZE_LIMIT_BYTES
 
@@ -123,7 +124,7 @@ def test_indexofbytes_nested_field_paths(collection):
     ),
     IndexOfBytesTest(
         "return_type_large_index",
-        args=["a" * (STRING_SIZE_LIMIT_BYTES - 2) + "b", "b"],
+        args=lazy(lambda: ["a" * (STRING_SIZE_LIMIT_BYTES - 2) + "b", "b"]),
         msg="$indexOfBytes should return int type for large index value",
     ),
 ]