feat(xtest): 2.1 GiB ZIP64 boundary conformance (DSPX-4592)

.github/workflows/check.yml

@@ -36,9 +36,9 @@ jobs:
         working-directory: xtest
       # Offline tests for the harnesses whose own correctness gates a nightly
       # job: the benchmark statistics, measurement and CLI command builders,
-      # and the encryption fixture cache. No platform and no SDK builds
-      # required, so the part that has to be *correct* is checked on every PR
-      # rather than only when the nightly runs.
+      # the encryption fixture cache, and the ZIP64 central-directory parser.
+      # No platform and no SDK builds required, so the part that has to be
+      # *correct* is checked on every PR rather than only when the nightly runs.
       #
       # --frozen --no-build: resolve nothing and build nothing, so a
       # dependency cannot slip in an unlocked version or a setup script on a
@@ -48,7 +48,7 @@ jobs:
           uv run --frozen --no-build pytest --no-header -q
           test_bench_stats.py test_bench_measure.py test_bench_runner.py
           test_bench_arms.py test_sdk_commands.py test_tdfs_units.py
-          test_encryption_units.py test_sizes_units.py
+          test_encryption_units.py test_sizes_units.py test_zip64_units.py
         working-directory: xtest
       - name: Lint and test otdf-local
         run: |

xtest/AGENTS.md

@@ -27,15 +27,15 @@ fixture system.
 | `--sdks-encrypt`, `--sdks-decrypt` | Asymmetric encrypt/decrypt SDK selection (use when reproducing cross-SDK interop bugs). |
 | `--containers ztdf ztdf-ecwrap` | Which TDF container types to exercise. |
 | `--no-audit-logs` | Skip audit-log assertions for this run. CLI equivalent of `DISABLE_AUDIT_ASSERTIONS=1`. |
-| `--sizes small,chunky` | Which payload sizes to parametrize over (`small` 128 B, `chunky` 5 MiB, `large` 5 GiB). Defaults to `small`. Every extra size fans out every test taking `pt_file`. `--large` is a deprecated alias for `small,large`. |
+| `--sizes small,chunky` | Which payload sizes to parametrize over (`small` 128 B, `chunky` 5 MiB, `medium` 2.1 GiB, `large` 5 GiB). Defaults to `small`. Every extra size fans out every test taking `pt_file`. `--large` is a deprecated alias for `small,large`. |
 
 ## Environment Variables
 
 Beyond the repo-wide ones in `../AGENTS.md`:
 
 | Variable | Purpose |
 |----------|---------|
-| `XT_TMP_DIR` | Root for generated fixtures and ciphertexts (default `tmp/`). Point at a large volume for `large` runs. |
+| `XT_TMP_DIR` | Root for generated fixtures and ciphertexts (default `tmp/`). Point at a large volume for `medium`/`large` runs. |
 | `XT_FORCE_SUPPORTS` | Comma-separated features to treat as supported, bypassing the `cli.sh supports` gate. For evaluating a fix before it releases — see `../AGENTS.md`. Unknown names raise. |
 
 ## Authoring a New Test

xtest/conftest.py

@@ -126,7 +126,19 @@ def sizes_opt_type(v: str) -> list[str]:
             )
     # Cheapest first, so a fan-out run reports its fast cells before spending
     # minutes on a multi-GiB one.
-    return [n for n in sizes.SIZE_ORDER if n in set(names)]
+    ordered = [n for n in sizes.SIZE_ORDER if n in set(names)]
+    # SIZE_ORDER is derived from SIZES, so this cannot fire today. It is here
+    # because the failure it guards is invisible: a name validated against
+    # SIZES but absent from SIZE_ORDER is dropped here, which empties the
+    # parameter set, which pytest reports as "got empty parameter set" -- a
+    # *skip*, exit 0. A whole matrix disappears and the run stays green.
+    dropped = sorted(set(names) - set(ordered))
+    if dropped:
+        raise argparse.ArgumentTypeError(
+            f"size(s) {', '.join(dropped)} are in SIZES but missing from "
+            "SIZE_ORDER; they would be silently dropped from the run"
+        )
+    return ordered
 
 
 _SIZES_KEY = pytest.StashKey[list[str]]()
@@ -151,7 +163,8 @@ def resolve_sizes(config: pytest.Config) -> list[str]:
                 "deprecated spelling of --sizes small,large"
             )
         warnings.warn(
-            "--large is deprecated; use --sizes small,large",
+            "--large is deprecated; use --sizes small,large (or --sizes medium "
+            "for the 2-4 GiB ZIP64 band, which --large steps straight over)",
             DeprecationWarning,
             stacklevel=2,
         )
@@ -428,23 +441,51 @@ def pytest_configure(config: pytest.Config):
         )
 
 
+def _item_exercises_zip64_window(item: pytest.Item, session_sizes: list[str]) -> bool:
+    """Whether this item has a payload large enough for the ZIP64 tests.
+
+    Size-aware items must be judged by their own parametrized value.  Marked
+    items without a ``size`` parameter retain the session-level behaviour so
+    a future ZIP64 test with a purpose-built fixture is not dropped merely
+    because it does not use :func:`pt_file`.
+    """
+    callspec = getattr(item, "callspec", None)
+    item_size = callspec.params.get("size") if callspec is not None else None
+    if isinstance(item_size, str):
+        return sizes.exercises_zip64_window(item_size)
+    return any(sizes.exercises_zip64_window(size) for size in session_sizes)
+
+
 def pytest_collection_modifyitems(
     config: pytest.Config, items: list[pytest.Item]
 ) -> None:
-    """Drop the benchmark cells entirely unless --bench asked for them.
-
-    Deselected rather than skipped: a 20-minute cell has no business in the
-    regular integration matrix, and a skip would report it as a test that
-    exists and was declined rather than one that was never in scope.
+    """Drop cells the session did not ask for.
+
+    Two groups, deselected rather than skipped for the same reason: neither a
+    20-minute benchmark nor a 2.1 GiB roundtrip has any business in the
+    regular integration matrix, and a skip would report them as tests that
+    exist and were declined rather than ones that were never in scope.
+
+    - ``benchmark``: needs --bench.
+    - ``zip64``: needs a payload size that can reach the 2**31 boundary. At
+      the default 128 bytes these tests cannot exercise anything, and the one
+      thing worse than not running them is running them green on a payload
+      that never touches the code path.
     """
-    if config.getoption("--bench", default=False):
-        return
-    keep, drop = [], []
+    drop: list[pytest.Item] = []
+    want_bench = bool(config.getoption("--bench", default=False))
+    session_sizes = resolve_sizes(config)
     for item in items:
-        (drop if item.get_closest_marker("benchmark") else keep).append(item)
+        if not want_bench and item.get_closest_marker("benchmark"):
+            drop.append(item)
+        elif item.get_closest_marker("zip64") and not _item_exercises_zip64_window(
+            item, session_sizes
+        ):
+            drop.append(item)
     if drop:
+        dropped = set(map(id, drop))
         config.hook.pytest_deselected(items=drop)
-        items[:] = keep
+        items[:] = [i for i in items if id(i) not in dropped]
 
 
 def pytest_sessionfinish(session: pytest.Session, exitstatus: int):
@@ -584,8 +625,9 @@ def pt_file(tmp_dir: Path, size: str) -> Path:
     Args:
         tmp_dir: Temporary directory for test files
         size: a key of :data:`sizes.SIZES` -- 'small' (128 bytes),
-            'chunky' (5 MiB, several default-sized segments), or
-            'large' (5 GiB)
+            'chunky' (5 MiB, several default-sized segments),
+            'medium' (2.1 GiB, inside the ZIP64 broken window), or
+            'large' (5 GiB, above it)
 
     Returns:
         Path to the generated plaintext file

xtest/pyproject.toml

@@ -89,6 +89,7 @@ known-first-party = [
     "fixtures",
     "perf",
     "sizes",
+    "zipinspect",
 ]
 
 [tool.ruff.format]
@@ -113,4 +114,5 @@ addopts = "-ra -v"
 markers = [
     "benchmark: paired A/B performance cell; only collected under --bench",
     "no_audit_logs: opt this test out of the default audit-log assertions",
+    "zip64: multi-GiB ZIP64 boundary cell; only collected when --sizes reaches 2**31",
 ]

xtest/sdk/go/cli.sh

@@ -132,6 +132,19 @@ if [ "$1" == "supports" ]; then
       echo "chunky unsupported: see DSPX-4590"
       exit 1
       ;;
+    zip64-at-2gib)
+      # Switch to the ZIP64 sentinel plus extra field at 2 GiB rather than at
+      # 4 GiB, so a reader that widens the central-directory fields with a
+      # signed read can still open the container. Every go build to date gates
+      # on ^uint32(0) and so writes a real 32-bit value across the whole
+      # 2-4 GiB band. Fix tracked as DSPX-4590 finding 1 (platform#3981, open);
+      # turn this into a version gate when it releases.
+      #
+      # Explicit rather than falling through to "Unknown feature" so that a
+      # typo'd feature name in tdfs.py cannot pass for a known-missing one.
+      echo "zip64-at-2gib unsupported: see DSPX-4590"
+      exit 1
+      ;;
     *)
       echo "Unknown feature: $2"
       exit 2

xtest/sdk/java/cli.sh

@@ -160,6 +160,20 @@ if [ "$1" == "supports" ]; then
       echo "chunky unsupported: see DSPX-4589"
       exit 1
       ;;
+    zip64-at-2gib)
+      # Switch to the ZIP64 sentinel plus extra field at 2 GiB rather than at
+      # 4 GiB. java-sdk adopted MAX_NON_ZIP64_VALUE = Integer.MAX_VALUE in
+      # java-sdk#393, merged 2026-09-03 and not in any release through v0.18.0
+      # -- and a branch build reports the last released version here, so this
+      # answers no for java@main too. Evaluate such a build with
+      # XT_FORCE_SUPPORTS=zip64-at-2gib; turn this into a version gate when the
+      # fix releases.
+      #
+      # Explicit rather than falling through to "Unknown feature" so that a
+      # typo'd feature name in tdfs.py cannot pass for a known-missing one.
+      echo "zip64-at-2gib unsupported: needs the release carrying java-sdk#393"
+      exit 1
+      ;;
     *)
       echo "Unknown feature: $2"
       exit 2

xtest/sdk/js/cli.sh

@@ -126,6 +126,12 @@ if [[ "$1" == "supports" ]]; then
       # test. See DSPX-4591.
       exit 0
       ;;
+    zip64-at-2gib)
+      # web-sdk writes the ZIP64 sentinel unconditionally, so it is trivially
+      # on the right side of the 2 GiB switch point. Predates any version we
+      # test. See DSPX-4591.
+      exit 0
+      ;;
     *)
       echo "Unknown feature: $2"
       exit 2

xtest/sizes.py

@@ -1,13 +1,53 @@
-"""Plaintext payload sizes for cross-SDK test fixtures.
+"""Plaintext payload sizes, and the ZIP64 window they are chosen around.
 
 Kept free of pytest and of ``tdfs`` so that both ``conftest.py`` and the test
 modules can name a size without importing each other.
+
+The ZIP central directory stores local-header offsets and entry sizes in 32-bit
+fields that are *unsigned on the wire*. Three regimes follow, and only one of
+them can expose a signed-widening bug:
+
+===========================  ==========================================
+value                        what a reader sees
+===========================  ==========================================
+``v < 2**31``                a signed read and an unsigned read agree
+``2**31 <= v < 2**32``       a signed read comes back negative
+``v >= 2**32``               ZIP64 sentinel; the 32-bit field is never
+                             populated with a real value, so the bug
+                             cannot fire
+===========================  ==========================================
+
+That middle row is the only broken window, and it is exactly what
+:data:`SIZES`'s ``medium`` entry exists to land a TDF's manifest offset in.
 """
 
 from __future__ import annotations
 
-#: 5 MiB. This is the smallest size at which *every* SDK's writer emits more
-#: than one **default-sized** segment.
+#: Smallest value a 32-bit field must be read as unsigned to survive.
+ZIP64_WINDOW_LOW = 2**31
+
+#: At and above this the format requires the ZIP64 sentinel plus an extra
+#: field, so the 32-bit field holds 0xFFFFFFFF rather than a real value.
+ZIP64_WINDOW_HIGH = 2**32
+
+#: 2.1 GiB. Sits ~102 MiB inside the low edge of the broken window.
+#:
+#: The margin is the point. A TDF writes ``0.payload`` first and
+#: ``0.manifest.json`` after it, so the manifest's local-header offset is
+#: roughly the payload size -- and that offset is the value under test. The
+#: gap to 2**31 has to be wider than anything that could shift it: segment
+#: padding, manifest length, per-entry header overhead. 102 MiB is not a
+#: round number because it does not need to be; it needs to be unarguably
+#: larger than those.
+#:
+#: Shrinking this below 2**31 does not make the test cheaper, it makes it
+#: vacuous -- every SDK takes the safe path and the test passes without
+#: exercising anything. See the assertion in test_zip64.py that fails loudly
+#: rather than letting that happen quietly.
+MEDIUM_BYTES = 2_254_857_830
+
+#: 5 MiB. Nothing to do with the ZIP64 window -- this is the smallest size at
+#: which *every* SDK's writer emits more than one **default-sized** segment.
 #:
 #: A segment only exercises the ``chunky`` path if its size equals the
 #: manifest-level default, because that is precisely the case web-sdk omits
@@ -20,11 +60,35 @@
 #: largest of them with room to spare. 2 MiB would only do it for web-sdk.
 CHUNKY_BYTES = 5 * 2**20
 
+#: Declared cheapest first, because :data:`SIZE_ORDER` is derived from it.
 SIZES: dict[str, int] = {
     "small": 128,
     "chunky": CHUNKY_BYTES,
+    "medium": MEDIUM_BYTES,
     "large": 5 * 2**30,
 }
 
 #: Order to emit parametrized sizes in, cheapest first.
-SIZE_ORDER: tuple[str, ...] = ("small", "chunky", "large")
+#:
+#: Derived, not restated. ``resolve_sizes`` filters the requested sizes
+#: through this while ``--sizes`` validates them against :data:`SIZES`, so a
+#: name in one and not the other is accepted on the command line and then
+#: silently dropped -- which empties the parameter set and reports
+#: ``got empty parameter set`` as a *skip*, exit 0.
+SIZE_ORDER: tuple[str, ...] = tuple(SIZES)
+
+
+def in_zip64_window(n: int) -> bool:
+    """True for values a signed 32-bit read would mangle."""
+    return ZIP64_WINDOW_LOW <= n < ZIP64_WINDOW_HIGH
+
+
+def exercises_zip64_window(size: str) -> bool:
+    """True if a payload of this size can put a real value in the broken window.
+
+    Note this is ``>=`` the low edge rather than :func:`in_zip64_window`: a
+    5 GiB payload does not itself land in the window, but the run that asked
+    for it is plainly a large-file run and the zip64 module has something to
+    say about its ZIP64 encoding too.
+    """
+    return SIZES[size] >= ZIP64_WINDOW_LOW

xtest/tdfs.py

@@ -169,6 +169,21 @@ def is_sdk_type(val: str) -> TypeIs[sdk_type]:
     "multikao",
     "ns_grants",
     "obligations",
+    # Writer-side: switch to the ZIP64 sentinel plus extra field at 2 GiB
+    # rather than at 4 GiB.
+    #
+    # A real 32-bit value in [2**31, 2**32) is *legal* -- the central-directory
+    # size and offset fields are unsigned -- so this is a cross-SDK interop
+    # convention rather than a spec rule, which is why it is a feature gate and
+    # not an unconditional assertion on every writer. A reader that widens
+    # those fields with a signed read sees a negative number, and that
+    # describes every java-sdk released to date. web-sdk always writes ZIP64;
+    # java-sdk adopted the 2 GiB switch in java-sdk#393; go-sdk still switches
+    # at 4 GiB. See DSPX-4590 finding 1.
+    #
+    # Only observable from a payload that reaches the window; hence
+    # sizes.MEDIUM_BYTES.
+    "zip64-at-2gib",
 ]
 
 
@@ -882,9 +897,10 @@ def elides_segment_sizes(ct_file: Path) -> bool:
 def skip_chunky_skew(ct_file: Path, decrypt_sdk: SDK):
     """Skip if ``ct_file`` needs segment-size defaulting and the reader lacks it.
 
-    A skip and not an xfail: this cell runs on the PR gate, where a
-    permanently-red job trains people to ignore it, and it needs no dated
-    guess about which release carries the fix.
+    A skip and not an asserted failure: this cell runs on the PR gate, where a
+    permanently-red job trains people to ignore it, and unlike
+    :func:`zip64_reader_is_broken` it needs no dated guess about which release
+    carries the fix.
 
     The cost is that it stays skipped until somebody edits
     ``sdk/{go,java}/cli.sh`` to answer yes -- the ``supports`` case statement
@@ -906,6 +922,38 @@ def skip_chunky_skew(ct_file: Path, decrypt_sdk: SDK):
     )
 
 
+#: First java-sdk release containing java-sdk#393.
+#:
+#: Before it, ``ZipReader.readInt()`` sign-extends, so a central-directory
+#: offset in ``[2**31, 2**32)`` comes back negative and the read fails or
+#: seeks to nonsense. A 2.1 GiB payload puts the manifest's offset exactly
+#: there. See DSPX-4592.
+#:
+#: Keep this honest, and note that both directions of getting it wrong fail
+#: the run rather than hiding: set too high, a release that does carry the fix
+#: reads the container correctly and the "must fail" assertion fires; set too
+#: low, a pre-fix release is expected to succeed and its real failure is
+#: reported as a defect. Update it when the release with #393 actually ships,
+#: not when the PR merges.
+JAVA_ZIP64_READER_FIX = (0, 19, 0)
+
+
+def zip64_reader_is_broken(decrypt_sdk: SDK) -> bool:
+    """True for decryptors known to mishandle a real 32-bit value in the band.
+
+    The caller asserts the decrypt *fails* for these, rather than marking the
+    cell xfail. That is deliberate on both counts: a node-level xfail would
+    swallow every unrelated failure in the rest of the cell, and asserting the
+    failure means a build that has quietly been fixed turns the cell red so
+    somebody comes and deletes this predicate.
+
+    Branch builds (``main``) have no semver and are never assumed broken --
+    they are the builds expected to carry the fix.
+    """
+    sv = decrypt_sdk.semver()
+    return decrypt_sdk.sdk == "java" and sv is not None and sv < JAVA_ZIP64_READER_FIX
+
+
 def _parse_semver(version: str) -> tuple[int, int, int] | None:
     """Parse a version string (with optional 'v' prefix) into (major, minor, patch)."""
     m = _version_re.match(version.lstrip("v"))