From 4e233076702291ded36b128fae6312d8abe5ab60 Mon Sep 17 00:00:00 2001 From: Pedro Sakuma Travi <39205549+pedrosakuma@users.noreply.github.com> Date: Sat, 25 Apr 2026 18:29:16 +0000 Subject: [PATCH] feat: foreach-style enumerator for top-level groups (#156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each top-level group on {Msg}DataReader now exposes a property returning a ref struct enumerator usable with foreach — zero allocation, no closure capture, ref readonly Current for zero-copy access. Independent per-group views: each property access creates a fresh enumerator that computes its start offset by skipping prior groups. This means out-of-order access, early break, and repeated iteration are all safe and do not corrupt one another. ReadGroups remains untouched and idempotent. Gating: emitted only when all top-level groups are simple (no nested groups, no group-level varData). Complex messages keep ReadGroups only. - src/SbeCodeGenerator/Generators/Types/MessageDefinition.cs: AppendGroupEnumerators + HasSimpleTopLevelGroupsForEnumerator gate - 6 new integration tests covering: in-order iteration, out-of-order access, repeated iteration, empty groups, early break, BytesConsumed unchanged - README + CHANGELOG updated, version 1.4.0 -> 1.5.0 Closes #156 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 19 ++ README.md | 7 +- .../Generators/Types/MessageDefinition.cs | 139 ++++++++++++++ .../SbeSourceGenerator.csproj | 2 +- .../GroupForeachEnumeratorTests.cs | 170 ++++++++++++++++++ 5 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 tests/SbeCodeGenerator.IntegrationTests/GroupForeachEnumeratorTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e850e5..5dddcfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.0] - 2026-04-25 + +### Added — DevEx P0 features + +- **Foreach-style enumerators on `{Msg}DataReader` for top-level groups** (#156): Each top-level group now also exposes a property returning a `ref struct` enumerator usable with `foreach`. Zero allocation, no closure capture, `ref readonly Current` for zero-copy access: + ```csharp + if (OrderBookData.TryParse(buffer, out var reader)) + { + foreach (ref readonly var bid in reader.Bids) { /* ... */ } + foreach (ref readonly var ask in reader.Asks) { /* ... */ } + } + ``` + The enumerators are **independent per-group views** — each access computes its own start offset by skipping prior groups, so out-of-order access (`Asks` before `Bids`), early `break`, and repeated iteration are all safe and don't corrupt one another. `ReadGroups` continues to work unchanged (idempotent) for messages that need group-level varData or nested groups. + + **Gating**: emitted only when *all* top-level groups are simple (no nested groups, no group-level varData). Complex messages keep `ReadGroups` only. + +### Performance +- The new enumerator API is allocation-free and aggressively inlined. `MoveNext` is a small loop with a single dimension read; `Current` is a `ref readonly` into the original buffer. Designed to JIT down to the same shape as a hand-written for-loop over `MemoryMarshal.AsRef`. + ## [1.4.0] - 2026-04-25 ### Added — DevEx P0 features diff --git a/README.md b/README.md index 83728dc..6759e37 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,17 @@ See the [v1.2.0 entry in CHANGELOG.md](./CHANGELOG.md) for the full list. ## What's New in v1.0.0 -**Zero-copy `MessageDataReader`** — `TryParse` returns a lightweight ref struct that holds a reference directly into the buffer. Access fields via `reader.Data` (zero-copy `ref readonly`), process groups via `reader.ReadGroups(...)`, and (since v1.3.0) recover the raw wire bytes via `reader.Buffer` / `reader.Block` for replay/forwarding scenarios — no re-parse needed. +**Zero-copy `MessageDataReader`** — `TryParse` returns a lightweight ref struct that holds a reference directly into the buffer. Access fields via `reader.Data` (zero-copy `ref readonly`), iterate top-level groups via `foreach` (since v1.5.0, no closure alloc) or `reader.ReadGroups(...)`, and recover the raw wire bytes via `reader.Buffer` / `reader.Block` (since v1.3.0) for replay/forwarding scenarios. ```csharp if (CarData.TryParse(buffer, out var car)) { Console.WriteLine(car.Data.SerialNumber); // zero-copy field access + + // v1.5.0: foreach-style enumerators on simple top-level groups (zero alloc, no closures). + foreach (ref readonly var fuel in car.FuelFigures) { /* ... */ } + + // For groups with nested groups or group-level varData, ReadGroups remains: car.ReadGroups( (in FuelFiguresData fuel) => { /* ... */ }, (in PerformanceFiguresData perf, AccelerationData.AccelerationHandler onAccel) => { /* ... */ }); diff --git a/src/SbeCodeGenerator/Generators/Types/MessageDefinition.cs b/src/SbeCodeGenerator/Generators/Types/MessageDefinition.cs index 7bcee57..b1a8ed9 100644 --- a/src/SbeCodeGenerator/Generators/Types/MessageDefinition.cs +++ b/src/SbeCodeGenerator/Generators/Types/MessageDefinition.cs @@ -663,9 +663,148 @@ private void AppendReaderStruct(StringBuilder sb, int tabs) AppendReadGroups(sb, tabs); } + // Issue #156: foreach-style enumerators for top-level groups (zero-alloc, no closure). + // Only emitted when ALL top-level groups are simple (no nested groups, no group-level varData), + // because computing offsets of later groups requires cheap O(1) skips. + if (HasSimpleTopLevelGroupsForEnumerator) + { + AppendGroupEnumerators(sb, tabs); + } + sb.AppendLine("}", --tabs); } + private bool HasSimpleTopLevelGroupsForEnumerator + { + get + { + if (TypedGroups.Count == 0) return false; + foreach (var g in TypedGroups) + { + if (g.HasGroupData || g.HasNestedGroups) return false; + } + return true; + } + } + + private void AppendGroupEnumerators(StringBuilder sb, int tabs) + { + // Per-group: emit a static skip helper + property + nested ref struct enumerator. + // Each enumerator is independent: it reads its own dimension prefix on first MoveNext. + // The property computes the start offset by chaining static skip helpers across prior groups. + for (int idx = 0; idx < TypedGroups.Count; idx++) + { + var group = TypedGroups[idx]; + var entryType = $"{Name}Data.{group.Name}Data"; + var dimType = group.DimensionType; + var enumName = $"{group.Name}Enumerator"; + + // Static skip helper: consumes dimension + numInGroup * blockLength bytes. + sb.AppendLine("", tabs); + sb.AppendLine("[MethodImpl(MethodImplOptions.AggressiveInlining)]", tabs); + sb.AppendTabs(tabs).Append("private static int Skip").Append(group.Name) + .AppendLine("(ReadOnlySpan buffer, int offset)"); + sb.AppendLine("{", tabs++); + sb.AppendTabs(tabs).Append("if ((uint)offset > (uint)(buffer.Length - ").Append(dimType) + .AppendLine(".MESSAGE_SIZE)) return buffer.Length;"); + sb.AppendTabs(tabs).Append("ref readonly var dim = ref Unsafe.As(ref MemoryMarshal.GetReference(buffer.Slice(offset)));"); + sb.AppendTabs(tabs).Append("return offset + ").Append(dimType) + .AppendLine(".MESSAGE_SIZE + (int)(dim.NumInGroup * dim.BlockLength);"); + sb.AppendLine("}", --tabs); + + // Property — builds the start offset by skipping all prior simple groups. + sb.AppendLine("", tabs); + sb.AppendLine("/// ", tabs); + sb.AppendTabs(tabs).Append("/// Iterates the ").Append(group.Name) + .AppendLine(" group entries with zero allocation."); + sb.AppendLine("/// Each access returns a fresh enumerator; iterate with foreach.", tabs); + sb.AppendLine("/// ", tabs); + sb.AppendTabs(tabs).Append("public ").Append(enumName).Append(" ").Append(group.Name).Append(" => new ") + .Append(enumName).Append("(_buffer, "); + if (idx == 0) + { + sb.Append("_blockLength"); + } + else + { + // Chain skip calls: Skip{prev}(buffer, ... Skip{first}(buffer, _blockLength) ...) + for (int j = 0; j < idx; j++) + sb.Append("Skip").Append(TypedGroups[j].Name).Append("(_buffer, "); + sb.Append("_blockLength"); + for (int j = 0; j < idx; j++) + sb.Append(")"); + } + sb.AppendLine(");"); + + // Nested enumerator ref struct. + sb.AppendLine("", tabs); + sb.AppendLine("/// ", tabs); + sb.AppendTabs(tabs).Append("/// Zero-allocation enumerator for the ").Append(group.Name) + .AppendLine(" group."); + sb.AppendLine("/// ", tabs); + sb.AppendTabs(tabs).Append("public ref struct ").Append(enumName).AppendLine(); + sb.AppendLine("{", tabs++); + sb.AppendLine("private readonly ReadOnlySpan _buffer;", tabs); + sb.AppendLine("private int _offset;", tabs); + sb.AppendLine("private int _remaining;", tabs); + sb.AppendLine("private int _entryBlockLength;", tabs); + sb.AppendLine("private bool _initialized;", tabs); + sb.AppendLine("", tabs); + + sb.AppendLine("[MethodImpl(MethodImplOptions.AggressiveInlining)]", tabs); + sb.AppendTabs(tabs).Append("internal ").Append(enumName) + .AppendLine("(ReadOnlySpan buffer, int startOffset)"); + sb.AppendLine("{", tabs++); + sb.AppendLine("_buffer = buffer;", tabs); + sb.AppendLine("_offset = startOffset;", tabs); + sb.AppendLine("_remaining = 0;", tabs); + sb.AppendLine("_entryBlockLength = 0;", tabs); + sb.AppendLine("_initialized = false;", tabs); + sb.AppendLine("}", --tabs); + sb.AppendLine("", tabs); + + sb.AppendLine("[MethodImpl(MethodImplOptions.AggressiveInlining)]", tabs); + sb.AppendTabs(tabs).Append("public ").Append(enumName).AppendLine(" GetEnumerator() => this;"); + sb.AppendLine("", tabs); + + // Current — returns ref readonly into buffer (zero-copy). + sb.AppendTabs(tabs).Append("public ref readonly ").Append(entryType).AppendLine(" Current"); + sb.AppendLine("{", tabs++); + sb.AppendLine("[MethodImpl(MethodImplOptions.AggressiveInlining)]", tabs); + sb.AppendTabs(tabs).Append("get => ref Unsafe.As(ref MemoryMarshal.GetReference(_buffer.Slice(_offset)));"); + sb.AppendLine("}", --tabs); + sb.AppendLine("", tabs); + + // MoveNext — reads dimension on first call, advances cursor on subsequent calls. + sb.AppendLine("public bool MoveNext()", tabs); + sb.AppendLine("{", tabs++); + sb.AppendLine("if (!_initialized)", tabs); + sb.AppendLine("{", tabs++); + sb.AppendLine("_initialized = true;", tabs); + sb.AppendTabs(tabs).Append("if ((uint)_offset > (uint)(_buffer.Length - ").Append(dimType) + .AppendLine(".MESSAGE_SIZE)) return false;"); + sb.AppendTabs(tabs).Append("ref readonly var dim = ref Unsafe.As(ref MemoryMarshal.GetReference(_buffer.Slice(_offset)));"); + sb.AppendLine("_entryBlockLength = (int)dim.BlockLength;", tabs); + sb.AppendLine("_remaining = (int)dim.NumInGroup;", tabs); + sb.AppendTabs(tabs).Append("_offset += ").Append(dimType).AppendLine(".MESSAGE_SIZE;"); + sb.AppendLine("}", --tabs); + sb.AppendLine("else", tabs); + sb.AppendLine("{", tabs++); + sb.AppendLine("_offset += _entryBlockLength;", tabs); + sb.AppendLine("}", --tabs); + sb.AppendLine("if (_remaining <= 0) return false;", tabs); + sb.AppendLine("if ((uint)_offset > (uint)(_buffer.Length - _entryBlockLength)) return false;", tabs); + sb.AppendLine("_remaining--;", tabs); + sb.AppendLine("return true;", tabs); + sb.AppendLine("}", --tabs); + + sb.AppendLine("}", --tabs); + } + } + private void AppendReadGroups(StringBuilder sb, int tabs) { var callbackParams = BuildCallbackParams($"{Name}Data."); diff --git a/src/SbeCodeGenerator/SbeSourceGenerator.csproj b/src/SbeCodeGenerator/SbeSourceGenerator.csproj index 0b5f0e8..9dc5c40 100644 --- a/src/SbeCodeGenerator/SbeSourceGenerator.csproj +++ b/src/SbeCodeGenerator/SbeSourceGenerator.csproj @@ -11,7 +11,7 @@ false SbeSourceGenerator SBE Source Generator - 1.4.0 + 1.5.0 Pedro Sakuma Pedro Sakuma SBE Source Generator diff --git a/tests/SbeCodeGenerator.IntegrationTests/GroupForeachEnumeratorTests.cs b/tests/SbeCodeGenerator.IntegrationTests/GroupForeachEnumeratorTests.cs new file mode 100644 index 0000000..e94c88e --- /dev/null +++ b/tests/SbeCodeGenerator.IntegrationTests/GroupForeachEnumeratorTests.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using Xunit; +using OB = Integration.Test.V0.OrderBookData; + +namespace SbeCodeGenerator.IntegrationTests +{ + /// + /// Issue #156: foreach-style enumerators on {Msg}DataReader for top-level groups. + /// Zero alloc, no closure, independent per-group views (safe out-of-order access). + /// + public class GroupForeachEnumeratorTests + { + private static byte[] Encode(long instrumentId, OB.BidsData[] bids, OB.AsksData[] asks) + { + var buffer = new byte[1024]; + Assert.True(OB.TryEncode(new OB { InstrumentId = instrumentId }, buffer, bids, asks, out int written)); + Array.Resize(ref buffer, written); + return buffer; + } + + [Fact] + public void Foreach_IteratesBidsAndAsks_InDeclarationOrder() + { + var bids = new[] + { + new OB.BidsData { Price = 1000, Quantity = 10 }, + new OB.BidsData { Price = 1010, Quantity = 11 }, + }; + var asks = new[] + { + new OB.AsksData { Price = 2000, Quantity = 20 }, + new OB.AsksData { Price = 2010, Quantity = 21 }, + new OB.AsksData { Price = 2020, Quantity = 22 }, + }; + var buffer = Encode(7, bids, asks); + + Assert.True(OB.TryParse(buffer, out var reader)); + + var bidPrices = new List(); + foreach (ref readonly var b in reader.Bids) + bidPrices.Add(b.Price.Value); + Assert.Equal(new[] { 1000L, 1010L }, bidPrices); + + var askPrices = new List(); + foreach (ref readonly var a in reader.Asks) + askPrices.Add(a.Price.Value); + Assert.Equal(new[] { 2000L, 2010L, 2020L }, askPrices); + } + + [Fact] + public void Foreach_OutOfOrderAccess_StillWorks() + { + // Independent per-group views: accessing Asks before Bids must not corrupt state. + var bids = new[] { new OB.BidsData { Price = 100, Quantity = 1 } }; + var asks = new[] { new OB.AsksData { Price = 200, Quantity = 2 } }; + var buffer = Encode(1, bids, asks); + + Assert.True(OB.TryParse(buffer, out var reader)); + + int askCount = 0; + foreach (ref readonly var a in reader.Asks) + { + askCount++; + Assert.Equal(200L, a.Price.Value); + } + + int bidCount = 0; + foreach (ref readonly var b in reader.Bids) + { + bidCount++; + Assert.Equal(100L, b.Price.Value); + } + + Assert.Equal(1, askCount); + Assert.Equal(1, bidCount); + } + + [Fact] + public void Foreach_RepeatedIteration_WorksOnFreshAccessor() + { + var bids = new[] + { + new OB.BidsData { Price = 1, Quantity = 1 }, + new OB.BidsData { Price = 2, Quantity = 2 }, + }; + var buffer = Encode(0, bids, Array.Empty()); + + Assert.True(OB.TryParse(buffer, out var reader)); + + int sum1 = 0; + foreach (ref readonly var b in reader.Bids) sum1 += (int)b.Price.Value; + int sum2 = 0; + foreach (ref readonly var b in reader.Bids) sum2 += (int)b.Price.Value; + + Assert.Equal(3, sum1); + Assert.Equal(sum1, sum2); + } + + [Fact] + public void Foreach_EmptyGroups_YieldNothing() + { + var buffer = Encode(99, Array.Empty(), Array.Empty()); + + Assert.True(OB.TryParse(buffer, out var reader)); + + int bidCount = 0; + foreach (ref readonly var _ in reader.Bids) bidCount++; + int askCount = 0; + foreach (ref readonly var _ in reader.Asks) askCount++; + + Assert.Equal(0, bidCount); + Assert.Equal(0, askCount); + } + + [Fact] + public void Foreach_EarlyBreak_DoesNotCorruptOtherGroup() + { + var bids = new[] + { + new OB.BidsData { Price = 1, Quantity = 1 }, + new OB.BidsData { Price = 2, Quantity = 2 }, + new OB.BidsData { Price = 3, Quantity = 3 }, + }; + var asks = new[] { new OB.AsksData { Price = 999, Quantity = 9 } }; + var buffer = Encode(0, bids, asks); + + Assert.True(OB.TryParse(buffer, out var reader)); + + // Break early on bids + foreach (ref readonly var b in reader.Bids) + { + if (b.Price.Value == 1) break; + } + + // Asks should still be intact + int askCount = 0; + foreach (ref readonly var a in reader.Asks) + { + askCount++; + Assert.Equal(999L, a.Price.Value); + } + Assert.Equal(1, askCount); + } + + [Fact] + public void Foreach_MutatesEnumerableLocally_ReaderRemainsUsable() + { + // BytesConsumed is set to BlockLength initially and only updated by ReadGroups. + // The foreach API must NOT mutate reader state. + var bids = new[] { new OB.BidsData { Price = 1, Quantity = 1 } }; + var buffer = Encode(0, bids, Array.Empty()); + + Assert.True(OB.TryParse(buffer, out var reader)); + int bytesBefore = reader.BytesConsumed; + + foreach (ref readonly var _ in reader.Bids) { } + foreach (ref readonly var _ in reader.Asks) { } + + Assert.Equal(bytesBefore, reader.BytesConsumed); + + // ReadGroups still works after foreach (idempotent) + int bidCallbacks = 0; + reader.ReadGroups( + (in OB.BidsData _) => bidCallbacks++, + (in OB.AsksData _) => { }); + Assert.Equal(1, bidCallbacks); + } + } +}