From d136dc1e01154b481886ba069a02de830d3ff1a9 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 09:00:31 +0300 Subject: [PATCH 01/45] Fix pubsub TTL cache eviction --- .../TtlCacheTests.cs | 45 +++++++ .../Libp2p.Protocols.Pubsub/TtlCache.cs | 110 +++++++++++++----- 2 files changed, 125 insertions(+), 30 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs new file mode 100644 index 00000000..e6ee9aee --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +public class TtlCacheTests +{ + [Test] + public void RemoveExpired_RemovesEntriesRegardlessOfKeyOrder() + { + using TtlCache cache = new(50); + MessageId expiredHigh = new([0xFF]); + MessageId liveLow = new([0x01]); + + cache.Add(expiredHigh); + Thread.Sleep(80); + cache.Add(liveLow); + + cache.RemoveExpired(DateTimeOffset.UtcNow); + + Assert.Multiple(() => + { + Assert.That(cache.Contains(expiredHigh), Is.False); + Assert.That(cache.Contains(liveLow), Is.True); + }); + } + + [Test] + public void ExpiredEntries_AreNotReturnedBeforeTheSweeperRuns() + { + using TtlCache cache = new(25); + MessageId id = new([0x01]); + cache.Add(id, "value"); + + Thread.Sleep(60); + + Assert.Multiple(() => + { + Assert.That(cache.Contains(id), Is.False); + Assert.That(cache.TryGet(id, out _), Is.False); + Assert.That(cache.ToList(), Is.Empty); + }); + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs index 1c777b74..f93d1fcf 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs @@ -6,68 +6,118 @@ namespace Nethermind.Libp2p.Protocols.Pubsub; internal class TtlCache : IDisposable where TKey : notnull { private readonly int ttl; + private readonly object sync = new(); + private readonly Dictionary items = []; + private readonly CancellationTokenSource sweeperCancellation = new(); + private int disposed; - private struct CachedItem - { - public TItem Item { get; set; } - public DateTimeOffset ValidTill { get; set; } - } - - private readonly SortedDictionary items = []; - private bool isDisposed; + private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill); public TtlCache(int ttl) { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(ttl); this.ttl = ttl; - Task.Run(async () => + _ = Task.Run(async () => { - while (!isDisposed) + try { - await Task.Delay(5_000); - DateTimeOffset now = DateTimeOffset.UtcNow; - lock (items) + while (true) { - TKey[] keys = items.TakeWhile(i => i.Value.ValidTill < now).Select(i => i.Key).ToArray(); - foreach (TKey keyToRemove in keys) - { - items.Remove(keyToRemove); - } + await Task.Delay(Math.Min(5_000, ttl), sweeperCancellation.Token); + RemoveExpired(DateTimeOffset.UtcNow); } } + catch (OperationCanceledException) when (sweeperCancellation.IsCancellationRequested) + { + } }); } - public bool Contains(TKey key) => items.ContainsKey(key); + public bool Contains(TKey key) => TryGet(key, out _); - public TItem Get(TKey key) => items.GetValueOrDefault(key).Item; + public TItem Get(TKey key) => TryGet(key, out TItem item) ? item : default!; - public void Add(TKey key, TItem item) + public bool TryGet(TKey key, out TItem item) { - lock (items) + lock (sync) { - items.TryAdd(key, new CachedItem + if (items.TryGetValue(key, out CachedItem cachedItem) && cachedItem.ValidTill > DateTimeOffset.UtcNow) { - Item = item, - ValidTill = DateTimeOffset.UtcNow.AddMilliseconds(ttl), - }); + item = cachedItem.Item; + return true; + } + } + + item = default!; + return false; + } + + internal void RemoveExpired(DateTimeOffset now) + { + lock (sync) + { + if (items.Count == 0) + { + return; + } + + List? expired = null; + foreach ((TKey key, CachedItem item) in items) + { + if (item.ValidTill <= now) + { + (expired ??= []).Add(key); + } + } + + if (expired is not null) + { + foreach (TKey key in expired) + { + items.Remove(key); + } + } + } + } + + public void Add(TKey key, TItem item) + { + lock (sync) + { + items.TryAdd(key, new CachedItem(item, DateTimeOffset.UtcNow.AddMilliseconds(ttl))); } } public void Dispose() { - isDisposed = true; + if (Interlocked.Exchange(ref disposed, 1) != 0) + { + return; + } + + sweeperCancellation.Cancel(); + sweeperCancellation.Dispose(); + + lock (sync) + { + items.Clear(); + } } internal IList ToList() { - lock (items) + DateTimeOffset now = DateTimeOffset.UtcNow; + lock (sync) { - return items.Values.Select(i => i.Item).ToList(); + return items.Values + .Where(item => item.ValidTill > now) + .Select(item => item.Item) + .ToList(); } } } internal class TtlCache(int ttl) : TtlCache(ttl) where TKey : notnull { - public void Add(TKey key) => Add(key, false); + public void Add(TKey key) => Add(key, true); } From e6f0db95306a1c5f0a56bc6b303474dc4ffaa36c Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 10:30:45 +0300 Subject: [PATCH 02/45] Repair expired TTL cache entries --- .../Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs | 13 +++++++++++++ src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs | 9 ++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs index e6ee9aee..8295e63b 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -42,4 +42,17 @@ public void ExpiredEntries_AreNotReturnedBeforeTheSweeperRuns() Assert.That(cache.ToList(), Is.Empty); }); } + + [Test] + public void Add_ReplacesAnExpiredEntry() + { + using TtlCache cache = new(25); + MessageId id = new([0x01]); + cache.Add(id, "expired"); + + Thread.Sleep(60); + cache.Add(id, "replacement"); + + Assert.That(cache.Get(id), Is.EqualTo("replacement")); + } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs index f93d1fcf..dab96e98 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs @@ -9,6 +9,7 @@ internal class TtlCache : IDisposable where TKey : notnull private readonly object sync = new(); private readonly Dictionary items = []; private readonly CancellationTokenSource sweeperCancellation = new(); + private readonly Task sweeperTask; private int disposed; private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill); @@ -17,7 +18,7 @@ public TtlCache(int ttl) { ArgumentOutOfRangeException.ThrowIfNegativeOrZero(ttl); this.ttl = ttl; - _ = Task.Run(async () => + sweeperTask = Task.Run(async () => { try { @@ -84,6 +85,11 @@ public void Add(TKey key, TItem item) { lock (sync) { + if (items.TryGetValue(key, out CachedItem cachedItem) && cachedItem.ValidTill <= DateTimeOffset.UtcNow) + { + items.Remove(key); + } + items.TryAdd(key, new CachedItem(item, DateTimeOffset.UtcNow.AddMilliseconds(ttl))); } } @@ -96,6 +102,7 @@ public void Dispose() } sweeperCancellation.Cancel(); + sweeperTask.GetAwaiter().GetResult(); sweeperCancellation.Dispose(); lock (sync) From ff883a863820a9d40251beb99a0d797624814f0d Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 10:55:09 +0300 Subject: [PATCH 03/45] Keep the TTL cache sweep cadence stable --- src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs | 2 +- src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs index 8295e63b..33ca2625 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -27,7 +27,7 @@ public void RemoveExpired_RemovesEntriesRegardlessOfKeyOrder() } [Test] - public void ExpiredEntries_AreNotReturnedBeforeTheSweeperRuns() + public void ExpiredEntries_AreNotReturned() { using TtlCache cache = new(25); MessageId id = new([0x01]); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs index dab96e98..92e49a82 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs @@ -24,7 +24,7 @@ public TtlCache(int ttl) { while (true) { - await Task.Delay(Math.Min(5_000, ttl), sweeperCancellation.Token); + await Task.Delay(5_000, sweeperCancellation.Token); RemoveExpired(DateTimeOffset.UtcNow); } } From 2026c7b8e14be5257329e4e7c215a1519755cd2b Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 11:00:04 +0300 Subject: [PATCH 04/45] Stabilize TTL cache timing tests --- .../Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs index 33ca2625..4efe490a 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -9,12 +9,12 @@ public class TtlCacheTests [Test] public void RemoveExpired_RemovesEntriesRegardlessOfKeyOrder() { - using TtlCache cache = new(50); + using TtlCache cache = new(500); MessageId expiredHigh = new([0xFF]); MessageId liveLow = new([0x01]); cache.Add(expiredHigh); - Thread.Sleep(80); + Thread.Sleep(750); cache.Add(liveLow); cache.RemoveExpired(DateTimeOffset.UtcNow); @@ -29,11 +29,11 @@ public void RemoveExpired_RemovesEntriesRegardlessOfKeyOrder() [Test] public void ExpiredEntries_AreNotReturned() { - using TtlCache cache = new(25); + using TtlCache cache = new(500); MessageId id = new([0x01]); cache.Add(id, "value"); - Thread.Sleep(60); + Thread.Sleep(750); Assert.Multiple(() => { @@ -46,11 +46,11 @@ public void ExpiredEntries_AreNotReturned() [Test] public void Add_ReplacesAnExpiredEntry() { - using TtlCache cache = new(25); + using TtlCache cache = new(500); MessageId id = new([0x01]); cache.Add(id, "expired"); - Thread.Sleep(60); + Thread.Sleep(750); cache.Add(id, "replacement"); Assert.That(cache.Get(id), Is.EqualTo("replacement")); From 7fda16b6f5c3a75cbdb69f7e853264c7d103cb4e Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 13:14:46 +0300 Subject: [PATCH 05/45] Bound TTL cache entries --- .../TtlCacheTests.cs | 20 +++++ .../Libp2p.Protocols.Pubsub/TtlCache.cs | 84 ++++++++++++++----- 2 files changed, 81 insertions(+), 23 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs index 4efe490a..b7079efd 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -55,4 +55,24 @@ public void Add_ReplacesAnExpiredEntry() Assert.That(cache.Get(id), Is.EqualTo("replacement")); } + + [Test] + public void Add_EvictsTheOldestLiveEntryAtCapacity() + { + using TtlCache cache = new(ttl: 1_000, maxEntries: 2); + MessageId first = new([0x01]); + MessageId second = new([0x02]); + MessageId third = new([0x03]); + + cache.Add(first, "first"); + cache.Add(second, "second"); + cache.Add(third, "third"); + + Assert.Multiple(() => + { + Assert.That(cache.Contains(first), Is.False); + Assert.That(cache.Get(second), Is.EqualTo("second")); + Assert.That(cache.Get(third), Is.EqualTo("third")); + }); + } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs index 92e49a82..b7b8613e 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs @@ -6,18 +6,23 @@ namespace Nethermind.Libp2p.Protocols.Pubsub; internal class TtlCache : IDisposable where TKey : notnull { private readonly int ttl; + private readonly int maxEntries; private readonly object sync = new(); private readonly Dictionary items = []; + private readonly Queue<(TKey Key, long Sequence)> insertionOrder = []; private readonly CancellationTokenSource sweeperCancellation = new(); private readonly Task sweeperTask; private int disposed; + private long sequence; - private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill); + private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill, long Sequence); - public TtlCache(int ttl) + public TtlCache(int ttl, int maxEntries = int.MaxValue) { ArgumentOutOfRangeException.ThrowIfNegativeOrZero(ttl); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxEntries); this.ttl = ttl; + this.maxEntries = maxEntries; sweeperTask = Task.Run(async () => { try @@ -42,10 +47,15 @@ public bool TryGet(TKey key, out TItem item) { lock (sync) { - if (items.TryGetValue(key, out CachedItem cachedItem) && cachedItem.ValidTill > DateTimeOffset.UtcNow) + if (items.TryGetValue(key, out CachedItem cachedItem)) { - item = cachedItem.Item; - return true; + if (cachedItem.ValidTill > DateTimeOffset.UtcNow) + { + item = cachedItem.Item; + return true; + } + + items.Remove(key); } } @@ -57,41 +67,68 @@ internal void RemoveExpired(DateTimeOffset now) { lock (sync) { - if (items.Count == 0) - { - return; - } + RemoveExpiredLocked(now); + } + } - List? expired = null; - foreach ((TKey key, CachedItem item) in items) + public void Add(TKey key, TItem item) + { + lock (sync) + { + DateTimeOffset now = DateTimeOffset.UtcNow; + if (items.TryGetValue(key, out CachedItem cachedItem)) { - if (item.ValidTill <= now) + if (cachedItem.ValidTill > now) { - (expired ??= []).Add(key); + return; } + + items.Remove(key); } - if (expired is not null) + while (items.Count >= maxEntries) { - foreach (TKey key in expired) - { - items.Remove(key); - } + EvictOldest(); } + + long itemSequence = ++sequence; + items.Add(key, new CachedItem(item, now.AddMilliseconds(ttl), itemSequence)); + insertionOrder.Enqueue((key, itemSequence)); } } - public void Add(TKey key, TItem item) + private void RemoveExpiredLocked(DateTimeOffset now) { - lock (sync) + List? expired = null; + foreach ((TKey key, CachedItem item) in items) + { + if (item.ValidTill <= now) + { + (expired ??= []).Add(key); + } + } + + if (expired is not null) { - if (items.TryGetValue(key, out CachedItem cachedItem) && cachedItem.ValidTill <= DateTimeOffset.UtcNow) + foreach (TKey key in expired) { items.Remove(key); } + } + } - items.TryAdd(key, new CachedItem(item, DateTimeOffset.UtcNow.AddMilliseconds(ttl))); + private void EvictOldest() + { + while (insertionOrder.TryDequeue(out (TKey Key, long Sequence) oldest)) + { + if (items.TryGetValue(oldest.Key, out CachedItem item) && item.Sequence == oldest.Sequence) + { + items.Remove(oldest.Key); + return; + } } + + throw new InvalidOperationException("TTL cache insertion order was unexpectedly empty."); } public void Dispose() @@ -108,6 +145,7 @@ public void Dispose() lock (sync) { items.Clear(); + insertionOrder.Clear(); } } @@ -124,7 +162,7 @@ internal IList ToList() } } -internal class TtlCache(int ttl) : TtlCache(ttl) where TKey : notnull +internal class TtlCache(int ttl, int maxEntries = int.MaxValue) : TtlCache(ttl, maxEntries) where TKey : notnull { public void Add(TKey key) => Add(key, true); } From 187c9cde0c5af99f2e4e4f867cdab2869d407311 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 13:40:28 +0300 Subject: [PATCH 06/45] Stabilize TTL cache regression coverage --- .../Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs | 8 +++++--- src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs | 11 +++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs index b7079efd..7e2f69fd 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -14,13 +14,15 @@ public void RemoveExpired_RemovesEntriesRegardlessOfKeyOrder() MessageId liveLow = new([0x01]); cache.Add(expiredHigh); - Thread.Sleep(750); + DateTimeOffset expiredAfter = DateTimeOffset.UtcNow.AddMilliseconds(500); + Assert.That(() => DateTimeOffset.UtcNow >= expiredAfter, Is.True.After(2_000, 25)); cache.Add(liveLow); cache.RemoveExpired(DateTimeOffset.UtcNow); Assert.Multiple(() => { + Assert.That(cache.Count, Is.EqualTo(1)); Assert.That(cache.Contains(expiredHigh), Is.False); Assert.That(cache.Contains(liveLow), Is.True); }); @@ -33,7 +35,7 @@ public void ExpiredEntries_AreNotReturned() MessageId id = new([0x01]); cache.Add(id, "value"); - Thread.Sleep(750); + Assert.That(() => cache.Contains(id), Is.False.After(2_000, 25)); Assert.Multiple(() => { @@ -50,7 +52,7 @@ public void Add_ReplacesAnExpiredEntry() MessageId id = new([0x01]); cache.Add(id, "expired"); - Thread.Sleep(750); + Assert.That(() => cache.Contains(id), Is.False.After(2_000, 25)); cache.Add(id, "replacement"); Assert.That(cache.Get(id), Is.EqualTo("replacement")); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs index b7b8613e..99c56c52 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs @@ -43,6 +43,17 @@ public TtlCache(int ttl, int maxEntries = int.MaxValue) public TItem Get(TKey key) => TryGet(key, out TItem item) ? item : default!; + internal int Count + { + get + { + lock (sync) + { + return items.Count; + } + } + } + public bool TryGet(TKey key, out TItem item) { lock (sync) From 73b1d01a6b0bd5e42fad7284c9601a7c87c8638b Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 14:01:23 +0300 Subject: [PATCH 07/45] Remove expired TTL cache entries from eviction order --- .../TtlCacheTests.cs | 1 + .../Libp2p.Protocols.Pubsub/TtlCache.cs | 49 ++++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs index 7e2f69fd..8328d26a 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -23,6 +23,7 @@ public void RemoveExpired_RemovesEntriesRegardlessOfKeyOrder() Assert.Multiple(() => { Assert.That(cache.Count, Is.EqualTo(1)); + Assert.That(cache.EntryOrderCount, Is.EqualTo(1)); Assert.That(cache.Contains(expiredHigh), Is.False); Assert.That(cache.Contains(liveLow), Is.True); }); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs index 99c56c52..00acdd00 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs @@ -9,13 +9,11 @@ internal class TtlCache : IDisposable where TKey : notnull private readonly int maxEntries; private readonly object sync = new(); private readonly Dictionary items = []; - private readonly Queue<(TKey Key, long Sequence)> insertionOrder = []; + private readonly LinkedList insertionOrder = []; private readonly CancellationTokenSource sweeperCancellation = new(); private readonly Task sweeperTask; private int disposed; - private long sequence; - - private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill, long Sequence); + private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill, LinkedListNode Node); public TtlCache(int ttl, int maxEntries = int.MaxValue) { @@ -54,6 +52,17 @@ internal int Count } } + internal int EntryOrderCount + { + get + { + lock (sync) + { + return insertionOrder.Count; + } + } + } + public bool TryGet(TKey key, out TItem item) { lock (sync) @@ -66,7 +75,7 @@ public bool TryGet(TKey key, out TItem item) return true; } - items.Remove(key); + Remove(key, cachedItem); } } @@ -94,7 +103,7 @@ public void Add(TKey key, TItem item) return; } - items.Remove(key); + Remove(key, cachedItem); } while (items.Count >= maxEntries) @@ -102,9 +111,8 @@ public void Add(TKey key, TItem item) EvictOldest(); } - long itemSequence = ++sequence; - items.Add(key, new CachedItem(item, now.AddMilliseconds(ttl), itemSequence)); - insertionOrder.Enqueue((key, itemSequence)); + LinkedListNode node = insertionOrder.AddLast(key); + items.Add(key, new CachedItem(item, now.AddMilliseconds(ttl), node)); } } @@ -123,23 +131,30 @@ private void RemoveExpiredLocked(DateTimeOffset now) { foreach (TKey key in expired) { - items.Remove(key); + Remove(key, items[key]); } } } private void EvictOldest() { - while (insertionOrder.TryDequeue(out (TKey Key, long Sequence) oldest)) + LinkedListNode? oldest = insertionOrder.First; + if (oldest is null) { - if (items.TryGetValue(oldest.Key, out CachedItem item) && item.Sequence == oldest.Sequence) - { - items.Remove(oldest.Key); - return; - } + throw new InvalidOperationException("TTL cache insertion order was unexpectedly empty."); + } + + insertionOrder.RemoveFirst(); + if (!items.Remove(oldest.Value)) + { + throw new InvalidOperationException("TTL cache insertion order was out of sync with its entries."); } + } - throw new InvalidOperationException("TTL cache insertion order was unexpectedly empty."); + private void Remove(TKey key, CachedItem item) + { + items.Remove(key); + insertionOrder.Remove(item.Node); } public void Dispose() From e7f6b7de376b738686516226fb3ffb50498d40f1 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 09:08:53 +0300 Subject: [PATCH 08/45] Bound incoming pubsub RPC frames --- src/libp2p/Libp2p.Core/IReader.cs | 22 +++++++-- src/libp2p/Libp2p.Core/VarInt.cs | 19 +++++--- .../PubsubFrameLimitTests.cs | 48 +++++++++++++++++++ .../Libp2p.Protocols.Pubsub/PubSubSettings.cs | 1 + .../Libp2p.Protocols.Pubsub/PubsubProtocol.cs | 13 +++-- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 2 + 6 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs diff --git a/src/libp2p/Libp2p.Core/IReader.cs b/src/libp2p/Libp2p.Core/IReader.cs index b4397331..aa5b74df 100644 --- a/src/libp2p/Libp2p.Core/IReader.cs +++ b/src/libp2p/Libp2p.Core/IReader.cs @@ -42,15 +42,27 @@ Task ReadVarintAsync(CancellationToken token = default) return VarInt.Decode(this, token); } - Task ReadVarintUlongAsync() + Task ReadVarintUlongAsync(CancellationToken token = default) { - return VarInt.DecodeUlong(this); + return VarInt.DecodeUlong(this, token); } - async ValueTask ReadPrefixedProtobufAsync(MessageParser parser, CancellationToken token = default) where T : IMessage + ValueTask ReadPrefixedProtobufAsync(MessageParser parser, CancellationToken token = default) where T : IMessage { - int messageLength = await ReadVarintAsync(token); - ReadOnlySequence serializedMessage = await ReadAsync(messageLength, token: token).OrThrow(); + return ReadPrefixedProtobufAsync(parser, int.MaxValue, token); + } + + async ValueTask ReadPrefixedProtobufAsync(MessageParser parser, int maxMessageLength, CancellationToken token = default) where T : IMessage + { + ArgumentOutOfRangeException.ThrowIfNegative(maxMessageLength); + + ulong messageLength = await ReadVarintUlongAsync(token); + if (messageLength > (ulong)maxMessageLength) + { + throw new InvalidDataException($"Incoming protobuf message size {messageLength} exceeds the limit of {maxMessageLength}"); + } + + ReadOnlySequence serializedMessage = await ReadAsync((int)messageLength, token: token).OrThrow(); return parser.ParseFrom(serializedMessage); } diff --git a/src/libp2p/Libp2p.Core/VarInt.cs b/src/libp2p/Libp2p.Core/VarInt.cs index f8cb609a..30e7c023 100644 --- a/src/libp2p/Libp2p.Core/VarInt.cs +++ b/src/libp2p/Libp2p.Core/VarInt.cs @@ -77,16 +77,23 @@ public static ulong Decode(Span source, ref int offset) throw new EndOfStreamException("Exhausted span before end of integer."); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static async Task DecodeUlong(IReader buf) + public static Task DecodeUlong(IReader buf) + { + return DecodeUlong(buf, default); + } + + public static async Task DecodeUlong(IReader buf, CancellationToken token = default) { ulong res = 0; - byte mul = 0; for (int i = 0; i < 10; i++) { - byte @byte = (await buf.ReadAsync(1).OrThrow()).FirstSpan[0]; - res += ((ulong)@byte & 127) << mul; - mul += 7; + byte @byte = (await buf.ReadAsync(1, token: token).OrThrow()).FirstSpan[0]; + if (i == 9 && (@byte & 0x7f) > 1) + { + throw new FormatException("Invalid 7-bit encoding"); + } + + res |= ((ulong)@byte & 127) << (i * 7); if ((@byte & 128) == 0) { return res; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs new file mode 100644 index 00000000..e7fe5658 --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Google.Protobuf; +using Nethermind.Libp2p.Protocols.Pubsub.Dto; +using System.Buffers; + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +public class PubsubFrameLimitTests +{ + [TestCase(1_025UL)] + [TestCase(2_147_483_648UL)] + [TestCase(4_294_967_295UL)] + public async Task ReadPrefixedProtobufAsync_RejectsFramesLargerThanLimit(ulong messageLength) + { + TestChannel channel = new(); + Task write = channel.Reverse().WriteAsync(new ReadOnlySequence(EncodeVarint(messageLength))).AsTask(); + IChannel reader = channel; + + InvalidDataException? exception = Assert.ThrowsAsync(async () => + await reader.ReadPrefixedProtobufAsync(Rpc.Parser, 1_024)); + + Assert.That(exception!.Message, Does.Contain(messageLength.ToString())); + await write; + } + + [Test] + public async Task ReadPrefixedProtobufAsync_RejectsOverflowingVarint() + { + byte[] overflowedLength = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02]; + TestChannel channel = new(); + Task write = channel.Reverse().WriteAsync(new ReadOnlySequence(overflowedLength)).AsTask(); + IChannel reader = channel; + + Assert.ThrowsAsync(async () => await reader.ReadPrefixedProtobufAsync(Rpc.Parser, 1_024)); + await write; + } + + private static byte[] EncodeVarint(ulong value) + { + byte[] bytes = new byte[VarInt.GetSizeInBytes(value)]; + int offset = 0; + VarInt.Encode(value, bytes, ref offset); + return bytes; + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index a2a62ad0..93a24549 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -25,6 +25,7 @@ public class PubsubSettings public int mcache_len { get; set; } = 5; // Number of history windows in message cache 5 public int mcache_gossip { get; set; } = 3; // Number of history windows to use when emitting gossip 3 public int MessageCacheTtl { get; set; } = 2 * 60 * 1000; // Expiry time for cache of seen message ids 2 minutes + public int MaxRpcBytes { get; set; } = 1 * 1024 * 1024; // Maximum incoming RPC frame size 1 MiB public SignaturePolicy DefaultSignaturePolicy { get; set; } = SignaturePolicy.StrictSign; public int MaxIdontwantMessages { get; set; } = 50; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs index 1cae9e32..2fadb463 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs @@ -34,7 +34,7 @@ public async Task DialAsync(IChannel channel, ISessionContext context) ArgumentNullException.ThrowIfNull(context.State.RemoteAddress); ArgumentNullException.ThrowIfNull(context.State.RemotePeerId); - PeerId? remotePeerId = context.State.RemotePeerId; + PeerId remotePeerId = context.State.RemotePeerId!; _logger?.LogDebug("Dialed({contextId}) {remoteAddress}", context.Id, context.State.RemoteAddress); @@ -61,7 +61,7 @@ public async Task ListenAsync(IChannel channel, ISessionContext context) ArgumentNullException.ThrowIfNull(context.State.RemoteAddress); ArgumentNullException.ThrowIfNull(context.State.RemotePeerId); - PeerId? remotePeerId = context.State.RemotePeerId; + PeerId remotePeerId = context.State.RemotePeerId!; _logger?.LogDebug("Listen({contextId}) to {remoteAddress}", context.Id, context.State.RemoteAddress); @@ -75,7 +75,7 @@ public async Task ListenAsync(IChannel channel, ISessionContext context) { while (!token.IsCancellationRequested) { - Rpc? rpc = await channel.ReadPrefixedProtobufAsync(Rpc.Parser, token); + Rpc? rpc = await channel.ReadPrefixedProtobufAsync(Rpc.Parser, router.MaxRpcBytes, token); if (rpc is null) { _logger?.LogDebug("Received a broken message or EOF from {remotePeerId}", remotePeerId); @@ -90,6 +90,13 @@ public async Task ListenAsync(IChannel channel, ISessionContext context) } } } + catch (Exception e) when (e is InvalidDataException or FormatException) + { + _logger?.LogDebug("Invalid RPC from {remotePeerId}: {message}", remotePeerId, e.Message); + context.Activity?.AddEvent(new ActivityEvent($"Invalid RPC from {remotePeerId}")); + context.Activity?.SetStatus(ActivityStatusCode.Error); + await context.DisconnectAsync(); + } catch (Exception e) { context.Activity?.AddEvent(new ActivityEvent($"Exception: {e.Message}")); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index ff3a153d..4a3f573d 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -126,6 +126,8 @@ public Action? SendRpc public event Action? OnMessage; public Func? VerifyMessage = null; + internal int MaxRpcBytes => _settings.MaxRpcBytes; + private readonly PubsubSettings _settings; private readonly TtlCache _messageCache; private readonly TtlCache _limboMessageCache; From a4dad34b30aeca9d617c75490b20704caf5c614f Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 10:31:46 +0300 Subject: [PATCH 09/45] Validate pubsub RPC frame limits --- .../Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs | 6 ++++++ src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs index e7fe5658..ca06f677 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs @@ -10,6 +10,12 @@ namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; [TestFixture] public class PubsubFrameLimitTests { + [Test] + public void MaxRpcBytes_RejectsNegativeValues() + { + Assert.Throws(() => new PubsubSettings { MaxRpcBytes = -1 }); + } + [TestCase(1_025UL)] [TestCase(2_147_483_648UL)] [TestCase(4_294_967_295UL)] diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs index 2fadb463..55712985 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs @@ -92,7 +92,7 @@ public async Task ListenAsync(IChannel channel, ISessionContext context) } catch (Exception e) when (e is InvalidDataException or FormatException) { - _logger?.LogDebug("Invalid RPC from {remotePeerId}: {message}", remotePeerId, e.Message); + _logger?.LogDebug(e, "Invalid RPC from {remotePeerId}: {message}", remotePeerId, e.Message); context.Activity?.AddEvent(new ActivityEvent($"Invalid RPC from {remotePeerId}")); context.Activity?.SetStatus(ActivityStatusCode.Error); await context.DisconnectAsync(); From 23455dfef9d879a5ce532aac9cdb8e2c9f8a1139 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 10:32:05 +0300 Subject: [PATCH 10/45] Reject invalid pubsub RPC frame settings --- .../Libp2p.Protocols.Pubsub/PubSubSettings.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index 93a24549..c7120184 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -8,6 +8,8 @@ namespace Nethermind.Libp2p.Protocols.Pubsub; public class PubsubSettings { + private int maxRpcBytes = 1 * 1024 * 1024; + public static PubsubSettings Default { get; } = new(); public int ReconnectionAttempts { get; set; } = 10; @@ -25,7 +27,16 @@ public class PubsubSettings public int mcache_len { get; set; } = 5; // Number of history windows in message cache 5 public int mcache_gossip { get; set; } = 3; // Number of history windows to use when emitting gossip 3 public int MessageCacheTtl { get; set; } = 2 * 60 * 1000; // Expiry time for cache of seen message ids 2 minutes - public int MaxRpcBytes { get; set; } = 1 * 1024 * 1024; // Maximum incoming RPC frame size 1 MiB + // Maximum incoming RPC frame size 1 MiB + public int MaxRpcBytes + { + get => maxRpcBytes; + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + maxRpcBytes = value; + } + } public SignaturePolicy DefaultSignaturePolicy { get; set; } = SignaturePolicy.StrictSign; public int MaxIdontwantMessages { get; set; } = 50; From 3baa4fc21c20a77d8e50bfd5cb4f4e1688ae0e40 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 10:46:21 +0300 Subject: [PATCH 11/45] Preserve the varint reader overload --- src/libp2p/Libp2p.Core/IReader.cs | 7 ++++++- .../PubsubFrameLimitTests.cs | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Core/IReader.cs b/src/libp2p/Libp2p.Core/IReader.cs index aa5b74df..c3ca2e79 100644 --- a/src/libp2p/Libp2p.Core/IReader.cs +++ b/src/libp2p/Libp2p.Core/IReader.cs @@ -42,7 +42,12 @@ Task ReadVarintAsync(CancellationToken token = default) return VarInt.Decode(this, token); } - Task ReadVarintUlongAsync(CancellationToken token = default) + Task ReadVarintUlongAsync() + { + return VarInt.DecodeUlong(this); + } + + Task ReadVarintUlongAsync(CancellationToken token) { return VarInt.DecodeUlong(this, token); } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs index ca06f677..98538f92 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs @@ -10,6 +10,17 @@ namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; [TestFixture] public class PubsubFrameLimitTests { + [Test] + public async Task ReadVarintUlongAsync_WithoutCancellationTokenRemainsAvailable() + { + TestChannel channel = new(); + Task write = channel.Reverse().WriteAsync(new ReadOnlySequence(EncodeVarint(42))).AsTask(); + IChannel reader = channel; + + Assert.That(await reader.ReadVarintUlongAsync(), Is.EqualTo(42)); + await write; + } + [Test] public void MaxRpcBytes_RejectsNegativeValues() { From b2712d8298fa99685f63e65a4cd2d056dbfe6014 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 12:39:12 +0300 Subject: [PATCH 12/45] Tighten pubsub frame limit validation --- .../PubsubFrameLimitTests.cs | 8 ++++++-- src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs | 2 +- src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs index 98538f92..3b93306b 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs @@ -22,9 +22,13 @@ public async Task ReadVarintUlongAsync_WithoutCancellationTokenRemainsAvailable( } [Test] - public void MaxRpcBytes_RejectsNegativeValues() + public void MaxRpcBytes_RejectsNonPositiveValues() { - Assert.Throws(() => new PubsubSettings { MaxRpcBytes = -1 }); + Assert.Multiple(() => + { + Assert.Throws(() => new PubsubSettings { MaxRpcBytes = -1 }); + Assert.Throws(() => new PubsubSettings { MaxRpcBytes = 0 }); + }); } [TestCase(1_025UL)] diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index c7120184..cae2662d 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -33,7 +33,7 @@ public int MaxRpcBytes get => maxRpcBytes; set { - ArgumentOutOfRangeException.ThrowIfNegative(value); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(value); maxRpcBytes = value; } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs index 55712985..1ff2628b 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs @@ -93,7 +93,7 @@ public async Task ListenAsync(IChannel channel, ISessionContext context) catch (Exception e) when (e is InvalidDataException or FormatException) { _logger?.LogDebug(e, "Invalid RPC from {remotePeerId}: {message}", remotePeerId, e.Message); - context.Activity?.AddEvent(new ActivityEvent($"Invalid RPC from {remotePeerId}")); + context.Activity?.AddEvent(new ActivityEvent($"Invalid RPC from {remotePeerId}: {e.Message}")); context.Activity?.SetStatus(ActivityStatusCode.Error); await context.DisconnectAsync(); } From 3a511106e697563fb17c247b9312792b02c6b5a4 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 09:15:18 +0300 Subject: [PATCH 13/45] Repair pubsub topic lifecycle --- .../TopicLifecycleTests.cs | 88 +++++++++++++++++++ .../PubsubRouter.Topics.cs | 66 ++++++-------- src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs | 4 +- 3 files changed, 117 insertions(+), 41 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs new file mode 100644 index 00000000..a816d787 --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Multiformats.Address; +using Nethermind.Libp2p.Core.Discovery; +using Nethermind.Libp2p.Protocols.Pubsub; +using Nethermind.Libp2p.Protocols.Pubsub.Dto; + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +public class TopicLifecycleTests +{ + [Test] + public void Topic_UnsubscribeStopsDeliveryUntilResubscribed() + { + const string topicName = "topic-lifecycle"; + PeerStore peerStore = new(); + PubsubRouter router = new(peerStore); + ITopic topic = router.GetTopic(topicName); + PeerId receivedFrom = TestPeers.PeerId(1); + Identity author = TestPeers.Identity(2); + int deliveries = 0; + topic.OnMessage += (_, _) => deliveries++; + + topic.Unsubscribe(); + router.OnRpc(receivedFrom, CreateMessage(topicName, author, 1)); + + Assert.Multiple(() => + { + Assert.That(topic.IsSubscribed, Is.False); + Assert.That(deliveries, Is.Zero); + }); + + topic.Subscribe(); + router.OnRpc(receivedFrom, CreateMessage(topicName, author, 2)); + + Assert.Multiple(() => + { + Assert.That(topic.IsSubscribed, Is.True); + Assert.That(deliveries, Is.EqualTo(1)); + }); + } + + [Test] + public async Task Topic_ResubscribeRetainsRemoteMembership() + { + const string topicName = "topic-lifecycle"; + PeerStore peerStore = new(); + PubsubRouter router = new(peerStore); + IRoutingStateContainer state = router; + ITopic topic = router.GetTopic(topicName); + Multiaddress peerAddress = TestPeers.Multiaddr(3); + PeerId peerId = peerAddress.GetPeerId()!; + TaskCompletionSource connectionClosed = new(); + List sent = []; + + router.OutboundConnection(peerAddress, PubsubRouter.GossipsubProtocolVersionV11, connectionClosed.Task, sent.Add); + router.OnRpc(peerId, new Rpc().WithTopics([topicName], [])); + sent.Clear(); + + topic.Unsubscribe(); + + Assert.Multiple(() => + { + Assert.That(state.GossipsubPeers[topicName], Has.Member(peerId)); + Assert.That(state.Mesh, Does.Not.ContainKey(topicName)); + }); + + sent.Clear(); + topic.Subscribe(); + await state.Heartbeat(); + + Assert.Multiple(() => + { + Assert.That(state.GossipsubPeers[topicName], Has.Member(peerId)); + Assert.That(state.Mesh[topicName], Has.Member(peerId)); + Assert.That(sent.Any(rpc => rpc.Subscriptions.Any(subscription => subscription.Subscribe && subscription.Topicid == topicName)), Is.True); + }); + + connectionClosed.SetResult(); + } + + private static Rpc CreateMessage(string topicName, Identity author, ulong sequenceNumber) + { + return new Rpc().WithMessages(topicName, sequenceNumber, author.PeerId.Bytes, [1, 2, 3], author); + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index 91304abf..74f5f5d5 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -27,14 +27,15 @@ public ITopic GetTopic(string topicId, bool subscribe = true) public void Subscribe(string topicId) { - topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)).IsSubscribed = true; - - if (!fPeers.TryAdd(topicId, [])) + Topic topic = topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)); + if (topic.IsSubscribed) { - // Already exists return; } + topic.IsSubscribed = true; + + fPeers.TryAdd(topicId, []); gPeers.TryAdd(topicId, []); HashSet meshPeers = mesh.GetOrAdd(topicId, []); @@ -58,9 +59,14 @@ public void Subscribe(string topicId) public void Unsubscribe(string topicId) { - topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)).IsSubscribed = false; + if (!topicState.TryGetValue(topicId, out Topic? topic) || !topic.IsSubscribed) + { + return; + } - foreach (PeerId peerId in fPeers[topicId]) + topic.IsSubscribed = false; + + foreach (PeerId peerId in fPeers.GetValueOrDefault(topicId) ?? []) { Rpc msg = new Rpc() .WithTopics([], [topicId]); @@ -68,7 +74,7 @@ public void Unsubscribe(string topicId) peerState.GetValueOrDefault(peerId)?.Send(msg); } - foreach (PeerId peerId in gPeers[topicId]) + foreach (PeerId peerId in gPeers.GetValueOrDefault(topicId) ?? []) { Rpc msg = new Rpc() .WithTopics([], [topicId]); @@ -79,45 +85,27 @@ public void Unsubscribe(string topicId) } peerState.GetValueOrDefault(peerId)?.Send(msg); } - } - public void UnsubscribeAll() - { - try + if (mesh.TryRemove(topicId, out HashSet? removedMesh)) { - foreach (PeerId? peerId in fPeers.SelectMany(kv => kv.Value)) - { - Rpc msg = new Rpc().WithTopics([], topicState.Keys); - - peerState.GetValueOrDefault(peerId)?.Send(msg); - } - - Dictionary peerMessages = []; - - foreach (PeerId? peerId in gPeers.SelectMany(kv => kv.Value)) + foreach (PeerId peerId in removedMesh) { - (peerMessages[peerId] ??= new Rpc()) - .WithTopics([], topicState.Keys); + RecordPeerLeaveMesh(peerId, topicId); } + } - foreach (KeyValuePair> topicMesh in mesh.ToDictionary()) - { - foreach (PeerId peerId in topicMesh.Value) - { - (peerMessages[peerId] ??= new Rpc()) - .Ensure(r => r.Control.Prune) - .Add(new ControlPrune { TopicID = topicMesh.Key }); - } - } + fanout.TryRemove(topicId, out _); + fanoutLastPublished.TryRemove(topicId, out _); + } - foreach (KeyValuePair peerMessage in peerMessages) - { - peerState.GetValueOrDefault(peerMessage.Key)?.Send(peerMessage.Value); - } - } - catch (Exception e) + public void UnsubscribeAll() + { + foreach (string topicId in topicState + .Where(pair => pair.Value.IsSubscribed) + .Select(pair => pair.Key) + .ToArray()) { - logger?.LogError(e, $"Error during {nameof(UnsubscribeAll)}"); + Unsubscribe(topicId); } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs index e5cbfdc7..18f838dd 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs @@ -19,7 +19,7 @@ public Topic(PubsubRouter router, string topicName) private void OnRouterMessage(string topicName, PeerId peerId, byte[] message) { - if (this.topicName != topicName) + if (!IsSubscribed || this.topicName != topicName) { return; } @@ -41,7 +41,7 @@ public void Publish(byte[] value) public void Unsubscribe() { - if (!IsSubscribed) router.Unsubscribe(topicName); + if (IsSubscribed) router.Unsubscribe(topicName); } public void Subscribe() From 41f5a0888e9596ec3b09db1ccf6c44a1daf0fc62 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 10:37:09 +0300 Subject: [PATCH 14/45] Complete pubsub topic unsubscribe lifecycle --- .../TopicLifecycleTests.cs | 40 +++++++++ .../PubsubRouter.Topics.cs | 83 ++++++++++--------- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 19 ++++- 3 files changed, 98 insertions(+), 44 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs index a816d787..4f846521 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -57,6 +57,7 @@ public async Task Topic_ResubscribeRetainsRemoteMembership() router.OutboundConnection(peerAddress, PubsubRouter.GossipsubProtocolVersionV11, connectionClosed.Task, sent.Add); router.OnRpc(peerId, new Rpc().WithTopics([topicName], [])); + router.OnRpc(peerId, CreateMessage(topicName, TestPeers.Identity(2), 1)); sent.Clear(); topic.Unsubscribe(); @@ -68,6 +69,9 @@ public async Task Topic_ResubscribeRetainsRemoteMembership() }); sent.Clear(); + await state.Heartbeat(); + Assert.That(sent, Is.Empty); + topic.Subscribe(); await state.Heartbeat(); @@ -81,6 +85,42 @@ public async Task Topic_ResubscribeRetainsRemoteMembership() connectionClosed.SetResult(); } + [Test] + public void Topic_UnsubscribeNotifiesAllConnectedPeersAndIsNotReadvertised() + { + const string topicName = "topic-lifecycle"; + PubsubRouter router = new(new PeerStore()); + ITopic topic = router.GetTopic(topicName); + TaskCompletionSource firstConnectionClosed = new(); + TaskCompletionSource secondConnectionClosed = new(); + TaskCompletionSource thirdConnectionClosed = new(); + List firstSent = []; + List secondSent = []; + List thirdSent = []; + Multiaddress firstPeerAddress = TestPeers.Multiaddr(3); + + router.OutboundConnection(firstPeerAddress, PubsubRouter.GossipsubProtocolVersionV11, firstConnectionClosed.Task, firstSent.Add); + router.OutboundConnection(TestPeers.Multiaddr(4), PubsubRouter.GossipsubProtocolVersionV11, secondConnectionClosed.Task, secondSent.Add); + router.OnRpc(firstPeerAddress.GetPeerId()!, new Rpc().WithTopics([topicName], [])); + firstSent.Clear(); + secondSent.Clear(); + + topic.Unsubscribe(); + + router.OutboundConnection(TestPeers.Multiaddr(5), PubsubRouter.GossipsubProtocolVersionV11, thirdConnectionClosed.Task, thirdSent.Add); + + Assert.Multiple(() => + { + Assert.That(firstSent.Any(rpc => rpc.Subscriptions.Any(subscription => !subscription.Subscribe && subscription.Topicid == topicName)), Is.True); + Assert.That(secondSent.Any(rpc => rpc.Subscriptions.Any(subscription => !subscription.Subscribe && subscription.Topicid == topicName)), Is.True); + Assert.That(thirdSent.Any(rpc => rpc.Subscriptions.Any(subscription => subscription.Subscribe && subscription.Topicid == topicName)), Is.False); + }); + + firstConnectionClosed.SetResult(); + secondConnectionClosed.SetResult(); + thirdConnectionClosed.SetResult(); + } + private static Rpc CreateMessage(string topicName, Identity author, ulong sequenceNumber) { return new Rpc().WithMessages(topicName, sequenceNumber, author.PeerId.Bytes, [1, 2, 3], author); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index 74f5f5d5..fbd27b48 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -27,75 +27,78 @@ public ITopic GetTopic(string topicId, bool subscribe = true) public void Subscribe(string topicId) { - Topic topic = topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)); - if (topic.IsSubscribed) + PubsubPeer[] peers; + lock (this) { - return; - } + Topic topic = topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)); + if (topic.IsSubscribed) + { + return; + } - topic.IsSubscribed = true; + topic.IsSubscribed = true; - fPeers.TryAdd(topicId, []); - gPeers.TryAdd(topicId, []); + fPeers.TryAdd(topicId, []); + gPeers.TryAdd(topicId, []); - HashSet meshPeers = mesh.GetOrAdd(topicId, []); + HashSet meshPeers = mesh.GetOrAdd(topicId, []); - if (fanout.TryGetValue(topicId, out HashSet? fanoutPeers)) - { - foreach (PeerId peerId in fanoutPeers.ToList()) + if (fanout.TryGetValue(topicId, out HashSet? fanoutPeers)) { - meshPeers.Add(peerId); + foreach (PeerId peerId in fanoutPeers.ToList()) + { + meshPeers.Add(peerId); + } + + fanoutPeers.Clear(); } - fanoutPeers.Clear(); + peers = peerState.Values.ToArray(); } Rpc topicUpdate = new Rpc().WithTopics([topicId], []); - foreach (KeyValuePair peer in peerState) + foreach (PubsubPeer peer in peers) { - peer.Value.Send(topicUpdate); + peer.Send(topicUpdate); } } public void Unsubscribe(string topicId) { - if (!topicState.TryGetValue(topicId, out Topic? topic) || !topic.IsSubscribed) + KeyValuePair[] peers; + HashSet? removedMesh; + lock (this) { - return; - } + if (!topicState.TryGetValue(topicId, out Topic? topic) || !topic.IsSubscribed) + { + return; + } - topic.IsSubscribed = false; + topic.IsSubscribed = false; - foreach (PeerId peerId in fPeers.GetValueOrDefault(topicId) ?? []) - { - Rpc msg = new Rpc() - .WithTopics([], [topicId]); + if (mesh.TryRemove(topicId, out removedMesh)) + { + foreach (PeerId peerId in removedMesh) + { + RecordPeerLeaveMesh(peerId, topicId); + } + } - peerState.GetValueOrDefault(peerId)?.Send(msg); + fanout.TryRemove(topicId, out _); + fanoutLastPublished.TryRemove(topicId, out _); + peers = peerState.ToArray(); } - foreach (PeerId peerId in gPeers.GetValueOrDefault(topicId) ?? []) + foreach ((PeerId peerId, PubsubPeer peer) in peers) { - Rpc msg = new Rpc() - .WithTopics([], [topicId]); - - if (mesh.TryGetValue(topicId, out HashSet? topicMesh) && topicMesh.Contains(peerId)) + Rpc msg = new Rpc().WithTopics([], [topicId]); + if (removedMesh?.Contains(peerId) is true) { msg.Ensure(r => r.Control.Prune).Add(new ControlPrune { TopicID = topicId }); } - peerState.GetValueOrDefault(peerId)?.Send(msg); - } - if (mesh.TryRemove(topicId, out HashSet? removedMesh)) - { - foreach (PeerId peerId in removedMesh) - { - RecordPeerLeaveMesh(peerId, topicId); - } + peer.Send(msg); } - - fanout.TryRemove(topicId, out _); - fanoutLastPublished.TryRemove(topicId, out _); } public void UnsubscribeAll() diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index 4a3f573d..e37ddc78 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -421,7 +421,9 @@ public Task Heartbeat() IEnumerable> msgs = _messageCache.ToList().GroupBy(m => m.Message.Topic); - foreach (string? topic in gPeers.Keys.Concat(fanout.Keys).Distinct().ToArray()) + foreach (string topic in gPeers.Keys.Concat(fanout.Keys).Distinct() + .Where(topic => topicState.GetValueOrDefault(topic)?.IsSubscribed is true) + .ToArray()) { IGrouping? msgsInTopic = msgs.FirstOrDefault(mit => mit.Key == topic); if (msgsInTopic is not null) @@ -430,9 +432,11 @@ public Task Heartbeat() ihave.MessageIDs.AddRange(msgsInTopic.Select(m => ByteString.CopyFrom(m.Id.Bytes))); // Only send gossip to peers above gossip threshold + HashSet? topicMesh = mesh.GetValueOrDefault(topic); + HashSet? topicFanout = fanout.GetValueOrDefault(topic); var eligiblePeers = gPeers[topic] - .Where(p => !mesh[topic].Contains(p) - && !fanout[topic].Contains(p) + .Where(p => !(topicMesh?.Contains(p) ?? false) + && !(topicFanout?.Contains(p) ?? false) && GetPeerScore(p) >= _settings.GossipThreshold); // Adaptive gossip: send to gossip_factor of eligible peers (min D_lazy) @@ -507,7 +511,14 @@ internal CancellationToken OutboundConnection(Multiaddress addr, string protocol reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); }); - string[] topics = topicState.Keys.ToArray(); + string[] topics; + lock (this) + { + topics = topicState + .Where(pair => pair.Value.IsSubscribed) + .Select(pair => pair.Key) + .ToArray(); + } if (topics.Any()) { From cfb12a20560723322edc3a20bbc0aabcabc3b75b Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 10:41:29 +0300 Subject: [PATCH 15/45] Use a clear topic lifecycle test name --- src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs index 4f846521..93758aad 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -86,7 +86,7 @@ public async Task Topic_ResubscribeRetainsRemoteMembership() } [Test] - public void Topic_UnsubscribeNotifiesAllConnectedPeersAndIsNotReadvertised() + public void Topic_UnsubscribeNotifiesAllConnectedPeersAndDoesNotAnnounceItToNewPeers() { const string topicName = "topic-lifecycle"; PubsubRouter router = new(new PeerStore()); From 8d4c668d40aa130ebec660168762220566155777 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 11:01:19 +0300 Subject: [PATCH 16/45] Gossip publish-only topics --- .../TopicLifecycleTests.cs | 34 +++++++++++++++++++ .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 6 ++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs index 93758aad..95f07390 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -121,6 +121,40 @@ public void Topic_UnsubscribeNotifiesAllConnectedPeersAndDoesNotAnnounceItToNewP thirdConnectionClosed.SetResult(); } + [Test] + public async Task PublishOnlyTopic_ContinuesGossiping() + { + const string topicName = "topic-lifecycle"; + PubsubSettings settings = new() { Degree = 1, LazyDegree = 1, GossipFactor = 1 }; + PubsubRouter router = new(new PeerStore(), settings); + IRoutingStateContainer state = router; + _ = router.GetTopic(topicName, subscribe: false); + Multiaddress fanoutPeerAddress = TestPeers.Multiaddr(3); + Multiaddress gossipPeerAddress = TestPeers.Multiaddr(4); + PeerId fanoutPeerId = fanoutPeerAddress.GetPeerId()!; + PeerId gossipPeerId = gossipPeerAddress.GetPeerId()!; + TaskCompletionSource fanoutConnectionClosed = new(); + TaskCompletionSource gossipConnectionClosed = new(); + List fanoutSent = []; + List gossipSent = []; + + router.OutboundConnection(fanoutPeerAddress, PubsubRouter.GossipsubProtocolVersionV11, fanoutConnectionClosed.Task, fanoutSent.Add); + router.OutboundConnection(gossipPeerAddress, PubsubRouter.GossipsubProtocolVersionV11, gossipConnectionClosed.Task, gossipSent.Add); + router.OnRpc(fanoutPeerId, new Rpc().WithTopics([topicName], [])); + router.OnRpc(gossipPeerId, new Rpc().WithTopics([topicName], [])); + state.Fanout.GetOrAdd(topicName, []).Add(fanoutPeerId); + router.OnRpc(fanoutPeerId, CreateMessage(topicName, TestPeers.Identity(5), 1)); + fanoutSent.Clear(); + gossipSent.Clear(); + + await state.Heartbeat(); + + Assert.That(gossipSent.Any(rpc => rpc.Control?.Ihave.Any(ihave => ihave.TopicID == topicName) is true), Is.True); + + fanoutConnectionClosed.SetResult(); + gossipConnectionClosed.SetResult(); + } + private static Rpc CreateMessage(string topicName, Identity author, ulong sequenceNumber) { return new Rpc().WithMessages(topicName, sequenceNumber, author.PeerId.Bytes, [1, 2, 3], author); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index e37ddc78..ad234800 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -422,11 +422,11 @@ public Task Heartbeat() IEnumerable> msgs = _messageCache.ToList().GroupBy(m => m.Message.Topic); foreach (string topic in gPeers.Keys.Concat(fanout.Keys).Distinct() - .Where(topic => topicState.GetValueOrDefault(topic)?.IsSubscribed is true) + .Where(topic => topicState.GetValueOrDefault(topic)?.IsSubscribed is true || fanout.ContainsKey(topic)) .ToArray()) { IGrouping? msgsInTopic = msgs.FirstOrDefault(mit => mit.Key == topic); - if (msgsInTopic is not null) + if (msgsInTopic is not null && gPeers.TryGetValue(topic, out HashSet? topicGossipsubPeers)) { ControlIHave ihave = new() { TopicID = topic }; ihave.MessageIDs.AddRange(msgsInTopic.Select(m => ByteString.CopyFrom(m.Id.Bytes))); @@ -434,7 +434,7 @@ public Task Heartbeat() // Only send gossip to peers above gossip threshold HashSet? topicMesh = mesh.GetValueOrDefault(topic); HashSet? topicFanout = fanout.GetValueOrDefault(topic); - var eligiblePeers = gPeers[topic] + var eligiblePeers = topicGossipsubPeers .Where(p => !(topicMesh?.Contains(p) ?? false) && !(topicFanout?.Contains(p) ?? false) && GetPeerScore(p) >= _settings.GossipThreshold); From 418d50137752afb15dd2df8a1312dd7e616b6901 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 11:09:19 +0300 Subject: [PATCH 17/45] Ignore gossip for inactive topics --- .../TopicLifecycleTests.cs | 24 +++++++++++++++++++ .../PubsubRouter.Rpc.cs | 16 ++++++------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs index 95f07390..54552102 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: MIT +using Google.Protobuf; using Multiformats.Address; using Nethermind.Libp2p.Core.Discovery; using Nethermind.Libp2p.Protocols.Pubsub; @@ -155,6 +156,29 @@ public async Task PublishOnlyTopic_ContinuesGossiping() gossipConnectionClosed.SetResult(); } + [Test] + public void UnsubscribedTopic_DoesNotRequestAdvertisedMessages() + { + const string topicName = "topic-lifecycle"; + PubsubRouter router = new(new PeerStore()); + ITopic topic = router.GetTopic(topicName); + Multiaddress peerAddress = TestPeers.Multiaddr(3); + TaskCompletionSource connectionClosed = new(); + List sent = []; + + router.OutboundConnection(peerAddress, PubsubRouter.GossipsubProtocolVersionV11, connectionClosed.Task, sent.Add); + topic.Unsubscribe(); + sent.Clear(); + + Rpc rpc = new() { Control = new ControlMessage() }; + rpc.Control.Ihave.Add(new ControlIHave { TopicID = topicName, MessageIDs = { ByteString.CopyFrom([1]) } }); + router.OnRpc(peerAddress.GetPeerId()!, rpc); + + Assert.That(sent, Is.Empty); + + connectionClosed.SetResult(); + } + private static Rpc CreateMessage(string topicName, Identity author, ulong sequenceNumber) { return new Rpc().WithMessages(topicName, sequenceNumber, author.PeerId.Bytes, [1, 2, 3], author); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index 960159f5..d552099b 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -210,10 +210,10 @@ private void HandleGraft(PeerId peerId, IEnumerable grafts, Concur { foreach (ControlGraft? graft in grafts) { - if (!topicState.ContainsKey(graft.TopicID)) + if (topicState.GetValueOrDefault(graft.TopicID)?.IsSubscribed is not true || + !mesh.TryGetValue(graft.TopicID, out HashSet? topicMesh)) { - // Ignore GRAFT for unknown topics (spam protection) - logger?.LogDebug("Ignoring GRAFT from {peerId} for unknown topic {topic}", peerId, graft.TopicID); + logger?.LogDebug("Ignoring GRAFT from {peerId} for inactive topic {topic}", peerId, graft.TopicID); continue; } @@ -233,8 +233,6 @@ private void HandleGraft(PeerId peerId, IEnumerable grafts, Concur } } - HashSet topicMesh = mesh[graft.TopicID]; - if (topicMesh.Count >= _settings.HighestDegree) { ControlPrune prune = new() { TopicID = graft.TopicID, Backoff = (ulong)(_settings.PruneBackoff / 1000) }; @@ -278,14 +276,16 @@ private void HandlePrune(PeerId peerId, IEnumerable prunes, Concur { foreach (ControlPrune? prune in prunes) { - if (topicState.ContainsKey(prune.TopicID) && mesh[prune.TopicID].Contains(peerId)) + if (topicState.GetValueOrDefault(prune.TopicID)?.IsSubscribed is true && + mesh.TryGetValue(prune.TopicID, out HashSet? topicMesh) && + topicMesh.Contains(peerId)) { if (peerState.TryGetValue(peerId, out PubsubPeer? state)) { ulong backoffSeconds = prune.Backoff == 0 ? (ulong)(_settings.PruneBackoff / 1000) : prune.Backoff; state.Backoff[prune.TopicID] = DateTime.Now.AddSeconds(backoffSeconds); } - mesh[prune.TopicID].Remove(peerId); + topicMesh.Remove(peerId); RecordPeerLeaveMesh(peerId, prune.TopicID); // Track for scoring (P1 and P3b) // Handle PX (Peer Exchange) only if peer score is above threshold @@ -306,7 +306,7 @@ private void HandleIhave(PeerId peerId, IEnumerable ihaves, Concur { List messageIds = []; - foreach (ControlIHave? ihave in ihaves.Where(iw => topicState.ContainsKey(iw.TopicID))) + foreach (ControlIHave? ihave in ihaves.Where(iw => topicState.GetValueOrDefault(iw.TopicID)?.IsSubscribed is true)) { messageIds.AddRange(ihave.MessageIDs.Select(m => new MessageId(m.ToByteArray())) .Where(mid => !_messageCache.Contains(mid))); From ab72e3e1a08e3b0af6305bcd1731ad2426f62bb5 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 11:13:39 +0300 Subject: [PATCH 18/45] Remove stale pubsub fanout state --- .../TopicLifecycleTests.cs | 19 +++++++++++++++++++ .../PubsubRouter.Topics.cs | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs index 54552102..07db24dd 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -179,6 +179,25 @@ public void UnsubscribedTopic_DoesNotRequestAdvertisedMessages() connectionClosed.SetResult(); } + [Test] + public void Topic_SubscribeMovesFanoutPeersIntoTheMesh() + { + const string topicName = "topic-lifecycle"; + PubsubRouter router = new(new PeerStore()); + IRoutingStateContainer state = router; + ITopic topic = router.GetTopic(topicName, subscribe: false); + PeerId fanoutPeer = TestPeers.PeerId(3); + state.Fanout.GetOrAdd(topicName, []).Add(fanoutPeer); + + topic.Subscribe(); + + Assert.Multiple(() => + { + Assert.That(state.Mesh[topicName], Has.Member(fanoutPeer)); + Assert.That(state.Fanout, Does.Not.ContainKey(topicName)); + }); + } + private static Rpc CreateMessage(string topicName, Identity author, ulong sequenceNumber) { return new Rpc().WithMessages(topicName, sequenceNumber, author.PeerId.Bytes, [1, 2, 3], author); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index fbd27b48..b6fbd0a0 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -43,14 +43,14 @@ public void Subscribe(string topicId) HashSet meshPeers = mesh.GetOrAdd(topicId, []); - if (fanout.TryGetValue(topicId, out HashSet? fanoutPeers)) + if (fanout.TryRemove(topicId, out HashSet? fanoutPeers)) { foreach (PeerId peerId in fanoutPeers.ToList()) { meshPeers.Add(peerId); } - fanoutPeers.Clear(); + fanoutLastPublished.TryRemove(topicId, out _); } peers = peerState.Values.ToArray(); From 344edad2356198e63a4c2ac44e02f1ccc9f4880e Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 13:41:57 +0300 Subject: [PATCH 19/45] Serialize pubsub topic lifecycle transitions --- .../PubsubRouter.Topics.cs | 37 +++++++++---------- src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs | 13 ++++--- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index b6fbd0a0..d55e514f 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -27,7 +27,6 @@ public ITopic GetTopic(string topicId, bool subscribe = true) public void Subscribe(string topicId) { - PubsubPeer[] peers; lock (this) { Topic topic = topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)); @@ -47,25 +46,25 @@ public void Subscribe(string topicId) { foreach (PeerId peerId in fanoutPeers.ToList()) { - meshPeers.Add(peerId); + if (meshPeers.Add(peerId)) + { + RecordPeerJoinMesh(peerId, topicId); + } } fanoutLastPublished.TryRemove(topicId, out _); } - peers = peerState.Values.ToArray(); - } - - Rpc topicUpdate = new Rpc().WithTopics([topicId], []); - foreach (PubsubPeer peer in peers) - { - peer.Send(topicUpdate); + Rpc topicUpdate = new Rpc().WithTopics([topicId], []); + foreach (PubsubPeer peer in peerState.Values) + { + peer.Send(topicUpdate); + } } } public void Unsubscribe(string topicId) { - KeyValuePair[] peers; HashSet? removedMesh; lock (this) { @@ -86,18 +85,16 @@ public void Unsubscribe(string topicId) fanout.TryRemove(topicId, out _); fanoutLastPublished.TryRemove(topicId, out _); - peers = peerState.ToArray(); - } - - foreach ((PeerId peerId, PubsubPeer peer) in peers) - { - Rpc msg = new Rpc().WithTopics([], [topicId]); - if (removedMesh?.Contains(peerId) is true) + foreach ((PeerId peerId, PubsubPeer peer) in peerState) { - msg.Ensure(r => r.Control.Prune).Add(new ControlPrune { TopicID = topicId }); - } + Rpc msg = new Rpc().WithTopics([], [topicId]); + if (removedMesh?.Contains(peerId) is true) + { + msg.Ensure(r => r.Control.Prune).Add(new ControlPrune { TopicID = topicId }); + } - peer.Send(msg); + peer.Send(msg); + } } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs index 18f838dd..811b830d 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs @@ -19,13 +19,16 @@ public Topic(PubsubRouter router, string topicName) private void OnRouterMessage(string topicName, PeerId peerId, byte[] message) { - if (!IsSubscribed || this.topicName != topicName) + lock (router) { - return; - } + if (!IsSubscribed || this.topicName != topicName) + { + return; + } - Action? onMessage = OnMessage; - onMessage?.Invoke(peerId, message); + Action? onMessage = OnMessage; + onMessage?.Invoke(peerId, message); + } } public DateTime LastPublished { get; set; } From 367e14666a667924e6087e73333b38b602e21702 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 14:03:58 +0300 Subject: [PATCH 20/45] Serialize pubsub publish routing --- .../TopicLifecycleTests.cs | 22 +++++ .../PubsubRouter.Topics.cs | 82 +++++++++---------- 2 files changed, 63 insertions(+), 41 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs index 07db24dd..e388f520 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -156,6 +156,28 @@ public async Task PublishOnlyTopic_ContinuesGossiping() gossipConnectionClosed.SetResult(); } + [Test] + public async Task PublishWithoutSubscriptionUsesGossipsubFanout() + { + const string topicName = "topic-lifecycle"; + PubsubRouter router = new(new PeerStore()); + await router.StartAsync(new LocalPeerStub()); + _ = router.GetTopic(topicName, subscribe: false); + Multiaddress peerAddress = TestPeers.Multiaddr(3); + PeerId peerId = peerAddress.GetPeerId()!; + TaskCompletionSource connectionClosed = new(); + List sent = []; + + router.OutboundConnection(peerAddress, PubsubRouter.GossipsubProtocolVersionV11, connectionClosed.Task, sent.Add); + router.OnRpc(peerId, new Rpc().WithTopics([topicName], [])); + sent.Clear(); + + router.Publish(topicName, [1, 2, 3]); + + Assert.That(sent.SelectMany(rpc => rpc.Publish).Count(), Is.EqualTo(1)); + connectionClosed.SetResult(); + } + [Test] public void UnsubscribedTopic_DoesNotRequestAdvertisedMessages() { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index d55e514f..a51ac635 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.Logging; using Nethermind.Libp2p.Core; using Nethermind.Libp2p.Protocols.Pubsub.Dto; -using System.Buffers.Binary; using System.Collections.Concurrent; namespace Nethermind.Libp2p.Protocols.Pubsub; @@ -119,37 +118,24 @@ public void Publish(string topicId, byte[] message) throw new InvalidOperationException("Router has not been started. Call StartAsync() first."); } - topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)); - - ulong seqNo = this.seqNo++; - Span seqNoBytes = stackalloc byte[8]; - BinaryPrimitives.WriteUInt64BigEndian(seqNoBytes, seqNo); - Rpc rpc = new Rpc().WithMessages(topicId, seqNo, localPeer.Identity.PeerId.Bytes, message, localPeer.Identity); - - // Floodsub peers always get the message - foreach (PeerId peerId in fPeers[topicId]) + lock (this) { - peerState.GetValueOrDefault(peerId)?.Send(rpc); - } + topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)); - // Gossipsub v1.1: Flood publishing - if (_settings.FloodPublish && gPeers.TryGetValue(topicId, out HashSet? allGossipsubPeers)) - { - // Send to all gossipsub peers above publish threshold - foreach (PeerId peerId in allGossipsubPeers) + ulong seqNo = this.seqNo++; + Rpc rpc = new Rpc().WithMessages(topicId, seqNo, localPeer.Identity.PeerId.Bytes, message, localPeer.Identity); + + // Floodsub peers always get the message + foreach (PeerId peerId in fPeers.GetValueOrDefault(topicId) ?? []) { - if (GetPeerScore(peerId) >= _settings.PublishThreshold) - { - peerState.GetValueOrDefault(peerId)?.Send(rpc); - } + peerState.GetValueOrDefault(peerId)?.Send(rpc); } - } - else - { - // Standard gossipsub v1.0 behavior: send to mesh or fanout - if (mesh.ContainsKey(topicId)) + + // Gossipsub v1.1: Flood publishing + if (_settings.FloodPublish && gPeers.TryGetValue(topicId, out HashSet? allGossipsubPeers)) { - foreach (PeerId peerId in mesh[topicId].ToList()) + // Send to all gossipsub peers above publish threshold + foreach (PeerId peerId in allGossipsubPeers) { if (GetPeerScore(peerId) >= _settings.PublishThreshold) { @@ -159,28 +145,42 @@ public void Publish(string topicId, byte[] message) } else { - fanoutLastPublished[topicId] = DateTime.Now; - HashSet topicFanout = fanout.GetOrAdd(topicId, _ => []); - - if (topicFanout.Count == 0) + // Standard gossipsub v1.0 behavior: send to mesh or fanout + if (mesh.TryGetValue(topicId, out HashSet? meshPeers)) { - HashSet? topicPeers = gPeers.GetValueOrDefault(topicId); - if (topicPeers is { Count: > 0 }) + foreach (PeerId peerId in meshPeers) { - // Select peers with non-negative scores - var eligiblePeers = topicPeers.Where(p => GetPeerScore(p) >= 0).ToList(); - foreach (PeerId peer in eligiblePeers.Take(_settings.Degree)) + if (GetPeerScore(peerId) >= _settings.PublishThreshold) { - topicFanout.Add(peer); + peerState.GetValueOrDefault(peerId)?.Send(rpc); } } } - - foreach (PeerId peerId in topicFanout) + else { - if (GetPeerScore(peerId) >= _settings.PublishThreshold) + fanoutLastPublished[topicId] = DateTime.Now; + HashSet topicFanout = fanout.GetOrAdd(topicId, _ => []); + + if (topicFanout.Count == 0) { - peerState.GetValueOrDefault(peerId)?.Send(rpc); + HashSet? topicPeers = gPeers.GetValueOrDefault(topicId); + if (topicPeers is { Count: > 0 }) + { + // Select peers with non-negative scores + var eligiblePeers = topicPeers.Where(p => GetPeerScore(p) >= 0).ToList(); + foreach (PeerId peer in eligiblePeers.Take(_settings.Degree)) + { + topicFanout.Add(peer); + } + } + } + + foreach (PeerId peerId in topicFanout) + { + if (GetPeerScore(peerId) >= _settings.PublishThreshold) + { + peerState.GetValueOrDefault(peerId)?.Send(rpc); + } } } } From 063c3fe84ff16cd73bf18e6b4ace157b8e6bfbe4 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 11:46:36 +0300 Subject: [PATCH 21/45] Add Gossipsub v1.3 extensions control support --- .../GossipsubV13ProtocolTests.cs | 98 ++ src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs | 839 +++++++++++++++++- .../Libp2p.Protocols.Pubsub/Dto/Rpc.proto | 29 +- .../Libp2p.Protocols.Pubsub/PubsubProtocol.cs | 2 + .../PubsubRouter.Rpc.cs | 26 +- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 14 +- src/libp2p/Libp2p/Libp2pPeerFactoryBuilder.cs | 1 + 7 files changed, 983 insertions(+), 26 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs new file mode 100644 index 00000000..2144933e --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Google.Protobuf; +using Multiformats.Address; +using Nethermind.Libp2p.Core.Discovery; +using Nethermind.Libp2p.Protocols; +using Nethermind.Libp2p.Protocols.Pubsub.Dto; +using System.Collections.ObjectModel; + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +public class GossipsubV13ProtocolTests +{ + [Test] + public void Protocol_UsesTheV13ProtocolId() + { + PubsubRouter router = new(new PeerStore()); + + GossipsubProtocolV13 protocol = new(router); + + Assert.That(protocol.Id, Is.EqualTo(PubsubRouter.GossipsubProtocolVersionV13)); + } + + [Test] + public void ExtensionRegistryMessages_RoundTrip() + { + Rpc rpc = new() + { + Control = new ControlMessage + { + Extensions = new ControlExtensions { PartialMessages = true }, + }, + Partial = new PartialMessagesExtension + { + TopicID = "topic", + GroupID = ByteString.CopyFrom([1, 2]), + PartialMessage = ByteString.CopyFrom([3]), + PartsMetadata = ByteString.CopyFrom([4]), + }, + }; + rpc.Subscriptions.Add(new Rpc.Types.SubOpts + { + Subscribe = true, + Topicid = "topic", + RequestsPartial = true, + SupportsSendingPartial = true, + }); + + Rpc decoded = Rpc.Parser.ParseFrom(rpc.ToByteArray()); + + Assert.Multiple(() => + { + Assert.That(decoded.Control.Extensions.PartialMessages, Is.True); + Assert.That(decoded.Partial.TopicID, Is.EqualTo("topic")); + Assert.That(decoded.Partial.GroupID.ToByteArray(), Is.EqualTo(new byte[] { 1, 2 })); + Assert.That(decoded.Subscriptions.Single().RequestsPartial, Is.True); + Assert.That(decoded.Subscriptions.Single().SupportsSendingPartial, Is.True); + }); + } + + [Test] + public async Task Router_PrefersV13WhenTheRemotePeerAdvertisesIt() + { + PeerStore peerStore = new(); + PubsubRouter router = new(peerStore); + Multiaddress remoteAddress = TestPeers.Multiaddr(1); + PeerId remotePeerId = remoteAddress.GetPeerId()!; + peerStore.GetPeerInfo(remotePeerId).SupportedProtocols = [ + PubsubRouter.GossipsubProtocolVersionV12, + PubsubRouter.GossipsubProtocolVersionV13, + ]; + + TaskCompletionSource selectedProtocol = new(TaskCreationOptions.RunContinuationsAsynchronously); + ISession session = Substitute.For(); + session.RemoteAddress.Returns(remoteAddress); + session.DialAsync(Arg.Any()).Returns(_ => + { + selectedProtocol.TrySetResult(); + return Task.CompletedTask; + }); + + ILocalPeer localPeer = Substitute.For(); + localPeer.Identity.Returns(TestPeers.Identity(2)); + localPeer.ListenAddresses.Returns(new ObservableCollection()); + localPeer.DialAsync(Arg.Any(), Arg.Any()).Returns(session); + + using CancellationTokenSource cancellation = new(); + await router.StartAsync(localPeer, cancellation.Token); + peerStore.Discover([remoteAddress]); + + await selectedProtocol.Task.WaitAsync(TimeSpan.FromSeconds(2)); + _ = session.Received(1).DialAsync(Arg.Any()); + + cancellation.Cancel(); + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs index 80d458ee..361f896f 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs @@ -24,36 +24,44 @@ public static partial class RpcReflection { static RpcReflection() { byte[] descriptorData = global::System.Convert.FromBase64String( string.Concat( - "CglScGMucHJvdG8ilgEKA1JwYxIjCg1zdWJzY3JpcHRpb25zGAEgAygLMgwu", + "CglScGMucHJvdG8i+wEKA1JwYxIjCg1zdWJzY3JpcHRpb25zGAEgAygLMgwu", "UnBjLlN1Yk9wdHMSGQoHcHVibGlzaBgCIAMoCzIILk1lc3NhZ2USIAoHY29u", - "dHJvbBgDIAEoCzIPLkNvbnRyb2xNZXNzYWdlGi0KB1N1Yk9wdHMSEQoJc3Vi", - "c2NyaWJlGAEgASgIEg8KB3RvcGljaWQYAiABKAkiYwoHTWVzc2FnZRIMCgRm", - "cm9tGAEgASgMEgwKBGRhdGEYAiABKAwSDQoFc2Vxbm8YAyABKAwSDQoFdG9w", - "aWMYBCACKAkSEQoJc2lnbmF0dXJlGAUgASgMEgsKA2tleRgGIAEoDCKuAQoO", - "Q29udHJvbE1lc3NhZ2USHAoFaWhhdmUYASADKAsyDS5Db250cm9sSUhhdmUS", - "HAoFaXdhbnQYAiADKAsyDS5Db250cm9sSVdhbnQSHAoFZ3JhZnQYAyADKAsy", - "DS5Db250cm9sR3JhZnQSHAoFcHJ1bmUYBCADKAsyDS5Db250cm9sUHJ1bmUS", - "JAoJaWRvbnR3YW50GAUgAygLMhEuQ29udHJvbElEb250V2FudCIzCgxDb250", - "cm9sSUhhdmUSDwoHdG9waWNJRBgBIAEoCRISCgptZXNzYWdlSURzGAIgAygM", - "IiIKDENvbnRyb2xJV2FudBISCgptZXNzYWdlSURzGAEgAygMIh8KDENvbnRy", - "b2xHcmFmdBIPCgd0b3BpY0lEGAEgASgJIkoKDENvbnRyb2xQcnVuZRIPCgd0", - "b3BpY0lEGAEgASgJEhgKBXBlZXJzGAIgAygLMgkuUGVlckluZm8SDwoHYmFj", - "a29mZhgDIAEoBCI0CghQZWVySW5mbxIOCgZwZWVySUQYASABKAwSGAoQc2ln", - "bmVkUGVlclJlY29yZBgCIAEoDCImChBDb250cm9sSURvbnRXYW50EhIKCm1l", - "c3NhZ2VJRHMYASADKAxCKaoCJk5ldGhlcm1pbmQuTGlicDJwLlByb3RvY29s", - "cy5QdWJzdWIuRHRv")); + "dHJvbBgDIAEoCzIPLkNvbnRyb2xNZXNzYWdlEioKB3BhcnRpYWwYCiABKAsy", + "GS5QYXJ0aWFsTWVzc2FnZXNFeHRlbnNpb24aZgoHU3ViT3B0cxIRCglzdWJz", + "Y3JpYmUYASABKAgSDwoHdG9waWNpZBgCIAEoCRIXCg9yZXF1ZXN0c1BhcnRp", + "YWwYAyABKAgSHgoWc3VwcG9ydHNTZW5kaW5nUGFydGlhbBgEIAEoCCJjCgdN", + "ZXNzYWdlEgwKBGZyb20YASABKAwSDAoEZGF0YRgCIAEoDBINCgVzZXFubxgD", + "IAEoDBINCgV0b3BpYxgEIAIoCRIRCglzaWduYXR1cmUYBSABKAwSCwoDa2V5", + "GAYgASgMItYBCg5Db250cm9sTWVzc2FnZRIcCgVpaGF2ZRgBIAMoCzINLkNv", + "bnRyb2xJSGF2ZRIcCgVpd2FudBgCIAMoCzINLkNvbnRyb2xJV2FudBIcCgVn", + "cmFmdBgDIAMoCzINLkNvbnRyb2xHcmFmdBIcCgVwcnVuZRgEIAMoCzINLkNv", + "bnRyb2xQcnVuZRIkCglpZG9udHdhbnQYBSADKAsyES5Db250cm9sSURvbnRX", + "YW50EiYKCmV4dGVuc2lvbnMYBiABKAsyEi5Db250cm9sRXh0ZW5zaW9ucyIz", + "CgxDb250cm9sSUhhdmUSDwoHdG9waWNJRBgBIAEoCRISCgptZXNzYWdlSURz", + "GAIgAygMIiIKDENvbnRyb2xJV2FudBISCgptZXNzYWdlSURzGAEgAygMIh8K", + "DENvbnRyb2xHcmFmdBIPCgd0b3BpY0lEGAEgASgJIkoKDENvbnRyb2xQcnVu", + "ZRIPCgd0b3BpY0lEGAEgASgJEhgKBXBlZXJzGAIgAygLMgkuUGVlckluZm8S", + "DwoHYmFja29mZhgDIAEoBCI0CghQZWVySW5mbxIOCgZwZWVySUQYASABKAwS", + "GAoQc2lnbmVkUGVlclJlY29yZBgCIAEoDCImChBDb250cm9sSURvbnRXYW50", + "EhIKCm1lc3NhZ2VJRHMYASADKAwiLAoRQ29udHJvbEV4dGVuc2lvbnMSFwoP", + "cGFydGlhbE1lc3NhZ2VzGAogASgIImsKGFBhcnRpYWxNZXNzYWdlc0V4dGVu", + "c2lvbhIPCgd0b3BpY0lEGAEgASgJEg8KB2dyb3VwSUQYAiABKAwSFgoOcGFy", + "dGlhbE1lc3NhZ2UYAyABKAwSFQoNcGFydHNNZXRhZGF0YRgEIAEoDEIpqgIm", + "TmV0aGVybWluZC5MaWJwMnAuUHJvdG9jb2xzLlB1YnN1Yi5EdG8=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(null, null, new pbr::GeneratedClrTypeInfo[] { - new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc.Parser, new[]{ "Subscriptions", "Publish", "Control" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc.Types.SubOpts), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc.Types.SubOpts.Parser, new[]{ "Subscribe", "Topicid" }, null, null, null, null)}), + new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc.Parser, new[]{ "Subscriptions", "Publish", "Control", "Partial" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc.Types.SubOpts), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Rpc.Types.SubOpts.Parser, new[]{ "Subscribe", "Topicid", "RequestsPartial", "SupportsSendingPartial" }, null, null, null, null)}), new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Message), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.Message.Parser, new[]{ "From", "Data", "Seqno", "Topic", "Signature", "Key" }, null, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlMessage), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlMessage.Parser, new[]{ "Ihave", "Iwant", "Graft", "Prune", "Idontwant" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlMessage), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlMessage.Parser, new[]{ "Ihave", "Iwant", "Graft", "Prune", "Idontwant", "Extensions" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIHave), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIHave.Parser, new[]{ "TopicID", "MessageIDs" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIWant), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIWant.Parser, new[]{ "MessageIDs" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlGraft), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlGraft.Parser, new[]{ "TopicID" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlPrune), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlPrune.Parser, new[]{ "TopicID", "Peers", "Backoff" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PeerInfo), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PeerInfo.Parser, new[]{ "PeerID", "SignedPeerRecord" }, null, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIDontWant), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIDontWant.Parser, new[]{ "MessageIDs" }, null, null, null, null) + new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIDontWant), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlIDontWant.Parser, new[]{ "MessageIDs" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlExtensions), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlExtensions.Parser, new[]{ "PartialMessages" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PartialMessagesExtension), global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PartialMessagesExtension.Parser, new[]{ "TopicID", "GroupID", "PartialMessage", "PartsMetadata" }, null, null, null, null) })); } #endregion @@ -98,6 +106,7 @@ public Rpc(Rpc other) : this() { subscriptions_ = other.subscriptions_.Clone(); publish_ = other.publish_.Clone(); control_ = other.control_ != null ? other.control_.Clone() : null; + partial_ = other.partial_ != null ? other.partial_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -141,6 +150,21 @@ public Rpc Clone() { } } + /// Field number for the "partial" field. + public const int PartialFieldNumber = 10; + private global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PartialMessagesExtension partial_; + /// + /// Canonical Gossipsub v1.3 extensions are registered here. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PartialMessagesExtension Partial { + get { return partial_; } + set { + partial_ = value; + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override bool Equals(object other) { @@ -159,6 +183,7 @@ public bool Equals(Rpc other) { if(!subscriptions_.Equals(other.subscriptions_)) return false; if(!publish_.Equals(other.publish_)) return false; if (!object.Equals(Control, other.Control)) return false; + if (!object.Equals(Partial, other.Partial)) return false; return Equals(_unknownFields, other._unknownFields); } @@ -169,6 +194,7 @@ public override int GetHashCode() { hash ^= subscriptions_.GetHashCode(); hash ^= publish_.GetHashCode(); if (control_ != null) hash ^= Control.GetHashCode(); + if (partial_ != null) hash ^= Partial.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -193,6 +219,10 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(26); output.WriteMessage(Control); } + if (partial_ != null) { + output.WriteRawTag(82); + output.WriteMessage(Partial); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -209,6 +239,10 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(26); output.WriteMessage(Control); } + if (partial_ != null) { + output.WriteRawTag(82); + output.WriteMessage(Partial); + } if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -224,6 +258,9 @@ public int CalculateSize() { if (control_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(Control); } + if (partial_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(Partial); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -244,6 +281,12 @@ public void MergeFrom(Rpc other) { } Control.MergeFrom(other.Control); } + if (other.partial_ != null) { + if (partial_ == null) { + Partial = new global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PartialMessagesExtension(); + } + Partial.MergeFrom(other.Partial); + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -278,6 +321,13 @@ public void MergeFrom(pb::CodedInputStream input) { input.ReadMessage(Control); break; } + case 82: { + if (partial_ == null) { + Partial = new global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PartialMessagesExtension(); + } + input.ReadMessage(Partial); + break; + } } } #endif @@ -312,6 +362,13 @@ public void MergeFrom(pb::CodedInputStream input) { input.ReadMessage(Control); break; } + case 82: { + if (partial_ == null) { + Partial = new global::Nethermind.Libp2p.Protocols.Pubsub.Dto.PartialMessagesExtension(); + } + input.ReadMessage(Partial); + break; + } } } } @@ -361,6 +418,8 @@ public SubOpts(SubOpts other) : this() { _hasBits0 = other._hasBits0; subscribe_ = other.subscribe_; topicid_ = other.topicid_; + requestsPartial_ = other.requestsPartial_; + supportsSendingPartial_ = other.supportsSendingPartial_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -423,6 +482,63 @@ public void ClearTopicid() { topicid_ = null; } + /// Field number for the "requestsPartial" field. + public const int RequestsPartialFieldNumber = 3; + private readonly static bool RequestsPartialDefaultValue = false; + + private bool requestsPartial_; + /// + /// Used with the Gossipsub v1.3 Partial Messages extension. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool RequestsPartial { + get { if ((_hasBits0 & 2) != 0) { return requestsPartial_; } else { return RequestsPartialDefaultValue; } } + set { + _hasBits0 |= 2; + requestsPartial_ = value; + } + } + /// Gets whether the "requestsPartial" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasRequestsPartial { + get { return (_hasBits0 & 2) != 0; } + } + /// Clears the value of the "requestsPartial" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearRequestsPartial() { + _hasBits0 &= ~2; + } + + /// Field number for the "supportsSendingPartial" field. + public const int SupportsSendingPartialFieldNumber = 4; + private readonly static bool SupportsSendingPartialDefaultValue = false; + + private bool supportsSendingPartial_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool SupportsSendingPartial { + get { if ((_hasBits0 & 4) != 0) { return supportsSendingPartial_; } else { return SupportsSendingPartialDefaultValue; } } + set { + _hasBits0 |= 4; + supportsSendingPartial_ = value; + } + } + /// Gets whether the "supportsSendingPartial" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasSupportsSendingPartial { + get { return (_hasBits0 & 4) != 0; } + } + /// Clears the value of the "supportsSendingPartial" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearSupportsSendingPartial() { + _hasBits0 &= ~4; + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override bool Equals(object other) { @@ -440,6 +556,8 @@ public bool Equals(SubOpts other) { } if (Subscribe != other.Subscribe) return false; if (Topicid != other.Topicid) return false; + if (RequestsPartial != other.RequestsPartial) return false; + if (SupportsSendingPartial != other.SupportsSendingPartial) return false; return Equals(_unknownFields, other._unknownFields); } @@ -449,6 +567,8 @@ public override int GetHashCode() { int hash = 1; if (HasSubscribe) hash ^= Subscribe.GetHashCode(); if (HasTopicid) hash ^= Topicid.GetHashCode(); + if (HasRequestsPartial) hash ^= RequestsPartial.GetHashCode(); + if (HasSupportsSendingPartial) hash ^= SupportsSendingPartial.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -475,6 +595,14 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(18); output.WriteString(Topicid); } + if (HasRequestsPartial) { + output.WriteRawTag(24); + output.WriteBool(RequestsPartial); + } + if (HasSupportsSendingPartial) { + output.WriteRawTag(32); + output.WriteBool(SupportsSendingPartial); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -493,6 +621,14 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(18); output.WriteString(Topicid); } + if (HasRequestsPartial) { + output.WriteRawTag(24); + output.WriteBool(RequestsPartial); + } + if (HasSupportsSendingPartial) { + output.WriteRawTag(32); + output.WriteBool(SupportsSendingPartial); + } if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -509,6 +645,12 @@ public int CalculateSize() { if (HasTopicid) { size += 1 + pb::CodedOutputStream.ComputeStringSize(Topicid); } + if (HasRequestsPartial) { + size += 1 + 1; + } + if (HasSupportsSendingPartial) { + size += 1 + 1; + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -527,6 +669,12 @@ public void MergeFrom(SubOpts other) { if (other.HasTopicid) { Topicid = other.Topicid; } + if (other.HasRequestsPartial) { + RequestsPartial = other.RequestsPartial; + } + if (other.HasSupportsSendingPartial) { + SupportsSendingPartial = other.SupportsSendingPartial; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -554,6 +702,14 @@ public void MergeFrom(pb::CodedInputStream input) { Topicid = input.ReadString(); break; } + case 24: { + RequestsPartial = input.ReadBool(); + break; + } + case 32: { + SupportsSendingPartial = input.ReadBool(); + break; + } } } #endif @@ -581,6 +737,14 @@ public void MergeFrom(pb::CodedInputStream input) { Topicid = input.ReadString(); break; } + case 24: { + RequestsPartial = input.ReadBool(); + break; + } + case 32: { + SupportsSendingPartial = input.ReadBool(); + break; + } } } } @@ -1100,6 +1264,7 @@ public ControlMessage(ControlMessage other) : this() { graft_ = other.graft_.Clone(); prune_ = other.prune_.Clone(); idontwant_ = other.idontwant_.Clone(); + extensions_ = other.extensions_ != null ? other.extensions_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1164,6 +1329,18 @@ public ControlMessage Clone() { get { return idontwant_; } } + /// Field number for the "extensions" field. + public const int ExtensionsFieldNumber = 6; + private global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlExtensions extensions_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlExtensions Extensions { + get { return extensions_; } + set { + extensions_ = value; + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override bool Equals(object other) { @@ -1184,6 +1361,7 @@ public bool Equals(ControlMessage other) { if(!graft_.Equals(other.graft_)) return false; if(!prune_.Equals(other.prune_)) return false; if(!idontwant_.Equals(other.idontwant_)) return false; + if (!object.Equals(Extensions, other.Extensions)) return false; return Equals(_unknownFields, other._unknownFields); } @@ -1196,6 +1374,7 @@ public override int GetHashCode() { hash ^= graft_.GetHashCode(); hash ^= prune_.GetHashCode(); hash ^= idontwant_.GetHashCode(); + if (extensions_ != null) hash ^= Extensions.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -1219,6 +1398,10 @@ public void WriteTo(pb::CodedOutputStream output) { graft_.WriteTo(output, _repeated_graft_codec); prune_.WriteTo(output, _repeated_prune_codec); idontwant_.WriteTo(output, _repeated_idontwant_codec); + if (extensions_ != null) { + output.WriteRawTag(50); + output.WriteMessage(Extensions); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -1234,6 +1417,10 @@ public void WriteTo(pb::CodedOutputStream output) { graft_.WriteTo(ref output, _repeated_graft_codec); prune_.WriteTo(ref output, _repeated_prune_codec); idontwant_.WriteTo(ref output, _repeated_idontwant_codec); + if (extensions_ != null) { + output.WriteRawTag(50); + output.WriteMessage(Extensions); + } if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -1249,6 +1436,9 @@ public int CalculateSize() { size += graft_.CalculateSize(_repeated_graft_codec); size += prune_.CalculateSize(_repeated_prune_codec); size += idontwant_.CalculateSize(_repeated_idontwant_codec); + if (extensions_ != null) { + size += 1 + pb::CodedOutputStream.ComputeMessageSize(Extensions); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -1266,6 +1456,12 @@ public void MergeFrom(ControlMessage other) { graft_.Add(other.graft_); prune_.Add(other.prune_); idontwant_.Add(other.idontwant_); + if (other.extensions_ != null) { + if (extensions_ == null) { + Extensions = new global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlExtensions(); + } + Extensions.MergeFrom(other.Extensions); + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1305,6 +1501,13 @@ public void MergeFrom(pb::CodedInputStream input) { idontwant_.AddEntriesFrom(input, _repeated_idontwant_codec); break; } + case 50: { + if (extensions_ == null) { + Extensions = new global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlExtensions(); + } + input.ReadMessage(Extensions); + break; + } } } #endif @@ -1344,6 +1547,13 @@ public void MergeFrom(pb::CodedInputStream input) { idontwant_.AddEntriesFrom(ref input, _repeated_idontwant_codec); break; } + case 50: { + if (extensions_ == null) { + Extensions = new global::Nethermind.Libp2p.Protocols.Pubsub.Dto.ControlExtensions(); + } + input.ReadMessage(Extensions); + break; + } } } } @@ -2736,6 +2946,595 @@ public void MergeFrom(pb::CodedInputStream input) { } + /// + /// Gossipsub v1.3 extension capabilities. Unknown fields are intentionally + /// ignored so future extensions remain wire compatible. + /// + [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] + public sealed partial class ControlExtensions : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new ControlExtensions()); + private pb::UnknownFieldSet _unknownFields; + private int _hasBits0; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Nethermind.Libp2p.Protocols.Pubsub.Dto.RpcReflection.Descriptor.MessageTypes[9]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ControlExtensions() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ControlExtensions(ControlExtensions other) : this() { + _hasBits0 = other._hasBits0; + partialMessages_ = other.partialMessages_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public ControlExtensions Clone() { + return new ControlExtensions(this); + } + + /// Field number for the "partialMessages" field. + public const int PartialMessagesFieldNumber = 10; + private readonly static bool PartialMessagesDefaultValue = false; + + private bool partialMessages_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool PartialMessages { + get { if ((_hasBits0 & 1) != 0) { return partialMessages_; } else { return PartialMessagesDefaultValue; } } + set { + _hasBits0 |= 1; + partialMessages_ = value; + } + } + /// Gets whether the "partialMessages" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasPartialMessages { + get { return (_hasBits0 & 1) != 0; } + } + /// Clears the value of the "partialMessages" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearPartialMessages() { + _hasBits0 &= ~1; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as ControlExtensions); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(ControlExtensions other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (PartialMessages != other.PartialMessages) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasPartialMessages) hash ^= PartialMessages.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasPartialMessages) { + output.WriteRawTag(80); + output.WriteBool(PartialMessages); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasPartialMessages) { + output.WriteRawTag(80); + output.WriteBool(PartialMessages); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasPartialMessages) { + size += 1 + 1; + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(ControlExtensions other) { + if (other == null) { + return; + } + if (other.HasPartialMessages) { + PartialMessages = other.PartialMessages; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 80: { + PartialMessages = input.ReadBool(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 80: { + PartialMessages = input.ReadBool(); + break; + } + } + } + } + #endif + + } + + /// + /// The Partial Messages extension uses an application-defined encoding for + /// message parts and metadata. topicID is a string to match the Go reference; + /// both it and the registry's bytes form use the same protobuf wire type. + /// + [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] + public sealed partial class PartialMessagesExtension : pb::IMessage + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + , pb::IBufferMessage + #endif + { + private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new PartialMessagesExtension()); + private pb::UnknownFieldSet _unknownFields; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pb::MessageParser Parser { get { return _parser; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public static pbr::MessageDescriptor Descriptor { + get { return global::Nethermind.Libp2p.Protocols.Pubsub.Dto.RpcReflection.Descriptor.MessageTypes[10]; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + pbr::MessageDescriptor pb::IMessage.Descriptor { + get { return Descriptor; } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public PartialMessagesExtension() { + OnConstruction(); + } + + partial void OnConstruction(); + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public PartialMessagesExtension(PartialMessagesExtension other) : this() { + topicID_ = other.topicID_; + groupID_ = other.groupID_; + partialMessage_ = other.partialMessage_; + partsMetadata_ = other.partsMetadata_; + _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public PartialMessagesExtension Clone() { + return new PartialMessagesExtension(this); + } + + /// Field number for the "topicID" field. + public const int TopicIDFieldNumber = 1; + private readonly static string TopicIDDefaultValue = ""; + + private string topicID_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string TopicID { + get { return topicID_ ?? TopicIDDefaultValue; } + set { + topicID_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "topicID" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasTopicID { + get { return topicID_ != null; } + } + /// Clears the value of the "topicID" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearTopicID() { + topicID_ = null; + } + + /// Field number for the "groupID" field. + public const int GroupIDFieldNumber = 2; + private readonly static pb::ByteString GroupIDDefaultValue = pb::ByteString.Empty; + + private pb::ByteString groupID_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pb::ByteString GroupID { + get { return groupID_ ?? GroupIDDefaultValue; } + set { + groupID_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "groupID" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasGroupID { + get { return groupID_ != null; } + } + /// Clears the value of the "groupID" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearGroupID() { + groupID_ = null; + } + + /// Field number for the "partialMessage" field. + public const int PartialMessageFieldNumber = 3; + private readonly static pb::ByteString PartialMessageDefaultValue = pb::ByteString.Empty; + + private pb::ByteString partialMessage_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pb::ByteString PartialMessage { + get { return partialMessage_ ?? PartialMessageDefaultValue; } + set { + partialMessage_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "partialMessage" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasPartialMessage { + get { return partialMessage_ != null; } + } + /// Clears the value of the "partialMessage" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearPartialMessage() { + partialMessage_ = null; + } + + /// Field number for the "partsMetadata" field. + public const int PartsMetadataFieldNumber = 4; + private readonly static pb::ByteString PartsMetadataDefaultValue = pb::ByteString.Empty; + + private pb::ByteString partsMetadata_; + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public pb::ByteString PartsMetadata { + get { return partsMetadata_ ?? PartsMetadataDefaultValue; } + set { + partsMetadata_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + /// Gets whether the "partsMetadata" field is set + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool HasPartsMetadata { + get { return partsMetadata_ != null; } + } + /// Clears the value of the "partsMetadata" field + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void ClearPartsMetadata() { + partsMetadata_ = null; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override bool Equals(object other) { + return Equals(other as PartialMessagesExtension); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public bool Equals(PartialMessagesExtension other) { + if (ReferenceEquals(other, null)) { + return false; + } + if (ReferenceEquals(other, this)) { + return true; + } + if (TopicID != other.TopicID) return false; + if (GroupID != other.GroupID) return false; + if (PartialMessage != other.PartialMessage) return false; + if (PartsMetadata != other.PartsMetadata) return false; + return Equals(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override int GetHashCode() { + int hash = 1; + if (HasTopicID) hash ^= TopicID.GetHashCode(); + if (HasGroupID) hash ^= GroupID.GetHashCode(); + if (HasPartialMessage) hash ^= PartialMessage.GetHashCode(); + if (HasPartsMetadata) hash ^= PartsMetadata.GetHashCode(); + if (_unknownFields != null) { + hash ^= _unknownFields.GetHashCode(); + } + return hash; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public override string ToString() { + return pb::JsonFormatter.ToDiagnosticString(this); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void WriteTo(pb::CodedOutputStream output) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + output.WriteRawMessage(this); + #else + if (HasTopicID) { + output.WriteRawTag(10); + output.WriteString(TopicID); + } + if (HasGroupID) { + output.WriteRawTag(18); + output.WriteBytes(GroupID); + } + if (HasPartialMessage) { + output.WriteRawTag(26); + output.WriteBytes(PartialMessage); + } + if (HasPartsMetadata) { + output.WriteRawTag(34); + output.WriteBytes(PartsMetadata); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(output); + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { + if (HasTopicID) { + output.WriteRawTag(10); + output.WriteString(TopicID); + } + if (HasGroupID) { + output.WriteRawTag(18); + output.WriteBytes(GroupID); + } + if (HasPartialMessage) { + output.WriteRawTag(26); + output.WriteBytes(PartialMessage); + } + if (HasPartsMetadata) { + output.WriteRawTag(34); + output.WriteBytes(PartsMetadata); + } + if (_unknownFields != null) { + _unknownFields.WriteTo(ref output); + } + } + #endif + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public int CalculateSize() { + int size = 0; + if (HasTopicID) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(TopicID); + } + if (HasGroupID) { + size += 1 + pb::CodedOutputStream.ComputeBytesSize(GroupID); + } + if (HasPartialMessage) { + size += 1 + pb::CodedOutputStream.ComputeBytesSize(PartialMessage); + } + if (HasPartsMetadata) { + size += 1 + pb::CodedOutputStream.ComputeBytesSize(PartsMetadata); + } + if (_unknownFields != null) { + size += _unknownFields.CalculateSize(); + } + return size; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(PartialMessagesExtension other) { + if (other == null) { + return; + } + if (other.HasTopicID) { + TopicID = other.TopicID; + } + if (other.HasGroupID) { + GroupID = other.GroupID; + } + if (other.HasPartialMessage) { + PartialMessage = other.PartialMessage; + } + if (other.HasPartsMetadata) { + PartsMetadata = other.PartsMetadata; + } + _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public void MergeFrom(pb::CodedInputStream input) { + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + input.ReadRawMessage(this); + #else + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); + break; + case 10: { + TopicID = input.ReadString(); + break; + } + case 18: { + GroupID = input.ReadBytes(); + break; + } + case 26: { + PartialMessage = input.ReadBytes(); + break; + } + case 34: { + PartsMetadata = input.ReadBytes(); + break; + } + } + } + #endif + } + + #if !GOOGLE_PROTOBUF_REFSTRUCT_COMPATIBILITY_MODE + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + void pb::IBufferMessage.InternalMergeFrom(ref pb::ParseContext input) { + uint tag; + while ((tag = input.ReadTag()) != 0) { + if ((tag & 7) == 4) { + // Abort on any end group tag. + return; + } + switch(tag) { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: { + TopicID = input.ReadString(); + break; + } + case 18: { + GroupID = input.ReadBytes(); + break; + } + case 26: { + PartialMessage = input.ReadBytes(); + break; + } + case 34: { + PartsMetadata = input.ReadBytes(); + break; + } + } + } + } + #endif + + } + #endregion } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto index d8730e4a..ba8c5f85 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto @@ -7,9 +7,15 @@ message Rpc { repeated Message publish = 2; optional ControlMessage control = 3; + // Canonical Gossipsub v1.3 extensions are registered here. + optional PartialMessagesExtension partial = 10; + message SubOpts { optional bool subscribe = 1; optional string topicid = 2; + // Used with the Gossipsub v1.3 Partial Messages extension. + optional bool requestsPartial = 3; + optional bool supportsSendingPartial = 4; } } @@ -28,6 +34,7 @@ message ControlMessage { repeated ControlGraft graft = 3; repeated ControlPrune prune = 4; repeated ControlIDontWant idontwant = 5; + optional ControlExtensions extensions = 6; } message ControlIHave { @@ -49,11 +56,27 @@ message ControlPrune { optional uint64 backoff = 3; // gossipsub v1.1 backoff time (in seconds) } +message PeerInfo { + optional bytes peerID = 1; + optional bytes signedPeerRecord = 2; +} + message ControlIDontWant { repeated bytes messageIDs = 1; } -message PeerInfo { - optional bytes peerID = 1; - optional SignedPeerRecordEnvelope signedPeerRecord = 2; +// Gossipsub v1.3 extension capabilities. Unknown fields are intentionally +// ignored so future extensions remain wire compatible. +message ControlExtensions { + optional bool partialMessages = 10; +} + +// The Partial Messages extension uses an application-defined encoding for +// message parts and metadata. topicID is a string to match the Go reference; +// both it and the registry's bytes form use the same protobuf wire type. +message PartialMessagesExtension { + optional string topicID = 1; + optional bytes groupID = 2; + optional bytes partialMessage = 3; + optional bytes partsMetadata = 4; } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs index 1ff2628b..ddc20a7a 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs @@ -117,3 +117,5 @@ public class GossipsubProtocol(PubsubRouter router, ILoggerFactory? loggerFactor public class GossipsubProtocolV11(PubsubRouter router, ILoggerFactory? loggerFactory = null) : PubsubProtocol(PubsubRouter.GossipsubProtocolVersionV11, router, loggerFactory); public class GossipsubProtocolV12(PubsubRouter router, ILoggerFactory? loggerFactory = null) : PubsubProtocol(PubsubRouter.GossipsubProtocolVersionV12, router, loggerFactory); + +public class GossipsubProtocolV13(PubsubRouter router, ILoggerFactory? loggerFactory = null) : PubsubProtocol(PubsubRouter.GossipsubProtocolVersionV13, router, loggerFactory); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index d552099b..2e7ea7dd 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -19,6 +19,8 @@ internal void OnRpc(PeerId peerId, Rpc rpc) List<(string Topic, PeerId PeerId, byte[] Data)> receivedMessages = []; lock (this) { + HandleExtensions(peerId, rpc); + if (rpc.Publish.Count != 0) { HandleNewMessages(peerId, rpc.Publish, peerMessages, receivedMessages); @@ -73,6 +75,28 @@ internal void OnRpc(PeerId peerId, Rpc rpc) } } + private void HandleExtensions(PeerId peerId, Rpc rpc) + { + if (!peerState.TryGetValue(peerId, out PubsubPeer? peer) || !peer.SupportsExtensions) + { + return; + } + + ControlExtensions? extensions = rpc.Control?.Extensions; + if (peer.ReceivedFirstRpc) + { + if (extensions is not null) + { + ApplyBehaviorPenalty(peerId, 1.0); + logger?.LogDebug("Ignoring repeated Gossipsub v1.3 extensions from {peerId}", peerId); + } + + return; + } + + peer.ReceivedFirstRpc = true; + } + private void HandleNewMessages(PeerId peerId, IEnumerable messages, ConcurrentDictionary peerMessages, List<(string Topic, PeerId PeerId, byte[] Data)> receivedMessages) { // Check if peer is graylisted (Gossipsub v1.1) @@ -237,7 +261,7 @@ private void HandleGraft(PeerId peerId, IEnumerable grafts, Concur { ControlPrune prune = new() { TopicID = graft.TopicID, Backoff = (ulong)(_settings.PruneBackoff / 1000) }; - if (peerState.TryGetValue(peerId, out PubsubPeer? peerData) && peerData.IsGossipSub && peerData.Protocol >= PubsubPeer.PubsubProtocol.GossipsubV11) + if (peerState.TryGetValue(peerId, out PubsubPeer? peerData) && peerData.SupportsPeerExchange) { peerData.Backoff[prune.TopicID] = DateTime.Now.AddMilliseconds(_settings.PruneBackoff); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index aad57b3c..2285df5f 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -31,6 +31,7 @@ public override string ToString() public const string GossipsubProtocolVersionV10 = "/meshsub/1.0.0"; public const string GossipsubProtocolVersionV11 = "/meshsub/1.1.0"; public const string GossipsubProtocolVersionV12 = "/meshsub/1.2.0"; + public const string GossipsubProtocolVersionV13 = "/meshsub/1.3.0"; class PubsubPeer { @@ -43,6 +44,7 @@ public PubsubPeer(PeerId peerId, string protocolId, ILogger? logger, PubsubSetti GossipsubProtocolVersionV10 => PubsubProtocol.GossipsubV10, GossipsubProtocolVersionV11 => PubsubProtocol.GossipsubV11, GossipsubProtocolVersionV12 => PubsubProtocol.GossipsubV12, + GossipsubProtocolVersionV13 => PubsubProtocol.GossipsubV13, _ => PubsubProtocol.Floodsub, }; TokenSource = new CancellationTokenSource(); @@ -58,7 +60,8 @@ public enum PubsubProtocol GossipsubV10 = 2, GossipsubV11 = 4, GossipsubV12 = 8, - AnyGossipsub = GossipsubV10 | GossipsubV11 | GossipsubV12, + GossipsubV13 = 16, + AnyGossipsub = GossipsubV10 | GossipsubV11 | GossipsubV12 | GossipsubV13, } public void Send(Rpc rpc) @@ -79,6 +82,7 @@ public void Send(Rpc rpc) public ConcurrentQueue SendRpcQueue { get; } private Action? _sendRpc; private readonly ILogger? _logger; + public bool ReceivedFirstRpc { get; set; } public Action? SendRpc { @@ -102,6 +106,8 @@ public Action? SendRpc public PubsubProtocol Protocol { get; set; } public bool IsGossipSub => (Protocol & PubsubProtocol.AnyGossipsub) != PubsubProtocol.None; public bool IsFloodSub => Protocol == PubsubProtocol.Floodsub; + public bool SupportsPeerExchange => Protocol is PubsubProtocol.GossipsubV11 or PubsubProtocol.GossipsubV12 or PubsubProtocol.GossipsubV13; + public bool SupportsExtensions => Protocol == PubsubProtocol.GossipsubV13; public ConnectionInitiation InitiatedBy { get; internal set; } public Multiaddress Address { get; internal set; } = null!; @@ -237,7 +243,11 @@ private async Task Connect(Multiaddress[] addrs, CancellationToken token, bool r if (!peerState.ContainsKey(session.RemoteAddress.Get().ToString())) { string[]? protocols = _peerStore.GetPeerInfo(session.RemoteAddress.GetPeerId()!)?.SupportedProtocols ?? []; - if (protocols.Contains(GossipsubProtocolVersionV12)) + if (protocols.Contains(GossipsubProtocolVersionV13)) + { + await session.DialAsync(token); + } + else if (protocols.Contains(GossipsubProtocolVersionV12)) { await session.DialAsync(token); } diff --git a/src/libp2p/Libp2p/Libp2pPeerFactoryBuilder.cs b/src/libp2p/Libp2p/Libp2pPeerFactoryBuilder.cs index 96bb05eb..5464c9da 100644 --- a/src/libp2p/Libp2p/Libp2pPeerFactoryBuilder.cs +++ b/src/libp2p/Libp2p/Libp2pPeerFactoryBuilder.cs @@ -99,6 +99,7 @@ protected override ProtocolRef[] BuildStack(IEnumerable additionalP ProtocolRef[] relay = addRelay ? [Get(), Get()] : []; ProtocolRef[] pubsub = addPubsub ? [ + Get(), Get(), Get(), Get(), From 57c3ee06335b2aba69ec9b689a2cd5c2da906cfd Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 09:16:22 +0300 Subject: [PATCH 22/45] Harden strict pubsub no-sign validation --- .../StrictNoSignTests.cs | 44 +++++++++++++++++++ .../Libp2p.Protocols.Pubsub/RpcExtensions.cs | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs new file mode 100644 index 00000000..9437f49d --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Google.Protobuf; +using Nethermind.Libp2p.Protocols.Pubsub; +using Nethermind.Libp2p.Protocols.Pubsub.Dto; + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +public class StrictNoSignTests +{ + [Test] + public void StrictNoSign_AcceptsMessagesWithoutAuthorshipFields() + { + Message message = new() + { + Topic = "no-sign", + Data = ByteString.CopyFrom([1, 2, 3]), + }; + + Assert.That(message.VerifySignature(PubsubSettings.SignaturePolicy.StrictNoSign), Is.True); + } + + [Test] + public void StrictNoSign_RejectsExplicitlyPresentAuthorshipFields() + { + Message[] messages = + [ + new Message { Signature = ByteString.Empty }, + new Message { From = ByteString.Empty }, + new Message { Seqno = ByteString.Empty }, + new Message { Key = ByteString.Empty }, + ]; + + Assert.Multiple(() => + { + foreach (Message message in messages) + { + Assert.That(message.VerifySignature(PubsubSettings.SignaturePolicy.StrictNoSign), Is.False); + } + }); + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/RpcExtensions.cs b/src/libp2p/Libp2p.Protocols.Pubsub/RpcExtensions.cs index c2a5a90d..d2a570a4 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/RpcExtensions.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/RpcExtensions.cs @@ -49,7 +49,7 @@ public static bool VerifySignature(this Message message, PubsubSettings.Signatur { if (signaturePolicy is PubsubSettings.SignaturePolicy.StrictNoSign) { - return message.Signature.IsEmpty; + return !message.HasSignature && !message.HasFrom && !message.HasSeqno && !message.HasKey; } PublicKey? pubKey = PeerId.ExtractPublicKey(message.From.ToArray()); From 5e8c21954a0b8130870478618b36534a333da63c Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 13:12:38 +0300 Subject: [PATCH 23/45] Use opaque extension topic IDs --- .../GossipsubV13ProtocolTests.cs | 22 ++++++++++++++++-- src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs | 23 +++++++++---------- .../Libp2p.Protocols.Pubsub/Dto/Rpc.proto | 7 +++--- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs index 2144933e..ba9a9bc0 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs @@ -34,7 +34,7 @@ public void ExtensionRegistryMessages_RoundTrip() }, Partial = new PartialMessagesExtension { - TopicID = "topic", + TopicID = ByteString.CopyFromUtf8("topic"), GroupID = ByteString.CopyFrom([1, 2]), PartialMessage = ByteString.CopyFrom([3]), PartsMetadata = ByteString.CopyFrom([4]), @@ -53,13 +53,31 @@ public void ExtensionRegistryMessages_RoundTrip() Assert.Multiple(() => { Assert.That(decoded.Control.Extensions.PartialMessages, Is.True); - Assert.That(decoded.Partial.TopicID, Is.EqualTo("topic")); + Assert.That(decoded.Partial.TopicID.ToStringUtf8(), Is.EqualTo("topic")); Assert.That(decoded.Partial.GroupID.ToByteArray(), Is.EqualTo(new byte[] { 1, 2 })); Assert.That(decoded.Subscriptions.Single().RequestsPartial, Is.True); Assert.That(decoded.Subscriptions.Single().SupportsSendingPartial, Is.True); }); } + [Test] + public void ExtensionRegistry_PreservesOpaquePartialMessageTopicBytes() + { + Rpc rpc = new() + { + Partial = new PartialMessagesExtension + { + TopicID = ByteString.CopyFrom([0xff, 0x00, 0x80]), + GroupID = ByteString.CopyFrom([1]), + PartialMessage = ByteString.CopyFrom([2]), + }, + }; + + Rpc decoded = Rpc.Parser.ParseFrom(rpc.ToByteArray()); + + Assert.That(decoded.Partial.TopicID.ToByteArray(), Is.EqualTo(new byte[] { 0xff, 0x00, 0x80 })); + } + [Test] public async Task Router_PrefersV13WhenTheRemotePeerAdvertisesIt() { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs index 361f896f..3d4a53de 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs @@ -45,7 +45,7 @@ static RpcReflection() { "GAoQc2lnbmVkUGVlclJlY29yZBgCIAEoDCImChBDb250cm9sSURvbnRXYW50", "EhIKCm1lc3NhZ2VJRHMYASADKAwiLAoRQ29udHJvbEV4dGVuc2lvbnMSFwoP", "cGFydGlhbE1lc3NhZ2VzGAogASgIImsKGFBhcnRpYWxNZXNzYWdlc0V4dGVu", - "c2lvbhIPCgd0b3BpY0lEGAEgASgJEg8KB2dyb3VwSUQYAiABKAwSFgoOcGFy", + "c2lvbhIPCgd0b3BpY0lEGAEgASgMEg8KB2dyb3VwSUQYAiABKAwSFgoOcGFy", "dGlhbE1lc3NhZ2UYAyABKAwSFQoNcGFydHNNZXRhZGF0YRgEIAEoDEIpqgIm", "TmV0aGVybWluZC5MaWJwMnAuUHJvdG9jb2xzLlB1YnN1Yi5EdG8=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, @@ -3166,9 +3166,8 @@ public void MergeFrom(pb::CodedInputStream input) { } /// - /// The Partial Messages extension uses an application-defined encoding for - /// message parts and metadata. topicID is a string to match the Go reference; - /// both it and the registry's bytes form use the same protobuf wire type. + /// The Partial Messages registry identifies topics as opaque bytes. Go's + /// current string declaration is wire compatible with this canonical form. /// [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] public sealed partial class PartialMessagesExtension : pb::IMessage @@ -3220,12 +3219,12 @@ public PartialMessagesExtension Clone() { /// Field number for the "topicID" field. public const int TopicIDFieldNumber = 1; - private readonly static string TopicIDDefaultValue = ""; + private readonly static pb::ByteString TopicIDDefaultValue = pb::ByteString.Empty; - private string topicID_; + private pb::ByteString topicID_; [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] - public string TopicID { + public pb::ByteString TopicID { get { return topicID_ ?? TopicIDDefaultValue; } set { topicID_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); @@ -3372,7 +3371,7 @@ public void WriteTo(pb::CodedOutputStream output) { #else if (HasTopicID) { output.WriteRawTag(10); - output.WriteString(TopicID); + output.WriteBytes(TopicID); } if (HasGroupID) { output.WriteRawTag(18); @@ -3398,7 +3397,7 @@ public void WriteTo(pb::CodedOutputStream output) { void pb::IBufferMessage.InternalWriteTo(ref pb::WriteContext output) { if (HasTopicID) { output.WriteRawTag(10); - output.WriteString(TopicID); + output.WriteBytes(TopicID); } if (HasGroupID) { output.WriteRawTag(18); @@ -3423,7 +3422,7 @@ public void WriteTo(pb::CodedOutputStream output) { public int CalculateSize() { int size = 0; if (HasTopicID) { - size += 1 + pb::CodedOutputStream.ComputeStringSize(TopicID); + size += 1 + pb::CodedOutputStream.ComputeBytesSize(TopicID); } if (HasGroupID) { size += 1 + pb::CodedOutputStream.ComputeBytesSize(GroupID); @@ -3478,7 +3477,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, input); break; case 10: { - TopicID = input.ReadString(); + TopicID = input.ReadBytes(); break; } case 18: { @@ -3513,7 +3512,7 @@ public void MergeFrom(pb::CodedInputStream input) { _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); break; case 10: { - TopicID = input.ReadString(); + TopicID = input.ReadBytes(); break; } case 18: { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto index ba8c5f85..dc235c7b 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto @@ -71,11 +71,10 @@ message ControlExtensions { optional bool partialMessages = 10; } -// The Partial Messages extension uses an application-defined encoding for -// message parts and metadata. topicID is a string to match the Go reference; -// both it and the registry's bytes form use the same protobuf wire type. +// The Partial Messages registry identifies topics as opaque bytes. Go's +// current string declaration is wire compatible with this canonical form. message PartialMessagesExtension { - optional string topicID = 1; + optional bytes topicID = 1; optional bytes groupID = 2; optional bytes partialMessage = 3; optional bytes partsMetadata = 4; From 0cb8a79f0d836bfd393c2b162110acfc9e6a3102 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 11:03:12 +0300 Subject: [PATCH 24/45] Require message IDs for unsigned pubsub --- .../StrictNoSignTests.cs | 52 +++++++++++++++++-- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 5 ++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs index 9437f49d..c72572eb 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT using Google.Protobuf; +using Nethermind.Libp2p.Core.Discovery; using Nethermind.Libp2p.Protocols.Pubsub; using Nethermind.Libp2p.Protocols.Pubsub.Dto; @@ -27,10 +28,10 @@ public void StrictNoSign_RejectsExplicitlyPresentAuthorshipFields() { Message[] messages = [ - new Message { Signature = ByteString.Empty }, - new Message { From = ByteString.Empty }, - new Message { Seqno = ByteString.Empty }, - new Message { Key = ByteString.Empty }, + new Message { Topic = "no-sign", Signature = ByteString.Empty }, + new Message { Topic = "no-sign", From = ByteString.Empty }, + new Message { Topic = "no-sign", Seqno = ByteString.Empty }, + new Message { Topic = "no-sign", Key = ByteString.Empty }, ]; Assert.Multiple(() => @@ -41,4 +42,47 @@ public void StrictNoSign_RejectsExplicitlyPresentAuthorshipFields() } }); } + + [Test] + public void StrictNoSign_RequiresACustomMessageId() + { + PubsubSettings settings = new() + { + DefaultSignaturePolicy = PubsubSettings.SignaturePolicy.StrictNoSign, + }; + + InvalidOperationException? exception = Assert.Throws(() => new PubsubRouter(new PeerStore(), settings)); + + Assert.That(exception!.Message, Does.Contain("GetMessageId")); + } + + [Test] + public void StrictNoSign_DeliversDistinctMessagesWithACustomMessageId() + { + PubsubSettings settings = new() + { + DefaultSignaturePolicy = PubsubSettings.SignaturePolicy.StrictNoSign, + GetMessageId = message => new(message.Data.ToByteArray()), + }; + using PubsubRouter router = new(new PeerStore(), settings); + ITopic topic = router.GetTopic("no-sign"); + int deliveries = 0; + topic.OnMessage += (_, _) => deliveries++; + + router.OnRpc(TestPeers.PeerId(1), CreateUnsignedMessage("first")); + router.OnRpc(TestPeers.PeerId(1), CreateUnsignedMessage("second")); + + Assert.That(deliveries, Is.EqualTo(2)); + } + + private static Rpc CreateUnsignedMessage(string payload) + { + Rpc rpc = new(); + rpc.Publish.Add(new Message + { + Topic = "no-sign", + Data = ByteString.CopyFromUtf8(payload), + }); + return rpc; + } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index ad234800..aad57b3c 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -171,6 +171,11 @@ public PubsubRouter(PeerStore store, PubsubSettings? settings = null, ILoggerFac _peerStore = store; _settings = settings ?? PubsubSettings.Default; + if (_settings.DefaultSignaturePolicy is PubsubSettings.SignaturePolicy.StrictNoSign && _settings.GetMessageId == PubsubSettings.ConcatFromAndSeqno) + { + throw new InvalidOperationException("StrictNoSign requires a custom GetMessageId function."); + } + _messageCache = new(_settings.MessageCacheTtl); _limboMessageCache = new(_settings.MessageCacheTtl); _idontwantMessages = new(_settings.MessageCacheTtl); From 7f6400b06fc4c45ee93bd69aa7ddc943fdfe3f49 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 13:43:10 +0300 Subject: [PATCH 25/45] Reject extensions outside Gossipsub v1.3 --- .../Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index 2e7ea7dd..56fc4244 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -77,12 +77,23 @@ internal void OnRpc(PeerId peerId, Rpc rpc) private void HandleExtensions(PeerId peerId, Rpc rpc) { - if (!peerState.TryGetValue(peerId, out PubsubPeer? peer) || !peer.SupportsExtensions) + if (!peerState.TryGetValue(peerId, out PubsubPeer? peer)) { return; } ControlExtensions? extensions = rpc.Control?.Extensions; + if (!peer.SupportsExtensions) + { + if (extensions is not null) + { + ApplyBehaviorPenalty(peerId, 1.0); + logger?.LogDebug("Ignoring Gossipsub v1.3 extensions from {peerId} on {protocol}", peerId, peer.Protocol); + } + + return; + } + if (peer.ReceivedFirstRpc) { if (extensions is not null) From 34d12d5d689684ac1b1b57c6d3d4cd061b177ada Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 11:59:03 +0300 Subject: [PATCH 26/45] Support Gossipsub partial messages --- .../PartialMessagesTests.cs | 155 ++++++++++++++++++ .../IPartialMessagesTopic.cs | 31 ++++ .../Libp2p.Protocols.Pubsub/PartialMessage.cs | 43 +++++ .../Libp2p.Protocols.Pubsub/PubSubSettings.cs | 6 + .../PubsubRouter.Rpc.cs | 47 ++++++ .../PubsubRouter.Topics.cs | 154 ++++++++++++++++- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 115 ++++++++++++- src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs | 39 ++++- 8 files changed, 582 insertions(+), 8 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub/IPartialMessagesTopic.cs create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs new file mode 100644 index 00000000..2bf56733 --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Google.Protobuf; +using Multiformats.Address; +using Nethermind.Libp2p.Core.Discovery; +using Nethermind.Libp2p.Protocols.Pubsub.Dto; + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +public class PartialMessagesTests +{ + [Test] + public void PartialMessagesTopics_RequireExplicitRouterAndTopicOptIn() + { + PubsubRouter disabledRouter = new(new PeerStore()); + Assert.That( + () => disabledRouter.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions { SupportSendingPartialMessages = true }), + Throws.TypeOf()); + + PubsubRouter enabledRouter = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + Assert.That( + () => enabledRouter.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions { RequestPartialMessages = true }), + Throws.TypeOf()); + } + + [Test] + public async Task PartialMessages_AdvertiseCapabilitiesAndGatePayloadsPerPeer() + { + const string topicName = "topic"; + PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + IPartialMessagesTopic topic = router.GetPartialMessagesTopic( + topicName, + new PartialMessagesTopicOptions + { + RequestPartialMessages = true, + SupportSendingPartialMessages = true, + }); + await router.StartAsync(new LocalPeerStub()); + + List requestedPeerRpcs = []; + List metadataOnlyPeerRpcs = []; + TaskCompletionSource firstConnection = new(); + TaskCompletionSource secondConnection = new(); + Multiaddress requestedPeerAddress = TestPeers.Multiaddr(1); + Multiaddress metadataOnlyPeerAddress = TestPeers.Multiaddr(2); + PeerId requestedPeerId = requestedPeerAddress.GetPeerId()!; + PeerId metadataOnlyPeerId = metadataOnlyPeerAddress.GetPeerId()!; + + router.OutboundConnection(requestedPeerAddress, PubsubRouter.GossipsubProtocolVersionV13, firstConnection.Task, requestedPeerRpcs.Add); + router.OutboundConnection(metadataOnlyPeerAddress, PubsubRouter.GossipsubProtocolVersionV13, secondConnection.Task, metadataOnlyPeerRpcs.Add); + + Assert.Multiple(() => + { + Assert.That(requestedPeerRpcs.Single().Control.Extensions.PartialMessages, Is.True); + Assert.That(requestedPeerRpcs.Single().Subscriptions.Single().RequestsPartial, Is.True); + Assert.That(requestedPeerRpcs.Single().Subscriptions.Single().SupportsSendingPartial, Is.True); + Assert.That(metadataOnlyPeerRpcs.Single().Control.Extensions.PartialMessages, Is.True); + }); + + router.OnRpc(requestedPeerId, CreateSubscriptionRpc(topicName, requestsPartialMessages: true, supportsSendingPartialMessages: true)); + router.OnRpc(metadataOnlyPeerId, CreateSubscriptionRpc(topicName, requestsPartialMessages: false, supportsSendingPartialMessages: true)); + requestedPeerRpcs.Clear(); + metadataOnlyPeerRpcs.Clear(); + + topic.PublishPartial([1, 2], partialMessage: [3], partsMetadata: [4]); + + PartialMessagesExtension requestedPartial = requestedPeerRpcs.Single().Partial; + PartialMessagesExtension metadataOnlyPartial = metadataOnlyPeerRpcs.Single().Partial; + Assert.Multiple(() => + { + Assert.That(requestedPartial.GroupID.ToByteArray(), Is.EqualTo(new byte[] { 1, 2 })); + Assert.That(requestedPartial.PartialMessage.ToByteArray(), Is.EqualTo(new byte[] { 3 })); + Assert.That(requestedPartial.PartsMetadata.ToByteArray(), Is.EqualTo(new byte[] { 4 })); + Assert.That(metadataOnlyPartial.HasPartialMessage, Is.False); + Assert.That(metadataOnlyPartial.PartsMetadata.ToByteArray(), Is.EqualTo(new byte[] { 4 })); + }); + + firstConnection.SetResult(); + secondConnection.SetResult(); + } + + [Test] + public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() + { + const string topicName = "topic"; + PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + IPartialMessagesTopic topic = router.GetPartialMessagesTopic( + topicName, + new PartialMessagesTopicOptions { SupportSendingPartialMessages = true }); + Multiaddress remoteAddress = TestPeers.Multiaddr(1); + PeerId remotePeerId = remoteAddress.GetPeerId()!; + TaskCompletionSource connection = new(); + router.OutboundConnection(remoteAddress, PubsubRouter.GossipsubProtocolVersionV13, connection.Task, _ => { }); + + PartialMessage? received = null; + topic.OnPartialMessage += (_, message) => received = message; + + router.OnRpc(remotePeerId, new Rpc + { + Control = new ControlMessage { Extensions = new ControlExtensions { PartialMessages = true } }, + }); + router.OnRpc(remotePeerId, new Rpc + { + Partial = new PartialMessagesExtension + { + TopicID = topicName, + GroupID = ByteString.CopyFrom([1]), + PartialMessage = ByteString.CopyFrom([2]), + PartsMetadata = ByteString.CopyFrom([3]), + }, + }); + + Assert.Multiple(() => + { + Assert.That(received, Is.Not.Null); + Assert.That(received!.GroupId, Is.EqualTo(new byte[] { 1 })); + Assert.That(received.PartialData, Is.EqualTo(new byte[] { 2 })); + Assert.That(received.PartsMetadata, Is.EqualTo(new byte[] { 3 })); + }); + + connection.SetResult(); + } + + [Test] + public void PartialMessages_AreNeverAdvertisedOnOlderProtocols() + { + PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + _ = router.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions { SupportSendingPartialMessages = true }); + List sentRpcs = []; + TaskCompletionSource connection = new(); + + router.OutboundConnection(TestPeers.Multiaddr(1), PubsubRouter.GossipsubProtocolVersionV12, connection.Task, sentRpcs.Add); + + Assert.That(sentRpcs.Single().Control, Is.Null); + connection.SetResult(); + } + + private static Rpc CreateSubscriptionRpc(string topicName, bool requestsPartialMessages, bool supportsSendingPartialMessages) + { + Rpc rpc = new() + { + Control = new ControlMessage { Extensions = new ControlExtensions { PartialMessages = true } }, + }; + rpc.Subscriptions.Add(new Rpc.Types.SubOpts + { + Subscribe = true, + Topicid = topicName, + RequestsPartial = requestsPartialMessages, + SupportsSendingPartial = supportsSendingPartialMessages, + }); + return rpc; + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/IPartialMessagesTopic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/IPartialMessagesTopic.cs new file mode 100644 index 00000000..2eaa5a1f --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub/IPartialMessagesTopic.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Nethermind.Libp2p.Core; + +namespace Nethermind.Libp2p.Protocols.Pubsub; + +/// +/// A topic configured for the Gossipsub v1.3 Partial Messages extension. +/// +public interface IPartialMessagesTopic : ITopic +{ + /// + /// Raised for application-defined partial message payloads received for this topic. + /// + event Action? OnPartialMessage; + + bool RequestsPartialMessages { get; } + bool SupportsSendingPartialMessages { get; } + + /// + /// Sends an application-defined partial message to the topic mesh or fanout peers. + /// + void PublishPartial(byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null); + + /// + /// Sends an application-defined partial message to a connected peer, including + /// a non-mesh peer selected by application gossip logic. + /// + void SendPartial(PeerId peerId, byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null); +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs new file mode 100644 index 00000000..c2719cc9 --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +namespace Nethermind.Libp2p.Protocols.Pubsub; + +/// +/// Application-defined data carried by the Gossipsub v1.3 Partial Messages extension. +/// +public sealed class PartialMessage +{ + public PartialMessage(string topicId, byte[] groupId, byte[]? partialData, byte[]? partsMetadata) + { + ArgumentNullException.ThrowIfNull(topicId); + ArgumentNullException.ThrowIfNull(groupId); + + TopicId = topicId; + GroupId = groupId; + PartialData = partialData; + PartsMetadata = partsMetadata; + } + + public string TopicId { get; } + public byte[] GroupId { get; } + public byte[]? PartialData { get; } + public byte[]? PartsMetadata { get; } +} + +/// +/// Per-topic capabilities advertised through Gossipsub subscription options. +/// +public sealed class PartialMessagesTopicOptions +{ + /// + /// Requests partial data from peers. This also requires support for sending + /// partial messages. + /// + public bool RequestPartialMessages { get; init; } + + /// + /// Signals that this topic can send partial data and receive parts metadata. + /// + public bool SupportSendingPartialMessages { get; init; } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index cae2662d..df33e9c2 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -40,6 +40,12 @@ public int MaxRpcBytes public SignaturePolicy DefaultSignaturePolicy { get; set; } = SignaturePolicy.StrictSign; public int MaxIdontwantMessages { get; set; } = 50; + /// + /// Enables the opt-in Gossipsub v1.3 Partial Messages extension. The router + /// advertises it only to v1.3 peers. + /// + public bool EnablePartialMessages { get; set; } + public Func GetMessageId { get; set; } = ConcatFromAndSeqno; public enum SignaturePolicy diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index 56fc4244..6b589453 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -17,6 +17,7 @@ internal void OnRpc(PeerId peerId, Rpc rpc) { ConcurrentDictionary peerMessages = new(); List<(string Topic, PeerId PeerId, byte[] Data)> receivedMessages = []; + List<(string Topic, PeerId PeerId, PartialMessage Message)> receivedPartialMessages = []; lock (this) { HandleExtensions(peerId, rpc); @@ -31,6 +32,11 @@ internal void OnRpc(PeerId peerId, Rpc rpc) HandleSubscriptions(peerId, rpc.Subscriptions); } + if (rpc.Partial is not null) + { + HandlePartialMessage(peerId, rpc.Partial, receivedPartialMessages); + } + if (rpc.Control is not null) { if (rpc.Control.Graft.Count != 0) @@ -64,6 +70,11 @@ internal void OnRpc(PeerId peerId, Rpc rpc) OnMessage?.Invoke(topic, receivedFrom, data); } + foreach ((string topic, PeerId receivedFrom, PartialMessage message) in receivedPartialMessages) + { + OnPartialMessage?.Invoke(topic, receivedFrom, message); + } + foreach (KeyValuePair peerMessage in peerMessages) { peerState.GetValueOrDefault(peerMessage.Key)?.Send(peerMessage.Value); @@ -106,6 +117,32 @@ private void HandleExtensions(PeerId peerId, Rpc rpc) } peer.ReceivedFirstRpc = true; + peer.SupportsPartialMessagesExtension = extensions?.PartialMessages ?? false; + } + + private void HandlePartialMessage(PeerId peerId, PartialMessagesExtension partialMessage, List<(string Topic, PeerId PeerId, PartialMessage Message)> receivedPartialMessages) + { + if (!_settings.EnablePartialMessages || + !peerState.TryGetValue(peerId, out PubsubPeer? peer) || + !peer.SupportsPartialMessagesExtension) + { + return; + } + + if (!partialMessage.HasTopicID) + { + logger?.LogDebug("Ignoring a Partial Messages extension payload without a topic from {peerId}", peerId); + return; + } + + receivedPartialMessages.Add(( + partialMessage.TopicID, + peerId, + new PartialMessage( + partialMessage.TopicID, + partialMessage.GroupID.ToByteArray(), + partialMessage.HasPartialMessage ? partialMessage.PartialMessage.ToByteArray() : null, + partialMessage.HasPartsMetadata ? partialMessage.PartsMetadata.ToByteArray() : null))); } private void HandleNewMessages(PeerId peerId, IEnumerable messages, ConcurrentDictionary peerMessages, List<(string Topic, PeerId PeerId, byte[] Data)> receivedMessages) @@ -215,9 +252,19 @@ private void HandleSubscriptions(PeerId peerId, IEnumerable s { fPeers.GetOrAdd(sub.Topicid, _ => []).Add(peerId); } + + if (_settings.EnablePartialMessages && state.SupportsPartialMessagesExtension) + { + bool requestsPartialMessages = sub.RequestsPartial; + state.UpdatePartialMessagesSubscription( + sub.Topicid, + requestsPartialMessages, + requestsPartialMessages || sub.SupportsSendingPartial); + } } else { + state.RemovePartialMessagesSubscription(sub.Topicid); if (state.IsGossipSub) { gPeers.GetOrAdd(sub.Topicid, _ => []).Remove(peerId); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index a51ac635..910533d5 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -24,6 +24,57 @@ public ITopic GetTopic(string topicId, bool subscribe = true) return topic; } + /// + /// Gets a topic configured for the opt-in Gossipsub v1.3 Partial Messages extension. + /// + public IPartialMessagesTopic GetPartialMessagesTopic(string topicId, PartialMessagesTopicOptions options, bool subscribe = true) + { + ArgumentNullException.ThrowIfNull(topicId); + ArgumentNullException.ThrowIfNull(options); + if (!_settings.EnablePartialMessages) + { + throw new InvalidOperationException("Partial messages are not enabled. Set EnablePartialMessages before creating a partial messages topic."); + } + + Topic topic = topicState.GetOrAdd(topicId, (tId) => new(this, tId)); + bool wasSubscribed = topic.IsSubscribed; + topic.ConfigurePartialMessages(options); + + if (subscribe) + { + Subscribe(topicId); + } + + if (wasSubscribed) + { + AnnounceSubscription(topicId); + } + + return topic; + } + + private Rpc.Types.SubOpts CreateSubscription(string topicId, bool subscribe) + { + Rpc.Types.SubOpts subscription = new() { Subscribe = subscribe, Topicid = topicId }; + if (subscribe && _settings.EnablePartialMessages && topicState.TryGetValue(topicId, out Topic? topic)) + { + subscription.RequestsPartial = topic.RequestsPartialMessages; + subscription.SupportsSendingPartial = topic.SupportsSendingPartialMessages; + } + + return subscription; + } + + private void AnnounceSubscription(string topicId) + { + Rpc topicUpdate = new(); + topicUpdate.Subscriptions.Add(CreateSubscription(topicId, subscribe: true)); + foreach (KeyValuePair peer in peerState) + { + peer.Value.Send(topicUpdate); + } + } + public void Subscribe(string topicId) { lock (this) @@ -54,7 +105,8 @@ public void Subscribe(string topicId) fanoutLastPublished.TryRemove(topicId, out _); } - Rpc topicUpdate = new Rpc().WithTopics([topicId], []); + Rpc topicUpdate = new(); + topicUpdate.Subscriptions.Add(CreateSubscription(topicId, subscribe: true)); foreach (PubsubPeer peer in peerState.Values) { peer.Send(topicUpdate); @@ -187,4 +239,104 @@ public void Publish(string topicId, byte[] message) } } + /// + /// Sends application-defined partial data to partial-message-capable mesh or fanout peers. + /// + public void PublishPartial(string topicId, byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + { + EnsurePartialMessagesEnabled(topicId); + ValidatePartialMessage(groupId, partialMessage, partsMetadata); + + if (localPeer is null) + { + throw new InvalidOperationException("Router has not been started. Call StartAsync() first."); + } + + PeerId[] recipients; + if (mesh.TryGetValue(topicId, out HashSet? meshPeers) && meshPeers.Count > 0) + { + recipients = meshPeers.Where(peerId => GetPeerScore(peerId) >= _settings.PublishThreshold).ToArray(); + } + else + { + fanoutLastPublished[topicId] = DateTime.Now; + HashSet fanoutPeers = fanout.GetOrAdd(topicId, _ => []); + if (fanoutPeers.Count == 0 && gPeers.TryGetValue(topicId, out HashSet? topicPeers)) + { + foreach (PeerId peerId in topicPeers.Where(peerId => GetPeerScore(peerId) >= 0).Take(_settings.Degree)) + { + fanoutPeers.Add(peerId); + } + } + + recipients = fanoutPeers.Where(peerId => GetPeerScore(peerId) >= _settings.PublishThreshold).ToArray(); + } + + foreach (PeerId peerId in recipients) + { + SendPartial(peerId, topicId, groupId, partialMessage, partsMetadata); + } + } + + /// + /// Sends application-defined partial data to a connected peer selected by the application. + /// + public void SendPartial(PeerId peerId, string topicId, byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + { + EnsurePartialMessagesEnabled(topicId); + ValidatePartialMessage(groupId, partialMessage, partsMetadata); + + if (!peerState.TryGetValue(peerId, out PubsubPeer? peer) || !peer.SupportsPartialMessagesExtension) + { + return; + } + + bool sendPartialData = peer.RequestsPartialMessages(topicId); + bool sendPartsMetadata = peer.SupportsSendingPartialMessages(topicId); + if (!sendPartialData && !sendPartsMetadata) + { + return; + } + + PartialMessagesExtension partial = new() + { + TopicID = topicId, + GroupID = Google.Protobuf.ByteString.CopyFrom(groupId), + }; + if (sendPartialData && partialMessage is not null) + { + partial.PartialMessage = Google.Protobuf.ByteString.CopyFrom(partialMessage); + } + + if (sendPartsMetadata && partsMetadata is not null) + { + partial.PartsMetadata = Google.Protobuf.ByteString.CopyFrom(partsMetadata); + } + + if (partial.HasPartialMessage || partial.HasPartsMetadata) + { + peer.Send(new Rpc { Partial = partial }); + } + } + + private void EnsurePartialMessagesEnabled(string topicId) + { + ArgumentNullException.ThrowIfNull(topicId); + if (!_settings.EnablePartialMessages || + !topicState.TryGetValue(topicId, out Topic? topic) || + !topic.SupportsSendingPartialMessages) + { + throw new InvalidOperationException($"Partial messages are not enabled for topic '{topicId}'."); + } + } + + private static void ValidatePartialMessage(byte[] groupId, byte[]? partialMessage, byte[]? partsMetadata) + { + ArgumentNullException.ThrowIfNull(groupId); + if (partialMessage is null && partsMetadata is null) + { + throw new ArgumentException("A partial message or parts metadata must be supplied."); + } + } + } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index 2285df5f..dafee568 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -47,6 +47,7 @@ public PubsubPeer(PeerId peerId, string protocolId, ILogger? logger, PubsubSetti GossipsubProtocolVersionV13 => PubsubProtocol.GossipsubV13, _ => PubsubProtocol.Floodsub, }; + _advertisesPartialMessages = settings.EnablePartialMessages; TokenSource = new CancellationTokenSource(); Backoff = []; SendRpcQueue = new ConcurrentQueue(); @@ -66,6 +67,7 @@ public enum PubsubProtocol public void Send(Rpc rpc) { + rpc = AddExtensionsIfNeeded(rpc); SendRpcQueue.Enqueue(rpc); if (SendRpc is not null) { @@ -82,7 +84,69 @@ public void Send(Rpc rpc) public ConcurrentQueue SendRpcQueue { get; } private Action? _sendRpc; private readonly ILogger? _logger; + private readonly bool _advertisesPartialMessages; + private readonly object _extensionsLock = new(); + private readonly ConcurrentDictionary _partialMessagesSubscriptions = new(); + + private readonly record struct PartialMessagesSubscription(bool RequestsPartialMessages, bool SupportsSendingPartialMessages); + + public bool ExtensionsSent { get; private set; } public bool ReceivedFirstRpc { get; set; } + public bool SupportsPartialMessagesExtension { get; set; } + + public bool NeedsExtensions + { + get + { + lock (_extensionsLock) + { + return _advertisesPartialMessages && SupportsExtensions && !ExtensionsSent; + } + } + } + + public void UpdatePartialMessagesSubscription(string topicId, bool requestsPartialMessages, bool supportsSendingPartialMessages) + { + _partialMessagesSubscriptions[topicId] = new(requestsPartialMessages, supportsSendingPartialMessages); + } + + public void RemovePartialMessagesSubscription(string topicId) + { + _partialMessagesSubscriptions.TryRemove(topicId, out _); + } + + public bool RequestsPartialMessages(string topicId) + { + return _partialMessagesSubscriptions.TryGetValue(topicId, out PartialMessagesSubscription subscription) && subscription.RequestsPartialMessages; + } + + public bool SupportsSendingPartialMessages(string topicId) + { + return _partialMessagesSubscriptions.TryGetValue(topicId, out PartialMessagesSubscription subscription) && subscription.SupportsSendingPartialMessages; + } + + private Rpc AddExtensionsIfNeeded(Rpc rpc) + { + if (!_advertisesPartialMessages || !SupportsExtensions || ExtensionsSent) + { + return rpc; + } + + lock (_extensionsLock) + { + if (ExtensionsSent) + { + return rpc; + } + + Rpc extendedRpc = rpc.Clone(); + extendedRpc.Control ??= new ControlMessage(); + extendedRpc.Control.Extensions ??= new ControlExtensions(); + extendedRpc.Control.Extensions.PartialMessages = true; + ExtensionsSent = true; + return extendedRpc; + } + } public Action? SendRpc { @@ -130,6 +194,17 @@ public Action? SendRpc #endregion public event Action? OnMessage; + /// + /// Raised for Gossipsub v1.3 Partial Messages extension payloads. The router + /// does not retain their application-defined state. + /// + public event Action? OnPartialMessage; + + /// + /// Raised with non-mesh peers that requested partial messages instead of an + /// IHAVE announcement. Applications can respond by calling . + /// + public event Action>? OnPartialGossip; public Func? VerifyMessage = null; internal int MaxRpcBytes => _settings.MaxRpcBytes; @@ -311,6 +386,8 @@ public Task Heartbeat() DecayScores(); ConcurrentDictionary peerMessages = new(); + List<(string Topic, PeerId[] Peers)> partialGossipNotifications = []; + Action>? onPartialGossip = OnPartialGossip; lock (this) { // First, prune peers with negative scores from all meshes (Gossipsub v1.1) @@ -449,15 +526,35 @@ public Task Heartbeat() // Only send gossip to peers above gossip threshold HashSet? topicMesh = mesh.GetValueOrDefault(topic); HashSet? topicFanout = fanout.GetValueOrDefault(topic); - var eligiblePeers = topicGossipsubPeers + PeerId[] eligiblePeers = topicGossipsubPeers .Where(p => !(topicMesh?.Contains(p) ?? false) && !(topicFanout?.Contains(p) ?? false) - && GetPeerScore(p) >= _settings.GossipThreshold); + && GetPeerScore(p) >= _settings.GossipThreshold) + .ToArray(); // Adaptive gossip: send to gossip_factor of eligible peers (min D_lazy) - int gossipCount = Math.Max(_settings.LazyDegree, (int)(eligiblePeers.Count() * _settings.GossipFactor)); + int gossipCount = Math.Max(_settings.LazyDegree, (int)(eligiblePeers.Length * _settings.GossipFactor)); + PeerId[] gossipPeers = eligiblePeers.Take(gossipCount).ToArray(); + + if (onPartialGossip is not null && + _settings.EnablePartialMessages && + topicState.TryGetValue(topic, out Topic? localTopic) && + localTopic.SupportsSendingPartialMessages) + { + PeerId[] partialGossipPeers = gossipPeers + .Where(peerId => peerState.TryGetValue(peerId, out PubsubPeer? peer) && + peer.SupportsPartialMessagesExtension && + peer.RequestsPartialMessages(topic)) + .ToArray(); + + if (partialGossipPeers.Length > 0) + { + partialGossipNotifications.Add((topic, partialGossipPeers)); + gossipPeers = gossipPeers.Except(partialGossipPeers).ToArray(); + } + } - foreach (PeerId? peer in eligiblePeers.Take(gossipCount)) + foreach (PeerId peer in gossipPeers) { peerMessages.GetOrAdd(peer, _ => new Rpc()) .Ensure(r => r.Control.Ihave).Add(ihave); @@ -471,6 +568,11 @@ public Task Heartbeat() peerState.GetValueOrDefault(peerMessage.Key)?.Send(peerMessage.Value); } + foreach ((string topic, PeerId[] peers) in partialGossipNotifications) + { + onPartialGossip?.Invoke(topic, peers); + } + return Task.CompletedTask; } @@ -535,11 +637,12 @@ internal CancellationToken OutboundConnection(Multiaddress addr, string protocol .ToArray(); } - if (topics.Any()) + if (topics.Any() || peer.NeedsExtensions) { logger?.LogDebug("Topics sent to {peerId}: {topics}", peerId, string.Join(",", topics)); - Rpc helloMessage = new Rpc().WithTopics(topics, []); + Rpc helloMessage = new(); + helloMessage.Subscriptions.AddRange(topics.Select(topic => CreateSubscription(topic, subscribe: true))); peer.Send(helloMessage); } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs index 811b830d..6af12ec0 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs @@ -5,7 +5,7 @@ namespace Nethermind.Libp2p.Protocols.Pubsub; -internal class Topic : ITopic +internal class Topic : IPartialMessagesTopic { private readonly PubsubRouter router; private readonly string topicName; @@ -15,6 +15,7 @@ public Topic(PubsubRouter router, string topicName) this.router = router; this.topicName = topicName; router.OnMessage += OnRouterMessage; + router.OnPartialMessage += OnRouterPartialMessage; } private void OnRouterMessage(string topicName, PeerId peerId, byte[] message) @@ -31,17 +32,53 @@ private void OnRouterMessage(string topicName, PeerId peerId, byte[] message) } } + private void OnRouterPartialMessage(string topicName, PeerId peerId, PartialMessage message) + { + if (this.topicName != topicName) + { + return; + } + + Action? onPartialMessage = OnPartialMessage; + onPartialMessage?.Invoke(peerId, message); + } + public DateTime LastPublished { get; set; } public bool IsSubscribed { get; internal set; } + public bool RequestsPartialMessages { get; private set; } + public bool SupportsSendingPartialMessages { get; private set; } public event Action? OnMessage; + public event Action? OnPartialMessage; + + internal void ConfigurePartialMessages(PartialMessagesTopicOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (options.RequestPartialMessages && !options.SupportSendingPartialMessages) + { + throw new ArgumentException("Requesting partial messages requires support for sending partial messages.", nameof(options)); + } + + RequestsPartialMessages = options.RequestPartialMessages; + SupportsSendingPartialMessages = options.SupportSendingPartialMessages; + } public void Publish(byte[] value) { router.Publish(topicName, value); } + public void PublishPartial(byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + { + router.PublishPartial(topicName, groupId, partialMessage, partsMetadata); + } + + public void SendPartial(PeerId peerId, byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + { + router.SendPartial(peerId, topicName, groupId, partialMessage, partsMetadata); + } + public void Unsubscribe() { if (IsSubscribed) router.Unsubscribe(topicName); From a5835b5b13b6d4953a52cbe6864708f342d6caaa Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 12:46:11 +0300 Subject: [PATCH 27/45] Harden partial message negotiation --- .../PartialMessagesTests.cs | 100 +++++++++++++++++- .../Libp2p.Protocols.Pubsub/PartialMessage.cs | 2 +- .../PubsubRouter.Rpc.cs | 6 +- .../PubsubRouter.Topics.cs | 11 +- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 19 ++++ src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs | 4 +- 6 files changed, 130 insertions(+), 12 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs index 2bf56733..85dcb1f6 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs @@ -16,7 +16,7 @@ public void PartialMessagesTopics_RequireExplicitRouterAndTopicOptIn() { PubsubRouter disabledRouter = new(new PeerStore()); Assert.That( - () => disabledRouter.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions { SupportSendingPartialMessages = true }), + () => disabledRouter.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }), Throws.TypeOf()); PubsubRouter enabledRouter = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); @@ -35,7 +35,7 @@ public async Task PartialMessages_AdvertiseCapabilitiesAndGatePayloadsPerPeer() new PartialMessagesTopicOptions { RequestPartialMessages = true, - SupportSendingPartialMessages = true, + SupportsSendingPartialMessages = true, }); await router.StartAsync(new LocalPeerStub()); @@ -88,7 +88,7 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); IPartialMessagesTopic topic = router.GetPartialMessagesTopic( topicName, - new PartialMessagesTopicOptions { SupportSendingPartialMessages = true }); + new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }); Multiaddress remoteAddress = TestPeers.Multiaddr(1); PeerId remotePeerId = remoteAddress.GetPeerId()!; TaskCompletionSource connection = new(); @@ -120,6 +120,26 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() Assert.That(received.PartsMetadata, Is.EqualTo(new byte[] { 3 })); }); + received = null; + router.OnRpc(remotePeerId, new Rpc + { + Partial = new PartialMessagesExtension + { + TopicID = topicName, + PartialMessage = ByteString.CopyFrom([2]), + }, + }); + router.OnRpc(remotePeerId, new Rpc + { + Partial = new PartialMessagesExtension + { + TopicID = topicName, + GroupID = ByteString.CopyFrom([1]), + }, + }); + + Assert.That(received, Is.Null); + connection.SetResult(); } @@ -127,13 +147,83 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() public void PartialMessages_AreNeverAdvertisedOnOlderProtocols() { PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); - _ = router.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions { SupportSendingPartialMessages = true }); + _ = router.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions + { + RequestPartialMessages = true, + SupportsSendingPartialMessages = true, + }); List sentRpcs = []; TaskCompletionSource connection = new(); router.OutboundConnection(TestPeers.Multiaddr(1), PubsubRouter.GossipsubProtocolVersionV12, connection.Task, sentRpcs.Add); - Assert.That(sentRpcs.Single().Control, Is.Null); + Rpc.Types.SubOpts subscription = sentRpcs.Single().Subscriptions.Single(); + Assert.Multiple(() => + { + Assert.That(sentRpcs.Single().Control, Is.Null); + Assert.That(subscription.HasRequestsPartial, Is.False); + Assert.That(subscription.HasSupportsSendingPartial, Is.False); + }); + connection.SetResult(); + } + + [Test] + public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave() + { + const string topicName = "topic"; + PubsubRouter router = new(new PeerStore(), new PubsubSettings + { + EnablePartialMessages = true, + HeartbeatInterval = int.MaxValue, + LowestDegree = 0, + LazyDegree = 1, + }); + _ = router.GetPartialMessagesTopic(topicName, new PartialMessagesTopicOptions + { + RequestPartialMessages = true, + SupportsSendingPartialMessages = true, + }); + using CancellationTokenSource cancellation = new(); + await router.StartAsync(new LocalPeerStub(), cancellation.Token); + + TaskCompletionSource connection = new(); + Dictionary> sentRpcs = []; + List? partialGossipRecipients = null; + router.OnPartialGossip += (topic, peers) => + { + Assert.That(topic, Is.EqualTo(topicName)); + partialGossipRecipients = peers.ToList(); + }; + + foreach (int peerNumber in Enumerable.Range(1, 3)) + { + Multiaddress address = TestPeers.Multiaddr(peerNumber); + PeerId peerId = address.GetPeerId()!; + List peerRpcs = []; + sentRpcs.Add(peerId, peerRpcs); + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV13, connection.Task, peerRpcs.Add); + router.OnRpc(peerId, CreateSubscriptionRpc(topicName, requestsPartialMessages: true, supportsSendingPartialMessages: true)); + } + + foreach (List peerRpcs in sentRpcs.Values) + { + peerRpcs.Clear(); + } + + Identity author = TestPeers.Identity(4); + router.OnRpc(TestPeers.PeerId(1), new Rpc().WithMessages(topicName, 1, author.PeerId.Bytes, [1, 2, 3], author)); + foreach (List peerRpcs in sentRpcs.Values) + { + peerRpcs.Clear(); + } + + await router.Heartbeat(); + + Assert.That(partialGossipRecipients, Is.Not.Null.And.Not.Empty); + Assert.That(partialGossipRecipients!.All(sentRpcs.ContainsKey), Is.True); + Assert.That(sentRpcs.Values.SelectMany(rpcs => rpcs).SelectMany(rpc => rpc.Control?.Ihave ?? []), Is.Empty); + + cancellation.Cancel(); connection.SetResult(); } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs index c2719cc9..1d13a897 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessage.cs @@ -39,5 +39,5 @@ public sealed class PartialMessagesTopicOptions /// /// Signals that this topic can send partial data and receive parts metadata. /// - public bool SupportSendingPartialMessages { get; init; } + public bool SupportsSendingPartialMessages { get; init; } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index 6b589453..ea93d2c3 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -129,9 +129,11 @@ private void HandlePartialMessage(PeerId peerId, PartialMessagesExtension partia return; } - if (!partialMessage.HasTopicID) + if (!partialMessage.HasTopicID || + !partialMessage.HasGroupID || + (!partialMessage.HasPartialMessage && !partialMessage.HasPartsMetadata)) { - logger?.LogDebug("Ignoring a Partial Messages extension payload without a topic from {peerId}", peerId); + logger?.LogDebug("Ignoring an incomplete Partial Messages extension payload from {peerId}", peerId); return; } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index 910533d5..7e98e5d4 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -58,8 +58,15 @@ private Rpc.Types.SubOpts CreateSubscription(string topicId, bool subscribe) Rpc.Types.SubOpts subscription = new() { Subscribe = subscribe, Topicid = topicId }; if (subscribe && _settings.EnablePartialMessages && topicState.TryGetValue(topicId, out Topic? topic)) { - subscription.RequestsPartial = topic.RequestsPartialMessages; - subscription.SupportsSendingPartial = topic.SupportsSendingPartialMessages; + if (topic.RequestsPartialMessages) + { + subscription.RequestsPartial = true; + } + + if (topic.SupportsSendingPartialMessages) + { + subscription.SupportsSendingPartial = true; + } } return subscription; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index dafee568..6d7e5572 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -67,6 +67,7 @@ public enum PubsubProtocol public void Send(Rpc rpc) { + rpc = RemoveUnsupportedPartialSubscriptionOptions(rpc); rpc = AddExtensionsIfNeeded(rpc); SendRpcQueue.Enqueue(rpc); if (SendRpc is not null) @@ -80,6 +81,24 @@ public void Send(Rpc rpc) } } } + + private Rpc RemoveUnsupportedPartialSubscriptionOptions(Rpc rpc) + { + if (SupportsExtensions || !rpc.Subscriptions.Any(subscription => subscription.HasRequestsPartial || subscription.HasSupportsSendingPartial)) + { + return rpc; + } + + Rpc filteredRpc = rpc.Clone(); + foreach (Rpc.Types.SubOpts subscription in filteredRpc.Subscriptions) + { + subscription.ClearRequestsPartial(); + subscription.ClearSupportsSendingPartial(); + } + + return filteredRpc; + } + public Dictionary Backoff { get; internal set; } public ConcurrentQueue SendRpcQueue { get; } private Action? _sendRpc; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs index 6af12ec0..2a9f2627 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs @@ -55,13 +55,13 @@ private void OnRouterPartialMessage(string topicName, PeerId peerId, PartialMess internal void ConfigurePartialMessages(PartialMessagesTopicOptions options) { ArgumentNullException.ThrowIfNull(options); - if (options.RequestPartialMessages && !options.SupportSendingPartialMessages) + if (options.RequestPartialMessages && !options.SupportsSendingPartialMessages) { throw new ArgumentException("Requesting partial messages requires support for sending partial messages.", nameof(options)); } RequestsPartialMessages = options.RequestPartialMessages; - SupportsSendingPartialMessages = options.SupportSendingPartialMessages; + SupportsSendingPartialMessages = options.SupportsSendingPartialMessages; } public void Publish(byte[] value) From 1b034fd75cc4507164584f173b488a279f5afa14 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 12:51:42 +0300 Subject: [PATCH 28/45] Honor partial message subscription flags --- .../PartialMessagesTests.cs | 27 +++++++++++++++++++ .../PubsubRouter.Rpc.cs | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs index 85dcb1f6..c926ff0b 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs @@ -143,6 +143,33 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() connection.SetResult(); } + [Test] + public void PartialMessages_HonorIndependentRemoteSubscriptionFlags() + { + const string topicName = "topic"; + PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + IPartialMessagesTopic topic = router.GetPartialMessagesTopic( + topicName, + new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }); + Multiaddress remoteAddress = TestPeers.Multiaddr(1); + PeerId remotePeerId = remoteAddress.GetPeerId()!; + TaskCompletionSource connection = new(); + List sentRpcs = []; + router.OutboundConnection(remoteAddress, PubsubRouter.GossipsubProtocolVersionV13, connection.Task, sentRpcs.Add); + router.OnRpc(remotePeerId, CreateSubscriptionRpc(topicName, requestsPartialMessages: true, supportsSendingPartialMessages: false)); + sentRpcs.Clear(); + + topic.SendPartial(remotePeerId, [1], partialMessage: [2], partsMetadata: [3]); + + PartialMessagesExtension sentPartial = sentRpcs.Single().Partial; + Assert.Multiple(() => + { + Assert.That(sentPartial.PartialMessage.ToByteArray(), Is.EqualTo(new byte[] { 2 })); + Assert.That(sentPartial.HasPartsMetadata, Is.False); + }); + connection.SetResult(); + } + [Test] public void PartialMessages_AreNeverAdvertisedOnOlderProtocols() { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index ea93d2c3..94974f84 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -261,7 +261,7 @@ private void HandleSubscriptions(PeerId peerId, IEnumerable s state.UpdatePartialMessagesSubscription( sub.Topicid, requestsPartialMessages, - requestsPartialMessages || sub.SupportsSendingPartial); + sub.SupportsSendingPartial); } } else From 167fe48be43db968382b2eba44bd136c3a582259 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 13:11:07 +0300 Subject: [PATCH 29/45] Complete partial message routing --- .../PartialMessagesTests.cs | 138 ++++++++++++++-- src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs | 5 +- .../Libp2p.Protocols.Pubsub/Dto/Rpc.proto | 5 +- .../PartialMessageGossipCache.cs | 104 ++++++++++++ .../Libp2p.Protocols.Pubsub/PubSubSettings.cs | 6 + .../PubsubRouter.Rpc.cs | 34 +++- .../PubsubRouter.Topics.cs | 73 +++++---- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 148 ++++++++++-------- 8 files changed, 401 insertions(+), 112 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub/PartialMessageGossipCache.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs index c926ff0b..bed16310 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs @@ -5,6 +5,7 @@ using Multiformats.Address; using Nethermind.Libp2p.Core.Discovery; using Nethermind.Libp2p.Protocols.Pubsub.Dto; +using System.Collections.Concurrent; namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; @@ -25,6 +26,50 @@ public void PartialMessagesTopics_RequireExplicitRouterAndTopicOptIn() Throws.TypeOf()); } + [Test] + public void PartialMessageGossipCache_IsBoundedAndExpiresGroups() + { + PartialMessageGossipCache cache = new(maxGroupsPerTopic: 2, groupTtlHeartbeats: 2); + + cache.Track("topic", [1]); + cache.Track("topic", [2]); + cache.Track("topic", [3]); + + Assert.That(cache.GetGroupIds("topic"), Is.EqualTo(new[] { new byte[] { 2 }, new byte[] { 3 } })); + + cache.Heartbeat(); + Assert.That(cache.GetGroupIds("topic"), Has.Count.EqualTo(2)); + + cache.Heartbeat(); + Assert.That(cache.GetGroupIds("topic"), Is.Empty); + } + + [Test] + public async Task PartialMessages_ExtensionsAreAttachedToTheFirstConcurrentRpc() + { + PubsubRouter.PubsubPeer peer = new( + TestPeers.PeerId(1), + PubsubRouter.GossipsubProtocolVersionV13, + logger: null, + settings: new PubsubSettings { EnablePartialMessages = true }); + ConcurrentQueue sentRpcs = []; + peer.SendRpc = sentRpcs.Enqueue; + + using ManualResetEventSlim start = new(initialState: false); + Task[] sends = Enumerable.Range(0, 16) + .Select(_ => Task.Run(() => + { + start.Wait(); + peer.Send(new Rpc()); + })) + .ToArray(); + start.Set(); + await Task.WhenAll(sends); + + Assert.That(sentRpcs.TryPeek(out Rpc? first), Is.True); + Assert.That(first!.Control?.Extensions?.PartialMessages, Is.True); + } + [Test] public async Task PartialMessages_AdvertiseCapabilitiesAndGatePayloadsPerPeer() { @@ -105,7 +150,7 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() { Partial = new PartialMessagesExtension { - TopicID = topicName, + TopicID = ByteString.CopyFromUtf8(topicName), GroupID = ByteString.CopyFrom([1]), PartialMessage = ByteString.CopyFrom([2]), PartsMetadata = ByteString.CopyFrom([3]), @@ -125,7 +170,7 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() { Partial = new PartialMessagesExtension { - TopicID = topicName, + TopicID = ByteString.CopyFromUtf8(topicName), PartialMessage = ByteString.CopyFrom([2]), }, }); @@ -133,7 +178,7 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() { Partial = new PartialMessagesExtension { - TopicID = topicName, + TopicID = ByteString.CopyFromUtf8(topicName), GroupID = ByteString.CopyFrom([1]), }, }); @@ -202,10 +247,11 @@ public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave { EnablePartialMessages = true, HeartbeatInterval = int.MaxValue, - LowestDegree = 0, + Degree = 1, + LowestDegree = 1, LazyDegree = 1, }); - _ = router.GetPartialMessagesTopic(topicName, new PartialMessagesTopicOptions + IPartialMessagesTopic topic = router.GetPartialMessagesTopic(topicName, new PartialMessagesTopicOptions { RequestPartialMessages = true, SupportsSendingPartialMessages = true, @@ -216,9 +262,11 @@ public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave TaskCompletionSource connection = new(); Dictionary> sentRpcs = []; List? partialGossipRecipients = null; - router.OnPartialGossip += (topic, peers) => + byte[]? partialGossipGroupId = null; + router.OnPartialGossip += (topicId, groupId, peers) => { - Assert.That(topic, Is.EqualTo(topicName)); + Assert.That(topicId, Is.EqualTo(topicName)); + partialGossipGroupId = groupId; partialGossipRecipients = peers.ToList(); }; @@ -237,8 +285,13 @@ public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave peerRpcs.Clear(); } - Identity author = TestPeers.Identity(4); - router.OnRpc(TestPeers.PeerId(1), new Rpc().WithMessages(topicName, 1, author.PeerId.Bytes, [1, 2, 3], author)); + await router.Heartbeat(); + foreach (List peerRpcs in sentRpcs.Values) + { + peerRpcs.Clear(); + } + + topic.PublishPartial([7, 8], partialMessage: [1, 2, 3]); foreach (List peerRpcs in sentRpcs.Values) { peerRpcs.Clear(); @@ -247,6 +300,7 @@ public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave await router.Heartbeat(); Assert.That(partialGossipRecipients, Is.Not.Null.And.Not.Empty); + Assert.That(partialGossipGroupId, Is.EqualTo(new byte[] { 7, 8 })); Assert.That(partialGossipRecipients!.All(sentRpcs.ContainsKey), Is.True); Assert.That(sentRpcs.Values.SelectMany(rpcs => rpcs).SelectMany(rpc => rpc.Control?.Ihave ?? []), Is.Empty); @@ -254,6 +308,72 @@ public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave connection.SetResult(); } + [Test] + public async Task PartialMessages_RequestersDoNotReceiveFullPublishesOrRelays() + { + const string topicName = "topic"; + PubsubRouter router = new(new PeerStore(), new PubsubSettings + { + EnablePartialMessages = true, + HeartbeatInterval = int.MaxValue, + Degree = 3, + LowestDegree = 3, + HighestDegree = 3, + }); + IPartialMessagesTopic topic = router.GetPartialMessagesTopic(topicName, new PartialMessagesTopicOptions + { + SupportsSendingPartialMessages = true, + }); + using CancellationTokenSource cancellation = new(); + await router.StartAsync(new LocalPeerStub(), cancellation.Token); + + TaskCompletionSource connection = new(); + Dictionary> sentRpcs = []; + foreach (int peerNumber in Enumerable.Range(1, 3)) + { + Multiaddress address = TestPeers.Multiaddr(peerNumber); + PeerId peerId = address.GetPeerId()!; + List peerRpcs = []; + sentRpcs.Add(peerId, peerRpcs); + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV13, connection.Task, peerRpcs.Add); + router.OnRpc(peerId, CreateSubscriptionRpc( + topicName, + requestsPartialMessages: peerNumber == 1, + supportsSendingPartialMessages: true)); + } + + await router.Heartbeat(); + foreach (List peerRpcs in sentRpcs.Values) + { + peerRpcs.Clear(); + } + + topic.Publish([1]); + + Assert.Multiple(() => + { + Assert.That(sentRpcs[TestPeers.PeerId(1)].SelectMany(rpc => rpc.Publish), Is.Empty); + Assert.That(sentRpcs[TestPeers.PeerId(2)].SelectMany(rpc => rpc.Publish).Count(), Is.EqualTo(1)); + }); + + foreach (List peerRpcs in sentRpcs.Values) + { + peerRpcs.Clear(); + } + + Identity author = TestPeers.Identity(4); + router.OnRpc(TestPeers.PeerId(3), new Rpc().WithMessages(topicName, 1, author.PeerId.Bytes, [2], author)); + + Assert.Multiple(() => + { + Assert.That(sentRpcs[TestPeers.PeerId(1)].SelectMany(rpc => rpc.Publish), Is.Empty); + Assert.That(sentRpcs[TestPeers.PeerId(2)].SelectMany(rpc => rpc.Publish).Count(), Is.EqualTo(1)); + }); + + cancellation.Cancel(); + connection.SetResult(); + } + private static Rpc CreateSubscriptionRpc(string topicName, bool requestsPartialMessages, bool supportsSendingPartialMessages) { Rpc rpc = new() diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs index 3d4a53de..58cc30a8 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.cs @@ -3166,8 +3166,9 @@ public void MergeFrom(pb::CodedInputStream input) { } /// - /// The Partial Messages registry identifies topics as opaque bytes. Go's - /// current string declaration is wire compatible with this canonical form. + /// The Partial Messages registry identifies topics as opaque bytes. The base + /// pubsub API uses UTF-8 topic strings, so the router validates and decodes this + /// field at its boundary. Go's current string declaration is wire compatible. /// [global::System.Diagnostics.DebuggerDisplayAttribute("{ToString(),nq}")] public sealed partial class PartialMessagesExtension : pb::IMessage diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto index dc235c7b..856b0109 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Dto/Rpc.proto @@ -71,8 +71,9 @@ message ControlExtensions { optional bool partialMessages = 10; } -// The Partial Messages registry identifies topics as opaque bytes. Go's -// current string declaration is wire compatible with this canonical form. +// The Partial Messages registry identifies topics as opaque bytes. The base +// pubsub API uses UTF-8 topic strings, so the router validates and decodes this +// field at its boundary. Go's current string declaration is wire compatible. message PartialMessagesExtension { optional bytes topicID = 1; optional bytes groupID = 2; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessageGossipCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessageGossipCache.cs new file mode 100644 index 00000000..f2a0ca1d --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PartialMessageGossipCache.cs @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +namespace Nethermind.Libp2p.Protocols.Pubsub; + +/// +/// Bounded local partial-message group state used to give applications concrete +/// group identifiers when gossiping to non-mesh peers. +/// +internal sealed class PartialMessageGossipCache +{ + private sealed class Group(byte[] id, int remainingHeartbeats) + { + public byte[] Id { get; } = id; + public int RemainingHeartbeats { get; set; } = remainingHeartbeats; + public LinkedListNode? AgeNode { get; set; } + } + + private sealed class TopicGroups + { + public Dictionary ById { get; } = []; + public LinkedList ByAge { get; } = []; + } + + private readonly Dictionary topics = []; + private readonly int maxGroupsPerTopic; + private readonly int groupTtlHeartbeats; + + public PartialMessageGossipCache(int maxGroupsPerTopic, int groupTtlHeartbeats) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxGroupsPerTopic); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(groupTtlHeartbeats); + this.maxGroupsPerTopic = maxGroupsPerTopic; + this.groupTtlHeartbeats = groupTtlHeartbeats; + } + + public void Track(string topic, ReadOnlySpan groupId) + { + TopicGroups groups = topics.GetValueOrDefault(topic) ?? CreateTopic(topic); + string key = Convert.ToHexString(groupId); + if (groups.ById.TryGetValue(key, out Group? existing)) + { + existing.RemainingHeartbeats = groupTtlHeartbeats; + groups.ByAge.Remove(existing.AgeNode!); + existing.AgeNode = groups.ByAge.AddLast(existing); + return; + } + + if (groups.ById.Count == maxGroupsPerTopic) + { + Remove(groups, groups.ByAge.First!.Value); + } + + Group group = new(groupId.ToArray(), groupTtlHeartbeats); + group.AgeNode = groups.ByAge.AddLast(group); + groups.ById.Add(key, group); + } + + public IReadOnlyList GetGroupIds(string topic) + { + return topics.TryGetValue(topic, out TopicGroups? groups) + ? groups.ByAge.Select(group => group.Id.ToArray()).ToArray() + : []; + } + + public void Heartbeat() + { + foreach ((string topic, TopicGroups groups) in topics.ToArray()) + { + LinkedListNode? node = groups.ByAge.First; + while (node is not null) + { + LinkedListNode? next = node.Next; + Group group = node.Value; + group.RemainingHeartbeats--; + if (group.RemainingHeartbeats == 0) + { + Remove(groups, group); + } + + node = next; + } + + if (groups.ById.Count == 0) + { + topics.Remove(topic); + } + } + } + + private TopicGroups CreateTopic(string topic) + { + TopicGroups groups = new(); + topics.Add(topic, groups); + return groups; + } + + private static void Remove(TopicGroups groups, Group group) + { + groups.ById.Remove(Convert.ToHexString(group.Id)); + groups.ByAge.Remove(group.AgeNode!); + group.AgeNode = null; + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index df33e9c2..a3d78577 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -46,6 +46,12 @@ public int MaxRpcBytes /// public bool EnablePartialMessages { get; set; } + /// Number of heartbeats to retain a locally published partial-message group for gossip. + public int PartialMessageGossipTtlHeartbeats { get; set; } = 3; + + /// Maximum locally published partial-message groups retained for one topic. + public int MaxPartialMessageGroupsPerTopic { get; set; } = 255; + public Func GetMessageId { get; set; } = ConcatFromAndSeqno; public enum SignaturePolicy diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index 94974f84..fa58a023 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -6,11 +6,14 @@ using Nethermind.Libp2p.Core; using Nethermind.Libp2p.Protocols.Pubsub.Dto; using System.Collections.Concurrent; +using System.Text; namespace Nethermind.Libp2p.Protocols.Pubsub; public partial class PubsubRouter : IRoutingStateContainer, IDisposable { + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + internal void OnRpc(PeerId peerId, Rpc rpc) { try @@ -137,11 +140,17 @@ private void HandlePartialMessage(PeerId peerId, PartialMessagesExtension partia return; } + if (!TryDecodeTopicId(partialMessage.TopicID, out string topicId)) + { + logger?.LogDebug("Ignoring a Partial Messages extension payload with a non-UTF-8 topic from {peerId}", peerId); + return; + } + receivedPartialMessages.Add(( - partialMessage.TopicID, + topicId, peerId, new PartialMessage( - partialMessage.TopicID, + topicId, partialMessage.GroupID.ToByteArray(), partialMessage.HasPartialMessage ? partialMessage.PartialMessage.ToByteArray() : null, partialMessage.HasPartsMetadata ? partialMessage.PartsMetadata.ToByteArray() : null))); @@ -213,7 +222,10 @@ private void HandleNewMessages(PeerId peerId, IEnumerable messages, Con { continue; } - peerMessages.GetOrAdd(peer, _ => new Rpc()).Publish.Add(message); + if (ShouldSendFullMessage(peer, message.Topic)) + { + peerMessages.GetOrAdd(peer, _ => new Rpc()).Publish.Add(message); + } } } if (mesh.TryGetValue(message.Topic, out topicPeers)) @@ -226,7 +238,7 @@ private void HandleNewMessages(PeerId peerId, IEnumerable messages, Con } // Only forward to peers above publish threshold (Gossipsub v1.1) - if (GetPeerScore(peer) >= _settings.PublishThreshold) + if (GetPeerScore(peer) >= _settings.PublishThreshold && ShouldSendFullMessage(peer, message.Topic)) { peerMessages.GetOrAdd(peer, _ => new Rpc()).Publish.Add(message); } @@ -235,6 +247,20 @@ private void HandleNewMessages(PeerId peerId, IEnumerable messages, Con } } + private static bool TryDecodeTopicId(ByteString topicIdBytes, out string topicId) + { + try + { + topicId = StrictUtf8.GetString(topicIdBytes.Span); + return true; + } + catch (DecoderFallbackException) + { + topicId = string.Empty; + return false; + } + } + private void HandleSubscriptions(PeerId peerId, IEnumerable subscriptions) { foreach (Rpc.Types.SubOpts? sub in subscriptions) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index 7e98e5d4..4c5fd3d7 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -184,10 +184,13 @@ public void Publish(string topicId, byte[] message) ulong seqNo = this.seqNo++; Rpc rpc = new Rpc().WithMessages(topicId, seqNo, localPeer.Identity.PeerId.Bytes, message, localPeer.Identity); - // Floodsub peers always get the message + // Floodsub peers always get the message. foreach (PeerId peerId in fPeers.GetValueOrDefault(topicId) ?? []) { - peerState.GetValueOrDefault(peerId)?.Send(rpc); + if (ShouldSendFullMessage(peerId, topicId)) + { + peerState.GetValueOrDefault(peerId)?.Send(rpc); + } } // Gossipsub v1.1: Flood publishing @@ -196,50 +199,47 @@ public void Publish(string topicId, byte[] message) // Send to all gossipsub peers above publish threshold foreach (PeerId peerId in allGossipsubPeers) { - if (GetPeerScore(peerId) >= _settings.PublishThreshold) + if (GetPeerScore(peerId) >= _settings.PublishThreshold && ShouldSendFullMessage(peerId, topicId)) { peerState.GetValueOrDefault(peerId)?.Send(rpc); } } } - else + else if (mesh.TryGetValue(topicId, out HashSet? meshPeers)) { // Standard gossipsub v1.0 behavior: send to mesh or fanout - if (mesh.TryGetValue(topicId, out HashSet? meshPeers)) + foreach (PeerId peerId in meshPeers) { - foreach (PeerId peerId in meshPeers) + if (GetPeerScore(peerId) >= _settings.PublishThreshold && ShouldSendFullMessage(peerId, topicId)) { - if (GetPeerScore(peerId) >= _settings.PublishThreshold) - { - peerState.GetValueOrDefault(peerId)?.Send(rpc); - } + peerState.GetValueOrDefault(peerId)?.Send(rpc); } } - else - { - fanoutLastPublished[topicId] = DateTime.Now; - HashSet topicFanout = fanout.GetOrAdd(topicId, _ => []); + } + else + { + fanoutLastPublished[topicId] = DateTime.Now; + HashSet topicFanout = fanout.GetOrAdd(topicId, _ => []); - if (topicFanout.Count == 0) + if (topicFanout.Count == 0) + { + HashSet? topicPeers = gPeers.GetValueOrDefault(topicId); + if (topicPeers is { Count: > 0 }) { - HashSet? topicPeers = gPeers.GetValueOrDefault(topicId); - if (topicPeers is { Count: > 0 }) + // Select peers with non-negative scores + var eligiblePeers = topicPeers.Where(p => GetPeerScore(p) >= 0).ToList(); + foreach (PeerId peer in eligiblePeers.Take(_settings.Degree)) { - // Select peers with non-negative scores - var eligiblePeers = topicPeers.Where(p => GetPeerScore(p) >= 0).ToList(); - foreach (PeerId peer in eligiblePeers.Take(_settings.Degree)) - { - topicFanout.Add(peer); - } + topicFanout.Add(peer); } } + } - foreach (PeerId peerId in topicFanout) + foreach (PeerId peerId in topicFanout) + { + if (GetPeerScore(peerId) >= _settings.PublishThreshold && ShouldSendFullMessage(peerId, topicId)) { - if (GetPeerScore(peerId) >= _settings.PublishThreshold) - { - peerState.GetValueOrDefault(peerId)?.Send(rpc); - } + peerState.GetValueOrDefault(peerId)?.Send(rpc); } } } @@ -259,6 +259,11 @@ public void PublishPartial(string topicId, byte[] groupId, byte[]? partialMessag throw new InvalidOperationException("Router has not been started. Call StartAsync() first."); } + lock (this) + { + partialMessageGossip.Track(topicId, groupId); + } + PeerId[] recipients; if (mesh.TryGetValue(topicId, out HashSet? meshPeers) && meshPeers.Count > 0) { @@ -307,7 +312,7 @@ public void SendPartial(PeerId peerId, string topicId, byte[] groupId, byte[]? p PartialMessagesExtension partial = new() { - TopicID = topicId, + TopicID = Google.Protobuf.ByteString.CopyFromUtf8(topicId), GroupID = Google.Protobuf.ByteString.CopyFrom(groupId), }; if (sendPartialData && partialMessage is not null) @@ -337,6 +342,16 @@ private void EnsurePartialMessagesEnabled(string topicId) } } + private bool ShouldSendFullMessage(PeerId peerId, string topicId) + { + return !_settings.EnablePartialMessages || + !topicState.TryGetValue(topicId, out Topic? topic) || + !topic.SupportsSendingPartialMessages || + !peerState.TryGetValue(peerId, out PubsubPeer? peer) || + !peer.SupportsPartialMessagesExtension || + !peer.RequestsPartialMessages(topicId); + } + private static void ValidatePartialMessage(byte[] groupId, byte[]? partialMessage, byte[]? partsMetadata) { ArgumentNullException.ThrowIfNull(groupId); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index 6d7e5572..e4b5beb6 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -33,7 +33,7 @@ public override string ToString() public const string GossipsubProtocolVersionV12 = "/meshsub/1.2.0"; public const string GossipsubProtocolVersionV13 = "/meshsub/1.3.0"; - class PubsubPeer + internal sealed class PubsubPeer { public PubsubPeer(PeerId peerId, string protocolId, ILogger? logger, PubsubSettings settings) { @@ -67,16 +67,16 @@ public enum PubsubProtocol public void Send(Rpc rpc) { - rpc = RemoveUnsupportedPartialSubscriptionOptions(rpc); - rpc = AddExtensionsIfNeeded(rpc); - SendRpcQueue.Enqueue(rpc); - if (SendRpc is not null) + lock (SendRpcQueue) { - lock (SendRpcQueue) + rpc = RemoveUnsupportedPartialSubscriptionOptions(rpc); + rpc = AddExtensionsIfNeeded(rpc); + SendRpcQueue.Enqueue(rpc); + if (_sendRpc is not null) { while (SendRpcQueue.TryDequeue(out Rpc? rpcToSend)) { - SendRpc.Invoke(rpcToSend); + _sendRpc.Invoke(rpcToSend); } } } @@ -104,7 +104,6 @@ private Rpc RemoveUnsupportedPartialSubscriptionOptions(Rpc rpc) private Action? _sendRpc; private readonly ILogger? _logger; private readonly bool _advertisesPartialMessages; - private readonly object _extensionsLock = new(); private readonly ConcurrentDictionary _partialMessagesSubscriptions = new(); private readonly record struct PartialMessagesSubscription(bool RequestsPartialMessages, bool SupportsSendingPartialMessages); @@ -117,7 +116,7 @@ public bool NeedsExtensions { get { - lock (_extensionsLock) + lock (SendRpcQueue) { return _advertisesPartialMessages && SupportsExtensions && !ExtensionsSent; } @@ -151,36 +150,34 @@ private Rpc AddExtensionsIfNeeded(Rpc rpc) return rpc; } - lock (_extensionsLock) - { - if (ExtensionsSent) - { - return rpc; - } - - Rpc extendedRpc = rpc.Clone(); - extendedRpc.Control ??= new ControlMessage(); - extendedRpc.Control.Extensions ??= new ControlExtensions(); - extendedRpc.Control.Extensions.PartialMessages = true; - ExtensionsSent = true; - return extendedRpc; - } + Rpc extendedRpc = rpc.Clone(); + extendedRpc.Control ??= new ControlMessage(); + extendedRpc.Control.Extensions ??= new ControlExtensions(); + extendedRpc.Control.Extensions.PartialMessages = true; + ExtensionsSent = true; + return extendedRpc; } public Action? SendRpc { - get => _sendRpc; set + get { - _logger?.LogDebug($"Set SENDRPC for {PeerId}: {value}"); - _sendRpc = value; - if (_sendRpc is not null) - lock (SendRpcQueue) + lock (SendRpcQueue) + { + return _sendRpc; + } + } + set + { + lock (SendRpcQueue) + { + _logger?.LogDebug($"Set SENDRPC for {PeerId}: {value}"); + _sendRpc = value; + while (_sendRpc is not null && SendRpcQueue.TryDequeue(out Rpc? rpcToSend)) { - while (SendRpcQueue.TryDequeue(out Rpc? rpcToSend)) - { - _sendRpc.Invoke(rpcToSend); - } + _sendRpc.Invoke(rpcToSend); } + } } } public CancellationTokenSource TokenSource { get; init; } @@ -220,10 +217,11 @@ public Action? SendRpc public event Action? OnPartialMessage; /// - /// Raised with non-mesh peers that requested partial messages instead of an - /// IHAVE announcement. Applications can respond by calling . + /// Raised with a locally published partial-message group and non-mesh peers + /// that requested it instead of an IHAVE announcement. Applications can + /// respond by calling for the supplied group. /// - public event Action>? OnPartialGossip; + public event Action>? OnPartialGossip; public Func? VerifyMessage = null; internal int MaxRpcBytes => _settings.MaxRpcBytes; @@ -232,6 +230,7 @@ public Action? SendRpc private readonly TtlCache _messageCache; private readonly TtlCache _limboMessageCache; private readonly TtlCache<(PeerId, MessageId)> _idontwantMessages; + private readonly PartialMessageGossipCache partialMessageGossip; private ILocalPeer? localPeer; private readonly ILogger? logger; @@ -279,6 +278,7 @@ public PubsubRouter(PeerStore store, PubsubSettings? settings = null, ILoggerFac _messageCache = new(_settings.MessageCacheTtl); _limboMessageCache = new(_settings.MessageCacheTtl); _idontwantMessages = new(_settings.MessageCacheTtl); + partialMessageGossip = new(_settings.MaxPartialMessageGroupsPerTopic, _settings.PartialMessageGossipTtlHeartbeats); } public Task StartAsync(ILocalPeer localPeer, CancellationToken token = default) @@ -405,8 +405,8 @@ public Task Heartbeat() DecayScores(); ConcurrentDictionary peerMessages = new(); - List<(string Topic, PeerId[] Peers)> partialGossipNotifications = []; - Action>? onPartialGossip = OnPartialGossip; + List<(string Topic, byte[] GroupId, PeerId[] Peers)> partialGossipNotifications = []; + Action>? onPartialGossip = OnPartialGossip; lock (this) { // First, prune peers with negative scores from all meshes (Gossipsub v1.1) @@ -537,42 +537,56 @@ public Task Heartbeat() .ToArray()) { IGrouping? msgsInTopic = msgs.FirstOrDefault(mit => mit.Key == topic); - if (msgsInTopic is not null && gPeers.TryGetValue(topic, out HashSet? topicGossipsubPeers)) + bool canGossipPartialMessages = + onPartialGossip is not null && + _settings.EnablePartialMessages && + topicState.TryGetValue(topic, out Topic? localTopic) && + localTopic.SupportsSendingPartialMessages; + IReadOnlyList partialMessageGroupIds = canGossipPartialMessages + ? partialMessageGossip.GetGroupIds(topic) + : []; + if (msgsInTopic is null && partialMessageGroupIds.Count == 0) { - ControlIHave ihave = new() { TopicID = topic }; - ihave.MessageIDs.AddRange(msgsInTopic.Select(m => ByteString.CopyFrom(m.Id.Bytes))); + continue; + } - // Only send gossip to peers above gossip threshold - HashSet? topicMesh = mesh.GetValueOrDefault(topic); - HashSet? topicFanout = fanout.GetValueOrDefault(topic); - PeerId[] eligiblePeers = topicGossipsubPeers - .Where(p => !(topicMesh?.Contains(p) ?? false) - && !(topicFanout?.Contains(p) ?? false) - && GetPeerScore(p) >= _settings.GossipThreshold) - .ToArray(); + // Only send gossip to peers above gossip threshold. + HashSet fanoutPeers = fanout.GetValueOrDefault(topic) ?? []; + HashSet topicMesh = mesh.GetValueOrDefault(topic) ?? []; + HashSet topicGossipsubPeers = gPeers.GetValueOrDefault(topic) ?? []; + PeerId[] eligiblePeers = topicGossipsubPeers + .Where(p => !topicMesh.Contains(p) + && !fanoutPeers.Contains(p) + && GetPeerScore(p) >= _settings.GossipThreshold) + .ToArray(); - // Adaptive gossip: send to gossip_factor of eligible peers (min D_lazy) - int gossipCount = Math.Max(_settings.LazyDegree, (int)(eligiblePeers.Length * _settings.GossipFactor)); - PeerId[] gossipPeers = eligiblePeers.Take(gossipCount).ToArray(); + // Adaptive gossip: send to gossip_factor of eligible peers (min D_lazy). + int gossipCount = Math.Max(_settings.LazyDegree, (int)(eligiblePeers.Length * _settings.GossipFactor)); + PeerId[] gossipPeers = eligiblePeers.Take(gossipCount).ToArray(); - if (onPartialGossip is not null && - _settings.EnablePartialMessages && - topicState.TryGetValue(topic, out Topic? localTopic) && - localTopic.SupportsSendingPartialMessages) - { - PeerId[] partialGossipPeers = gossipPeers - .Where(peerId => peerState.TryGetValue(peerId, out PubsubPeer? peer) && - peer.SupportsPartialMessagesExtension && - peer.RequestsPartialMessages(topic)) - .ToArray(); + if (partialMessageGroupIds.Count > 0) + { + PeerId[] partialGossipPeers = gossipPeers + .Where(peerId => peerState.TryGetValue(peerId, out PubsubPeer? peer) && + peer.SupportsPartialMessagesExtension && + peer.RequestsPartialMessages(topic)) + .ToArray(); - if (partialGossipPeers.Length > 0) + if (partialGossipPeers.Length > 0) + { + foreach (byte[] groupId in partialMessageGroupIds) { - partialGossipNotifications.Add((topic, partialGossipPeers)); - gossipPeers = gossipPeers.Except(partialGossipPeers).ToArray(); + partialGossipNotifications.Add((topic, groupId, partialGossipPeers)); } + + gossipPeers = gossipPeers.Except(partialGossipPeers).ToArray(); } + } + if (msgsInTopic is not null) + { + ControlIHave ihave = new() { TopicID = topic }; + ihave.MessageIDs.AddRange(msgsInTopic.Select(m => ByteString.CopyFrom(m.Id.Bytes))); foreach (PeerId peer in gossipPeers) { peerMessages.GetOrAdd(peer, _ => new Rpc()) @@ -580,6 +594,8 @@ public Task Heartbeat() } } } + + partialMessageGossip.Heartbeat(); } foreach (KeyValuePair peerMessage in peerMessages) @@ -587,9 +603,9 @@ public Task Heartbeat() peerState.GetValueOrDefault(peerMessage.Key)?.Send(peerMessage.Value); } - foreach ((string topic, PeerId[] peers) in partialGossipNotifications) + foreach ((string topic, byte[] groupId, PeerId[] peers) in partialGossipNotifications) { - onPartialGossip?.Invoke(topic, peers); + onPartialGossip?.Invoke(topic, groupId, peers); } return Task.CompletedTask; From ead2e857b29495fbd3709a5a34f155031010c7d5 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 13:44:06 +0300 Subject: [PATCH 30/45] Serialize partial subscription announcements --- .../PubsubRouter.Topics.cs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index 4c5fd3d7..747b0fab 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -74,11 +74,19 @@ private Rpc.Types.SubOpts CreateSubscription(string topicId, bool subscribe) private void AnnounceSubscription(string topicId) { - Rpc topicUpdate = new(); - topicUpdate.Subscriptions.Add(CreateSubscription(topicId, subscribe: true)); - foreach (KeyValuePair peer in peerState) + lock (this) { - peer.Value.Send(topicUpdate); + if (topicState.GetValueOrDefault(topicId)?.IsSubscribed is not true) + { + return; + } + + Rpc topicUpdate = new(); + topicUpdate.Subscriptions.Add(CreateSubscription(topicId, subscribe: true)); + foreach (KeyValuePair peer in peerState) + { + peer.Value.Send(topicUpdate); + } } } From dd5a697b20304775a8f4f672755663cb51d3578e Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 14:07:46 +0300 Subject: [PATCH 31/45] Keep partial-message topics opt-in --- .../PartialMessagesTests.cs | 18 +++++ .../PubsubRouter.Topics.cs | 4 +- src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs | 80 +++++++++++++------ 3 files changed, 75 insertions(+), 27 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs index bed16310..4ed39383 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs @@ -26,6 +26,24 @@ public void PartialMessagesTopics_RequireExplicitRouterAndTopicOptIn() Throws.TypeOf()); } + [Test] + public void GetTopic_DoesNotExposePartialMessagesWithoutTheExplicitApi() + { + PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + ITopic topic = router.GetTopic("topic"); + + IPartialMessagesTopic partialTopic = router.GetPartialMessagesTopic( + "topic", + new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }); + + Assert.Multiple(() => + { + Assert.That(topic, Is.Not.InstanceOf()); + Assert.That(partialTopic, Is.Not.SameAs(topic)); + Assert.That(partialTopic.IsSubscribed, Is.True); + }); + } + [Test] public void PartialMessageGossipCache_IsBoundedAndExpiresGroups() { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index 747b0fab..d5bdd5ae 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -38,7 +38,7 @@ public IPartialMessagesTopic GetPartialMessagesTopic(string topicId, PartialMess Topic topic = topicState.GetOrAdd(topicId, (tId) => new(this, tId)); bool wasSubscribed = topic.IsSubscribed; - topic.ConfigurePartialMessages(options); + IPartialMessagesTopic partialMessagesTopic = topic.ConfigurePartialMessages(options); if (subscribe) { @@ -50,7 +50,7 @@ public IPartialMessagesTopic GetPartialMessagesTopic(string topicId, PartialMess AnnounceSubscription(topicId); } - return topic; + return partialMessagesTopic; } private Rpc.Types.SubOpts CreateSubscription(string topicId, bool subscribe) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs index 2a9f2627..62f1d817 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs @@ -5,17 +5,17 @@ namespace Nethermind.Libp2p.Protocols.Pubsub; -internal class Topic : IPartialMessagesTopic +internal class Topic : ITopic { private readonly PubsubRouter router; private readonly string topicName; + private PartialMessagesTopic? partialMessagesTopic; public Topic(PubsubRouter router, string topicName) { this.router = router; this.topicName = topicName; router.OnMessage += OnRouterMessage; - router.OnPartialMessage += OnRouterPartialMessage; } private void OnRouterMessage(string topicName, PeerId peerId, byte[] message) @@ -32,27 +32,17 @@ private void OnRouterMessage(string topicName, PeerId peerId, byte[] message) } } - private void OnRouterPartialMessage(string topicName, PeerId peerId, PartialMessage message) - { - if (this.topicName != topicName) - { - return; - } - - Action? onPartialMessage = OnPartialMessage; - onPartialMessage?.Invoke(peerId, message); - } - public DateTime LastPublished { get; set; } public bool IsSubscribed { get; internal set; } - public bool RequestsPartialMessages { get; private set; } - public bool SupportsSendingPartialMessages { get; private set; } + internal bool RequestsPartialMessages { get; private set; } + internal bool SupportsSendingPartialMessages { get; private set; } + internal PubsubRouter Router => router; + internal string Name => topicName; public event Action? OnMessage; - public event Action? OnPartialMessage; - internal void ConfigurePartialMessages(PartialMessagesTopicOptions options) + internal IPartialMessagesTopic ConfigurePartialMessages(PartialMessagesTopicOptions options) { ArgumentNullException.ThrowIfNull(options); if (options.RequestPartialMessages && !options.SupportsSendingPartialMessages) @@ -62,6 +52,7 @@ internal void ConfigurePartialMessages(PartialMessagesTopicOptions options) RequestsPartialMessages = options.RequestPartialMessages; SupportsSendingPartialMessages = options.SupportsSendingPartialMessages; + return partialMessagesTopic ??= new PartialMessagesTopic(this); } public void Publish(byte[] value) @@ -69,23 +60,62 @@ public void Publish(byte[] value) router.Publish(topicName, value); } - public void PublishPartial(byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + public void Unsubscribe() { - router.PublishPartial(topicName, groupId, partialMessage, partsMetadata); + if (IsSubscribed) router.Unsubscribe(topicName); } - public void SendPartial(PeerId peerId, byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + public void Subscribe() { - router.SendPartial(peerId, topicName, groupId, partialMessage, partsMetadata); + if (!IsSubscribed) router.Subscribe(topicName); } +} - public void Unsubscribe() +internal sealed class PartialMessagesTopic : IPartialMessagesTopic +{ + private readonly Topic topic; + + public PartialMessagesTopic(Topic topic) { - if (IsSubscribed) router.Unsubscribe(topicName); + this.topic = topic; + topic.Router.OnPartialMessage += OnRouterPartialMessage; } - public void Subscribe() + public event Action? OnMessage { - if (!IsSubscribed) router.Subscribe(topicName); + add => topic.OnMessage += value; + remove => topic.OnMessage -= value; + } + + public event Action? OnPartialMessage; + + public bool IsSubscribed => topic.IsSubscribed; + public bool RequestsPartialMessages => topic.RequestsPartialMessages; + public bool SupportsSendingPartialMessages => topic.SupportsSendingPartialMessages; + + public void Publish(byte[] value) => topic.Publish(value); + + public void PublishPartial(byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + { + topic.Router.PublishPartial(topic.Name, groupId, partialMessage, partsMetadata); + } + + public void SendPartial(PeerId peerId, byte[] groupId, byte[]? partialMessage = null, byte[]? partsMetadata = null) + { + topic.Router.SendPartial(peerId, topic.Name, groupId, partialMessage, partsMetadata); + } + + public void Unsubscribe() => topic.Unsubscribe(); + + public void Subscribe() => topic.Subscribe(); + + private void OnRouterPartialMessage(string topicName, PeerId peerId, PartialMessage message) + { + if (topic.Name != topicName) + { + return; + } + + OnPartialMessage?.Invoke(peerId, message); } } From 452436152087799e19a724dbd45ab96394d64e4b Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 12:08:07 +0300 Subject: [PATCH 32/45] Add Gossipsub direct peer support --- .../DirectPeersTests.cs | 155 ++++++++++++++++++ .../Libp2p.Protocols.Pubsub/PubSubSettings.cs | 13 ++ .../PubsubRouter.Rpc.cs | 23 ++- .../PubsubRouter.Topics.cs | 25 ++- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 58 ++++++- 5 files changed, 265 insertions(+), 9 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs new file mode 100644 index 00000000..f637141c --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Multiformats.Address; +using Nethermind.Libp2p.Core.Discovery; +using Nethermind.Libp2p.Protocols; +using Nethermind.Libp2p.Protocols.Pubsub.Dto; +using System.Collections.ObjectModel; + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +public class DirectPeersTests +{ + [Test] + public void DirectPeers_ForwardValidMessagesDespitePeerScores() + { + const string topic = "topic"; + Multiaddress senderAddress = TestPeers.Multiaddr(1); + Multiaddress receiverAddress = TestPeers.Multiaddr(2); + PeerId senderPeerId = senderAddress.GetPeerId()!; + PeerId receiverPeerId = receiverAddress.GetPeerId()!; + PubsubRouter router = new( + new PeerStore(), + new PubsubSettings { DirectPeers = [senderAddress, receiverAddress] }); + ITopic localTopic = router.GetTopic(topic); + List senderRpcs = []; + List receiverRpcs = []; + TaskCompletionSource senderConnection = new(); + TaskCompletionSource receiverConnection = new(); + router.OutboundConnection(senderAddress, PubsubRouter.GossipsubProtocolVersionV11, senderConnection.Task, senderRpcs.Add); + router.OutboundConnection(receiverAddress, PubsubRouter.GossipsubProtocolVersionV11, receiverConnection.Task, receiverRpcs.Add); + router.OnRpc(senderPeerId, new Rpc().WithTopics([topic], [])); + router.OnRpc(receiverPeerId, new Rpc().WithTopics([topic], [])); + + router.SetAppSpecificScore(senderPeerId, -20); + router.SetAppSpecificScore(receiverPeerId, -20); + senderRpcs.Clear(); + receiverRpcs.Clear(); + + PeerId? receivedFrom = null; + localTopic.OnMessage += (peerId, _) => receivedFrom = peerId; + Identity author = TestPeers.Identity(1); + router.OnRpc(senderPeerId, new Rpc().WithMessages(topic, 1, author.PeerId.Bytes, [1, 2, 3], author)); + + Assert.Multiple(() => + { + Assert.That(receivedFrom, Is.EqualTo(senderPeerId)); + Assert.That(receiverRpcs.Single().Publish.Single().Data.ToByteArray(), Is.EqualTo(new byte[] { 1, 2, 3 })); + Assert.That(senderRpcs, Is.Empty); + }); + + senderConnection.SetResult(); + receiverConnection.SetResult(); + } + + [Test] + public void DirectPeers_AreNeverAddedToTheMesh() + { + const string topic = "topic"; + Multiaddress directAddress = TestPeers.Multiaddr(1); + Multiaddress firstMeshAddress = TestPeers.Multiaddr(2); + Multiaddress secondMeshAddress = TestPeers.Multiaddr(3); + PubsubRouter router = new( + new PeerStore(), + new PubsubSettings { DirectPeers = [directAddress] }); + IRoutingStateContainer state = router; + _ = router.GetTopic(topic); + TaskCompletionSource connection = new(); + + foreach (Multiaddress address in new[] { directAddress, firstMeshAddress, secondMeshAddress }) + { + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV11, connection.Task, _ => { }); + router.OnRpc(address.GetPeerId()!, new Rpc().WithTopics([topic], [])); + } + + router.Heartbeat().GetAwaiter().GetResult(); + + Assert.Multiple(() => + { + Assert.That(state.GossipsubPeers[topic], Has.Member(directAddress.GetPeerId())); + Assert.That(state.Mesh[topic], Does.Not.Contain(directAddress.GetPeerId())); + Assert.That(state.Mesh[topic], Has.Count.EqualTo(2)); + }); + + connection.SetResult(); + } + + [Test] + public void DirectPeerGrafts_AreRejectedWithPrune() + { + const string topic = "topic"; + Multiaddress directAddress = TestPeers.Multiaddr(1); + PeerId directPeerId = directAddress.GetPeerId()!; + PubsubRouter router = new( + new PeerStore(), + new PubsubSettings { DirectPeers = [directAddress] }); + _ = router.GetTopic(topic); + List sentRpcs = []; + TaskCompletionSource connection = new(); + router.OutboundConnection(directAddress, PubsubRouter.GossipsubProtocolVersionV11, connection.Task, sentRpcs.Add); + sentRpcs.Clear(); + + Rpc graft = new() { Control = new ControlMessage() }; + graft.Control.Graft.Add(new ControlGraft { TopicID = topic }); + router.OnRpc(directPeerId, graft); + + Assert.That(sentRpcs.Single().Control.Prune.Single().TopicID, Is.EqualTo(topic)); + connection.SetResult(); + } + + [Test] + public async Task Router_ConnectsConfiguredDirectPeersAtStartup() + { + PeerStore peerStore = new(); + Multiaddress directAddress = TestPeers.Multiaddr(1); + PeerId directPeerId = directAddress.GetPeerId()!; + peerStore.GetPeerInfo(directPeerId).SupportedProtocols = [PubsubRouter.GossipsubProtocolVersionV12]; + PubsubRouter router = new(peerStore, new PubsubSettings { DirectPeers = [directAddress] }); + + TaskCompletionSource protocolDialed = new(TaskCreationOptions.RunContinuationsAsynchronously); + ISession session = Substitute.For(); + session.RemoteAddress.Returns(directAddress); + session.DialAsync(Arg.Any()).Returns(_ => + { + protocolDialed.TrySetResult(); + return Task.CompletedTask; + }); + + ILocalPeer localPeer = Substitute.For(); + localPeer.Identity.Returns(TestPeers.Identity(2)); + localPeer.ListenAddresses.Returns(new ObservableCollection()); + localPeer.DialAsync(Arg.Any(), Arg.Any()).Returns(session); + + using CancellationTokenSource cancellation = new(); + await router.StartAsync(localPeer, cancellation.Token); + + await protocolDialed.Task.WaitAsync(TimeSpan.FromSeconds(2)); + _ = localPeer.Received(1).DialAsync(Arg.Any(), Arg.Any()); + _ = session.Received(1).DialAsync(Arg.Any()); + + cancellation.Cancel(); + } + + [Test] + public void DirectPeers_RequirePeerIdsInTheirAddresses() + { + PubsubSettings settings = new() + { + DirectPeers = new[] { Multiaddress.Decode("/ip4/127.0.0.1/tcp/4001") }, + }; + + Assert.That(() => new PubsubRouter(new PeerStore(), settings), Throws.TypeOf()); + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index a3d78577..1d8782d7 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -3,6 +3,7 @@ using Nethermind.Libp2p.Core; using Nethermind.Libp2p.Protocols.Pubsub.Dto; +using Multiformats.Address; namespace Nethermind.Libp2p.Protocols.Pubsub; @@ -22,6 +23,18 @@ public class PubsubSettings public int MaxConnections { get; set; } + /// + /// Peers with reciprocal explicit peering agreements. Each address must + /// contain a peer ID and is configured before the router starts. + /// + public Multiaddress[] DirectPeers { get; set; } = []; + + /// + /// Interval for reconnecting disconnected direct peers. Gossipsub recommends + /// five minutes. + /// + public int DirectConnectPeriod { get; set; } = 5 * 60 * 1000; + public int HeartbeatInterval { get; set; } = 1_000; // Time between heartbeats 1 second public int FanoutTtl { get; set; } = 60 * 1000; // Time-to-live for each topic's fanout state 60 seconds public int mcache_len { get; set; } = 5; // Number of history windows in message cache 5 diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index fa58a023..304028cd 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -159,7 +159,7 @@ private void HandlePartialMessage(PeerId peerId, PartialMessagesExtension partia private void HandleNewMessages(PeerId peerId, IEnumerable messages, ConcurrentDictionary peerMessages, List<(string Topic, PeerId PeerId, byte[] Data)> receivedMessages) { // Check if peer is graylisted (Gossipsub v1.1) - if (ShouldGraylistPeer(peerId)) + if (!IsDirectPeer(peerId) && ShouldGraylistPeer(peerId)) { logger?.LogDebug("Ignoring messages from graylisted peer {peerId}", peerId); return; @@ -214,11 +214,19 @@ private void HandleNewMessages(PeerId peerId, IEnumerable messages, Con PeerId author = new(message.From.ToArray()); receivedMessages.Add((message.Topic, peerId, message.Data.ToByteArray())); + foreach (PeerId directPeerId in GetDirectPeersForTopic(message.Topic)) + { + if (directPeerId != author && directPeerId != peerId && ShouldSendFullMessage(directPeerId, message.Topic)) + { + peerMessages.GetOrAdd(directPeerId, _ => new Rpc()).Publish.Add(message); + } + } + if (fPeers.TryGetValue(message.Topic, out HashSet? topicPeers)) { foreach (PeerId peer in topicPeers) { - if (peer == author || peer == peerId) + if (peer == author || peer == peerId || IsDirectPeer(peer)) { continue; } @@ -232,7 +240,7 @@ private void HandleNewMessages(PeerId peerId, IEnumerable messages, Con { foreach (PeerId peer in topicPeers) { - if (peer == author || peer == peerId) + if (peer == author || peer == peerId || IsDirectPeer(peer)) { continue; } @@ -320,6 +328,15 @@ private void HandleGraft(PeerId peerId, IEnumerable grafts, Concur { foreach (ControlGraft? graft in grafts) { + if (IsDirectPeer(peerId)) + { + logger?.LogWarning("Rejecting GRAFT from direct peer {peerId} for topic {topic}", peerId, graft.TopicID); + peerMessages.GetOrAdd(peerId, _ => new Rpc()) + .Ensure(r => r.Control.Prune) + .Add(new ControlPrune { TopicID = graft.TopicID }); + continue; + } + if (topicState.GetValueOrDefault(graft.TopicID)?.IsSubscribed is not true || !mesh.TryGetValue(graft.TopicID, out HashSet? topicMesh)) { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index d5bdd5ae..4f18bae7 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -192,10 +192,19 @@ public void Publish(string topicId, byte[] message) ulong seqNo = this.seqNo++; Rpc rpc = new Rpc().WithMessages(topicId, seqNo, localPeer.Identity.PeerId.Bytes, message, localPeer.Identity); + HashSet directRecipients = GetDirectPeersForTopic(topicId).ToHashSet(); + foreach (PeerId peerId in directRecipients) + { + if (ShouldSendFullMessage(peerId, topicId)) + { + peerState.GetValueOrDefault(peerId)?.Send(rpc); + } + } + // Floodsub peers always get the message. foreach (PeerId peerId in fPeers.GetValueOrDefault(topicId) ?? []) { - if (ShouldSendFullMessage(peerId, topicId)) + if (!directRecipients.Contains(peerId) && ShouldSendFullMessage(peerId, topicId)) { peerState.GetValueOrDefault(peerId)?.Send(rpc); } @@ -207,7 +216,9 @@ public void Publish(string topicId, byte[] message) // Send to all gossipsub peers above publish threshold foreach (PeerId peerId in allGossipsubPeers) { - if (GetPeerScore(peerId) >= _settings.PublishThreshold && ShouldSendFullMessage(peerId, topicId)) + if (!directRecipients.Contains(peerId) && + GetPeerScore(peerId) >= _settings.PublishThreshold && + ShouldSendFullMessage(peerId, topicId)) { peerState.GetValueOrDefault(peerId)?.Send(rpc); } @@ -218,7 +229,9 @@ public void Publish(string topicId, byte[] message) // Standard gossipsub v1.0 behavior: send to mesh or fanout foreach (PeerId peerId in meshPeers) { - if (GetPeerScore(peerId) >= _settings.PublishThreshold && ShouldSendFullMessage(peerId, topicId)) + if (!directRecipients.Contains(peerId) && + GetPeerScore(peerId) >= _settings.PublishThreshold && + ShouldSendFullMessage(peerId, topicId)) { peerState.GetValueOrDefault(peerId)?.Send(rpc); } @@ -235,7 +248,7 @@ public void Publish(string topicId, byte[] message) if (topicPeers is { Count: > 0 }) { // Select peers with non-negative scores - var eligiblePeers = topicPeers.Where(p => GetPeerScore(p) >= 0).ToList(); + var eligiblePeers = topicPeers.Where(p => !IsDirectPeer(p) && GetPeerScore(p) >= 0).ToList(); foreach (PeerId peer in eligiblePeers.Take(_settings.Degree)) { topicFanout.Add(peer); @@ -245,7 +258,9 @@ public void Publish(string topicId, byte[] message) foreach (PeerId peerId in topicFanout) { - if (GetPeerScore(peerId) >= _settings.PublishThreshold && ShouldSendFullMessage(peerId, topicId)) + if (!directRecipients.Contains(peerId) && + GetPeerScore(peerId) >= _settings.PublishThreshold && + ShouldSendFullMessage(peerId, topicId)) { peerState.GetValueOrDefault(peerId)?.Send(rpc); } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index e4b5beb6..b3cb491f 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -253,6 +253,8 @@ public Action? SendRpc private readonly ConcurrentBag reconnections = []; private readonly PeerStore _peerStore; + private readonly IReadOnlyDictionary directPeers; + private DateTime nextDirectConnectionAttempt; private ulong seqNo = 1; private record Reconnection(Multiaddress[] Addresses, int Attempts); @@ -275,6 +277,12 @@ public PubsubRouter(PeerStore store, PubsubSettings? settings = null, ILoggerFac throw new InvalidOperationException("StrictNoSign requires a custom GetMessageId function."); } + if (_settings.DirectConnectPeriod <= 0) + { + throw new ArgumentOutOfRangeException(nameof(settings), "DirectConnectPeriod must be positive."); + } + + directPeers = CreateDirectPeers(_settings.DirectPeers); _messageCache = new(_settings.MessageCacheTtl); _limboMessageCache = new(_settings.MessageCacheTtl); _idontwantMessages = new(_settings.MessageCacheTtl); @@ -301,6 +309,12 @@ public Task StartAsync(ILocalPeer localPeer, CancellationToken token = default) _ = Connect(addrs, token, true); }; + foreach (Multiaddress[] directPeerAddresses in directPeers.Values) + { + _peerStore.Discover(directPeerAddresses); + } + nextDirectConnectionAttempt = DateTime.UtcNow.AddMilliseconds(_settings.DirectConnectPeriod); + _ = Task.Run(LoopHeartbeat, token); _ = Task.Run(LoopReconnect, token); @@ -397,6 +411,46 @@ private void Reconnect(CancellationToken token) } }, token); } + + ReconnectDirectPeers(token); + } + + private static IReadOnlyDictionary CreateDirectPeers(IEnumerable? configuredPeers) + { + return (configuredPeers ?? []) + .Select(address => (PeerId: address.GetPeerId() ?? throw new ArgumentException("A direct peer address must include a peer ID.", nameof(configuredPeers)), Address: address)) + .GroupBy(entry => entry.PeerId) + .ToDictionary(group => group.Key, group => group.Select(entry => entry.Address).ToArray()); + } + + private void ReconnectDirectPeers(CancellationToken token) + { + if (directPeers.Count == 0 || DateTime.UtcNow < nextDirectConnectionAttempt) + { + return; + } + + nextDirectConnectionAttempt = DateTime.UtcNow.AddMilliseconds(_settings.DirectConnectPeriod); + foreach ((PeerId peerId, Multiaddress[] addresses) in directPeers) + { + if (!peerState.ContainsKey(peerId)) + { + _ = Connect(addresses, token, reconnect: true); + } + } + } + + private bool IsDirectPeer(PeerId peerId) => directPeers.ContainsKey(peerId); + + private bool IsDirectPeerSubscribedTo(PeerId peerId, string topic) + { + return (fPeers.TryGetValue(topic, out HashSet? floodsubPeers) && floodsubPeers.Contains(peerId)) || + (gPeers.TryGetValue(topic, out HashSet? gossipsubPeers) && gossipsubPeers.Contains(peerId)); + } + + private IEnumerable GetDirectPeersForTopic(string topic) + { + return directPeers.Keys.Where(peerId => IsDirectPeerSubscribedTo(peerId, topic)); } public Task Heartbeat() @@ -437,6 +491,7 @@ public Task Heartbeat() // Need to graft more peers - exclude peers with negative scores PeerId[] peersToGraft = gPeers[topic] .Where(p => !meshPeers.Contains(p) + && !IsDirectPeer(p) && GetPeerScore(p) >= 0 // Only graft non-negative scoring peers && (peerState.GetValueOrDefault(p)?.Backoff.TryGetValue(topic, out DateTime backoff) != true || backoff < DateTime.Now)) .Take(_settings.Degree - meshPeers.Count).ToArray(); @@ -522,7 +577,7 @@ public Task Heartbeat() int peerCountToAdd = _settings.Degree - fanout[fanoutTopic].Count; if (peerCountToAdd > 0) { - foreach (PeerId? peerId in gPeers[fanoutTopic].Where(p => !fanout[fanoutTopic].Contains(p)).Take(peerCountToAdd)) + foreach (PeerId? peerId in gPeers[fanoutTopic].Where(p => !fanout[fanoutTopic].Contains(p) && !IsDirectPeer(p)).Take(peerCountToAdd)) { fanout[fanoutTopic].Add(peerId); } @@ -557,6 +612,7 @@ onPartialGossip is not null && PeerId[] eligiblePeers = topicGossipsubPeers .Where(p => !topicMesh.Contains(p) && !fanoutPeers.Contains(p) + && !IsDirectPeer(p) && GetPeerScore(p) >= _settings.GossipThreshold) .ToArray(); From ea9423323589e0b01ee90fed0ecdc2158b2a371c Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 12:41:12 +0300 Subject: [PATCH 33/45] Back off direct peer grafts --- .../Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs | 9 +++++++-- src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs | 2 +- src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs index f637141c..596edc0e 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs @@ -63,7 +63,7 @@ public void DirectPeers_AreNeverAddedToTheMesh() Multiaddress secondMeshAddress = TestPeers.Multiaddr(3); PubsubRouter router = new( new PeerStore(), - new PubsubSettings { DirectPeers = [directAddress] }); + new PubsubSettings { DirectPeers = [directAddress], PruneBackoff = 2_000 }); IRoutingStateContainer state = router; _ = router.GetTopic(topic); TaskCompletionSource connection = new(); @@ -105,7 +105,12 @@ public void DirectPeerGrafts_AreRejectedWithPrune() graft.Control.Graft.Add(new ControlGraft { TopicID = topic }); router.OnRpc(directPeerId, graft); - Assert.That(sentRpcs.Single().Control.Prune.Single().TopicID, Is.EqualTo(topic)); + ControlPrune prune = sentRpcs.Single().Control.Prune.Single(); + Assert.Multiple(() => + { + Assert.That(prune.TopicID, Is.EqualTo(topic)); + Assert.That(prune.Backoff, Is.EqualTo(2)); + }); connection.SetResult(); } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index 304028cd..b0407ca4 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -333,7 +333,7 @@ private void HandleGraft(PeerId peerId, IEnumerable grafts, Concur logger?.LogWarning("Rejecting GRAFT from direct peer {peerId} for topic {topic}", peerId, graft.TopicID); peerMessages.GetOrAdd(peerId, _ => new Rpc()) .Ensure(r => r.Control.Prune) - .Add(new ControlPrune { TopicID = graft.TopicID }); + .Add(new ControlPrune { TopicID = graft.TopicID, Backoff = (ulong)Math.Max(1, _settings.PruneBackoff / 1_000) }); continue; } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index b3cb491f..8b2898e9 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -279,7 +279,7 @@ public PubsubRouter(PeerStore store, PubsubSettings? settings = null, ILoggerFac if (_settings.DirectConnectPeriod <= 0) { - throw new ArgumentOutOfRangeException(nameof(settings), "DirectConnectPeriod must be positive."); + throw new ArgumentOutOfRangeException(nameof(PubsubSettings.DirectConnectPeriod), "DirectConnectPeriod must be positive."); } directPeers = CreateDirectPeers(_settings.DirectPeers); From d4f4e5fde3d27dfa795ac9d047a50836f288fea7 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 12:41:42 +0300 Subject: [PATCH 34/45] Test direct peer graft backoff --- src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs index 596edc0e..c14fc619 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs @@ -63,7 +63,7 @@ public void DirectPeers_AreNeverAddedToTheMesh() Multiaddress secondMeshAddress = TestPeers.Multiaddr(3); PubsubRouter router = new( new PeerStore(), - new PubsubSettings { DirectPeers = [directAddress], PruneBackoff = 2_000 }); + new PubsubSettings { DirectPeers = [directAddress] }); IRoutingStateContainer state = router; _ = router.GetTopic(topic); TaskCompletionSource connection = new(); @@ -94,7 +94,7 @@ public void DirectPeerGrafts_AreRejectedWithPrune() PeerId directPeerId = directAddress.GetPeerId()!; PubsubRouter router = new( new PeerStore(), - new PubsubSettings { DirectPeers = [directAddress] }); + new PubsubSettings { DirectPeers = [directAddress], PruneBackoff = 2_000 }); _ = router.GetTopic(topic); List sentRpcs = []; TaskCompletionSource connection = new(); From 4ef6232bc2d81fa859248af195181475f2be02e9 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 12:53:15 +0300 Subject: [PATCH 35/45] Allow publishing without a local subscription --- .../PubsubProtocolTests.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubProtocolTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubProtocolTests.cs index e534377d..652f7d95 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubProtocolTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubProtocolTests.cs @@ -39,6 +39,18 @@ public void Publish_WithNullMessage_ThrowsArgumentNullException() Assert.Throws(() => router.Publish("test-topic", null!)); } + [Test] + public async Task Publish_WithoutSubscription_DoesNotThrow() + { + PubsubRouter router = new(new PeerStore()); + using CancellationTokenSource cancellation = new(); + await router.StartAsync(new LocalPeerStub(), cancellation.Token); + + Assert.DoesNotThrow(() => router.Publish("test-topic", [1, 2, 3])); + + cancellation.Cancel(); + } + [Test] public void Topic_OnMessage_IncludesReceivedFromPeerId() { From afad1dbd43610b401ad1564e3761a3817e2ff1ea Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Mon, 24 Aug 2026 14:04:52 +0300 Subject: [PATCH 36/45] Name invalid direct peer configuration --- src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs | 3 ++- src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs index c14fc619..8d999010 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs @@ -155,6 +155,7 @@ public void DirectPeers_RequirePeerIdsInTheirAddresses() DirectPeers = new[] { Multiaddress.Decode("/ip4/127.0.0.1/tcp/4001") }, }; - Assert.That(() => new PubsubRouter(new PeerStore(), settings), Throws.TypeOf()); + ArgumentException exception = Assert.Throws(() => new PubsubRouter(new PeerStore(), settings))!; + Assert.That(exception.ParamName, Is.EqualTo(nameof(PubsubSettings.DirectPeers))); } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index 8b2898e9..a057c357 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -418,7 +418,7 @@ private void Reconnect(CancellationToken token) private static IReadOnlyDictionary CreateDirectPeers(IEnumerable? configuredPeers) { return (configuredPeers ?? []) - .Select(address => (PeerId: address.GetPeerId() ?? throw new ArgumentException("A direct peer address must include a peer ID.", nameof(configuredPeers)), Address: address)) + .Select(address => (PeerId: address.GetPeerId() ?? throw new ArgumentException("A direct peer address must include a peer ID.", nameof(PubsubSettings.DirectPeers)), Address: address)) .GroupBy(entry => entry.PeerId) .ToDictionary(group => group.Key, group => group.Select(entry => entry.Address).ToArray()); } From 19c858152381b4ca148035c35bff34c2d43afb0c Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 17:08:11 +0300 Subject: [PATCH 37/45] Address independent review feedback --- .../TopicLifecycleTests.cs | 226 +++++++++++++++++- .../PubsubRouter.Topics.cs | 68 ++++-- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 108 ++++----- src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs | 23 +- 4 files changed, 330 insertions(+), 95 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs index e388f520..40107bd4 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TopicLifecycleTests.cs @@ -48,7 +48,7 @@ public async Task Topic_ResubscribeRetainsRemoteMembership() { const string topicName = "topic-lifecycle"; PeerStore peerStore = new(); - PubsubRouter router = new(peerStore); + PubsubRouter router = new(peerStore, new PubsubSettings { UnsubscribeBackoff = 0 }); IRoutingStateContainer state = router; ITopic topic = router.GetTopic(topicName); Multiaddress peerAddress = TestPeers.Multiaddr(3); @@ -59,6 +59,8 @@ public async Task Topic_ResubscribeRetainsRemoteMembership() router.OutboundConnection(peerAddress, PubsubRouter.GossipsubProtocolVersionV11, connectionClosed.Task, sent.Add); router.OnRpc(peerId, new Rpc().WithTopics([topicName], [])); router.OnRpc(peerId, CreateMessage(topicName, TestPeers.Identity(2), 1)); + await state.Heartbeat(); + Assert.That(state.Mesh[topicName], Has.Member(peerId), "Exercise removal of an established mesh."); sent.Clear(); topic.Unsubscribe(); @@ -202,14 +204,23 @@ public void UnsubscribedTopic_DoesNotRequestAdvertisedMessages() } [Test] - public void Topic_SubscribeMovesFanoutPeersIntoTheMesh() + public async Task Topic_SubscribeMovesFanoutPeersIntoTheMesh() { const string topicName = "topic-lifecycle"; - PubsubRouter router = new(new PeerStore()); + using PubsubRouter router = new(new PeerStore(), new PubsubSettings { FloodPublish = false }); + using CancellationTokenSource stopped = new(); + stopped.Cancel(); + await router.StartAsync(new LocalPeerStub(), stopped.Token); IRoutingStateContainer state = router; ITopic topic = router.GetTopic(topicName, subscribe: false); - PeerId fanoutPeer = TestPeers.PeerId(3); - state.Fanout.GetOrAdd(topicName, []).Add(fanoutPeer); + Multiaddress address = TestPeers.Multiaddr(3); + PeerId fanoutPeer = address.GetPeerId()!; + List sent = []; + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV11, new TaskCompletionSource().Task, sent.Add); + router.OnRpc(fanoutPeer, new Rpc().WithTopics([topicName], [])); + router.Publish(topicName, [1]); + Assert.That(state.Fanout[topicName], Has.Member(fanoutPeer)); + sent.Clear(); topic.Subscribe(); @@ -217,6 +228,211 @@ public void Topic_SubscribeMovesFanoutPeersIntoTheMesh() { Assert.That(state.Mesh[topicName], Has.Member(fanoutPeer)); Assert.That(state.Fanout, Does.Not.ContainKey(topicName)); + Assert.That(state.FanoutLastPublished, Does.Not.ContainKey(topicName)); + Assert.That(sent, Has.Count.EqualTo(1)); + Assert.That(sent[0].Subscriptions.Single().Subscribe, Is.True); + Assert.That(sent[0].Control.Graft.Single().TopicID, Is.EqualTo(topicName)); + }); + } + + [TestCase(true)] + [TestCase(false)] + public async Task PublishToFreshTopic_HeartbeatSurvivesAndFindsLaterSubscribers(bool floodPublish) + { + const string topicName = "fresh-topic"; + using PubsubRouter router = new(new PeerStore(), new PubsubSettings { FloodPublish = floodPublish }); + using CancellationTokenSource stopped = new(); + stopped.Cancel(); + await router.StartAsync(new LocalPeerStub(), stopped.Token); + IRoutingStateContainer state = router; + + router.Publish(topicName, [1]); + Assert.That(state.GossipsubPeers, Does.Not.ContainKey(topicName)); + Assert.That(state.Fanout, Contains.Key(topicName)); + await state.Heartbeat(); + await state.Heartbeat(); + + Multiaddress address = TestPeers.Multiaddr(3); + List sent = []; + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV11, new TaskCompletionSource().Task, sent.Add); + router.OnRpc(address.GetPeerId()!, new Rpc().WithTopics([topicName], [])); + await state.Heartbeat(); + router.Publish(topicName, [2]); + + Assert.That(state.Fanout[topicName], Has.Member(address.GetPeerId()!)); + Assert.That(sent.SelectMany(rpc => rpc.Publish).Count(), Is.EqualTo(1)); + } + + [Test] + public void Topic_MessageHandlerCanWaitForRouterWorkOnAnotherThread() + { + const string topicName = "callback-topic"; + using PubsubRouter router = new(new PeerStore()); + ITopic topic = router.GetTopic(topicName); + bool? heldRouterLock = null; + bool heartbeatCompleted = false; + Task? heartbeat = null; + topic.OnMessage += (_, _) => + { + heldRouterLock = Monitor.IsEntered(router); + heartbeat = Task.Run(((IRoutingStateContainer)router).Heartbeat); + heartbeatCompleted = heartbeat.Wait(TimeSpan.FromSeconds(5)); + }; + + router.OnRpc(TestPeers.PeerId(3), CreateMessage(topicName, TestPeers.Identity(2), 1)); + heartbeat?.GetAwaiter().GetResult(); + + Assert.Multiple(() => + { + Assert.That(heldRouterLock, Is.False); + Assert.That(heartbeatCompleted, Is.True, "A message handler must not block the heartbeat's routing lock."); + }); + } + + [TestCase(false)] + [TestCase(true)] + public async Task Unsubscribe_AdvertisesAndEnforcesBackoff(bool publishBeforeResubscribe) + { + const string topicName = "backoff-topic"; + PubsubSettings settings = new() { UnsubscribeBackoff = 60_500, FloodPublish = false }; + using PubsubRouter router = new(new PeerStore(), settings); + using CancellationTokenSource stopped = new(); + stopped.Cancel(); + await router.StartAsync(new LocalPeerStub(), stopped.Token); + IRoutingStateContainer state = router; + ITopic topic = router.GetTopic(topicName); + Multiaddress address = TestPeers.Multiaddr(3); + PeerId peerId = address.GetPeerId()!; + List sent = []; + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV11, new TaskCompletionSource().Task, sent.Add); + router.OnRpc(peerId, new Rpc().WithTopics([topicName], [])); + await state.Heartbeat(); + Assert.That(state.Mesh[topicName], Has.Member(peerId)); + sent.Clear(); + + topic.Unsubscribe(); + Assert.That(sent.Single().Control.Prune.Single().Backoff, Is.EqualTo(61)); + if (publishBeforeResubscribe) + { + router.Publish(topicName, [1]); + Assert.That(state.Fanout[topicName], Has.Member(peerId)); + } + sent.Clear(); + topic.Subscribe(); + await state.Heartbeat(); + + Assert.Multiple(() => + { + Assert.That(state.GossipsubPeers[topicName], Has.Member(peerId)); + Assert.That(state.Mesh[topicName], Is.Empty); + Assert.That(sent.Any(rpc => rpc.Control?.Graft.Count > 0), Is.False); + }); + } + + [Test] + public async Task UnsubscribeAll_BatchesTopicsAndContinuesAfterSendFailure() + { + using PubsubRouter router = new(new PeerStore()); + IRoutingStateContainer state = router; + ITopic first = router.GetTopic("first"); + ITopic second = router.GetTopic("second"); + List sent = []; + int failedSends = 0; + bool fail = false; + Multiaddress brokenAddress = TestPeers.Multiaddr(3); + Multiaddress healthyAddress = TestPeers.Multiaddr(4); + router.OutboundConnection(brokenAddress, PubsubRouter.GossipsubProtocolVersionV11, new TaskCompletionSource().Task, _ => + { + if (fail) + { + failedSends++; + throw new IOException("Connection closed"); + } + }); + router.OutboundConnection(healthyAddress, PubsubRouter.GossipsubProtocolVersionV11, new TaskCompletionSource().Task, sent.Add); + foreach (Multiaddress address in new[] { brokenAddress, healthyAddress }) + { + router.OnRpc(address.GetPeerId()!, new Rpc().WithTopics(["first", "second"], [])); + } + await state.Heartbeat(); + sent.Clear(); + fail = true; + + Assert.DoesNotThrow(router.UnsubscribeAll); + + Assert.Multiple(() => + { + Assert.That(first.IsSubscribed || second.IsSubscribed, Is.False); + Assert.That(state.Mesh, Is.Empty); + Assert.That(failedSends, Is.EqualTo(1)); + Assert.That(sent, Has.Count.EqualTo(1)); + Assert.That(sent[0].Subscriptions.Select(sub => sub.Topicid), Is.EquivalentTo(new[] { "first", "second" })); + Assert.That(sent[0].Subscriptions.All(sub => !sub.Subscribe), Is.True); + Assert.That(sent[0].Control.Prune.Select(prune => prune.TopicID), Is.EquivalentTo(new[] { "first", "second" })); + }); + } + + [Test] + public async Task HelloAndHeartbeat_AreSentUnderTheLifecycleLock() + { + const string topicName = "ordered-topic"; + using PubsubRouter router = new(new PeerStore()); + IRoutingStateContainer state = router; + ITopic topic = router.GetTopic(topicName); + Multiaddress address = TestPeers.Multiaddr(3); + List<(Rpc Rpc, bool Ordered)> sent = []; + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV11, new TaskCompletionSource().Task, + rpc => sent.Add((rpc, Monitor.IsEntered(router)))); + Assert.That(sent.Single().Rpc.Subscriptions.Single().Subscribe, Is.True); + router.OnRpc(address.GetPeerId()!, new Rpc().WithTopics([topicName], [])); + await state.Heartbeat(); + Assert.That(sent.Any(entry => entry.Rpc.Control?.Graft.Count > 0), Is.True); + topic.Unsubscribe(); + + Assert.That(sent.All(entry => entry.Ordered), Is.True); + Assert.That(sent.Last().Rpc.Subscriptions.Single().Subscribe, Is.False); + } + + [TestCase(false)] + [TestCase(true)] + public async Task Disconnect_SerializesMembershipCleanupWithLifecycle(bool inbound) + { + using PubsubRouter router = new(new PeerStore(), new PubsubSettings { FloodPublish = false }); + using CancellationTokenSource stopped = new(); + stopped.Cancel(); + await router.StartAsync(new LocalPeerStub(), stopped.Token); + IRoutingStateContainer state = router; + router.GetTopic("mesh-topic"); + Multiaddress address = TestPeers.Multiaddr(3); + PeerId peerId = address.GetPeerId()!; + TaskCompletionSource connectionClosed = new(TaskCreationOptions.RunContinuationsAsynchronously); + CancellationToken disconnected = inbound + ? router.InboundConnection(address, PubsubRouter.GossipsubProtocolVersionV11, connectionClosed.Task, () => { }) + : router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV11, connectionClosed.Task, _ => { }); + TaskCompletionSource cleanupFinished = new(TaskCreationOptions.RunContinuationsAsynchronously); + using CancellationTokenRegistration registration = disconnected.Register(() => cleanupFinished.TrySetResult()); + router.OnRpc(peerId, new Rpc().WithTopics(["mesh-topic", "fanout-topic"], [])); + await state.Heartbeat(); + router.Publish("fanout-topic", [1]); + Assert.That(state.Mesh["mesh-topic"], Has.Member(peerId)); + Assert.That(state.Fanout["fanout-topic"], Has.Member(peerId)); + + lock (router) + { + connectionClosed.SetResult(); + Assert.That(cleanupFinished.Task.Wait(TimeSpan.FromMilliseconds(100)), Is.False, + "Disconnect cleanup must wait while a lifecycle transition holds the router lock."); + Assert.That(state.Mesh["mesh-topic"], Has.Member(peerId)); + Assert.That(state.Fanout["fanout-topic"], Has.Member(peerId)); + } + await cleanupFinished.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Multiple(() => + { + Assert.That(state.ConnectedPeers, Does.Not.Contain(peerId)); + Assert.That(state.Mesh["mesh-topic"], Is.Empty); + Assert.That(state.Fanout["fanout-topic"], Is.Empty); + Assert.That(state.GossipsubPeers.Values.All(peers => !peers.Contains(peerId)), Is.True); }); } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index a51ac635..c99b09f0 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: MIT -using Microsoft.Extensions.Logging; using Nethermind.Libp2p.Core; using Nethermind.Libp2p.Protocols.Pubsub.Dto; using System.Collections.Concurrent; @@ -40,23 +39,33 @@ public void Subscribe(string topicId) gPeers.TryAdd(topicId, []); HashSet meshPeers = mesh.GetOrAdd(topicId, []); + HashSet promotedPeers = []; if (fanout.TryRemove(topicId, out HashSet? fanoutPeers)) { - foreach (PeerId peerId in fanoutPeers.ToList()) + foreach (PeerId peerId in fanoutPeers) { - if (meshPeers.Add(peerId)) + if (gPeers[topicId].Contains(peerId) && + peerState.TryGetValue(peerId, out PubsubPeer? peer) && + GetPeerScore(peerId) >= 0 && + (!peer.Backoff.TryGetValue(topicId, out DateTime backoff) || backoff <= DateTime.Now) && + meshPeers.Add(peerId)) { RecordPeerJoinMesh(peerId, topicId); + promotedPeers.Add(peerId); } } fanoutLastPublished.TryRemove(topicId, out _); } - Rpc topicUpdate = new Rpc().WithTopics([topicId], []); foreach (PubsubPeer peer in peerState.Values) { + Rpc topicUpdate = new Rpc().WithTopics([topicId], []); + if (promotedPeers.Contains(peer.PeerId)) + { + topicUpdate.Ensure(r => r.Control.Graft).Add(new ControlGraft { TopicID = topicId }); + } peer.Send(topicUpdate); } } @@ -64,47 +73,72 @@ public void Subscribe(string topicId) public void Unsubscribe(string topicId) { - HashSet? removedMesh; lock (this) + { + UnsubscribeTopics([topicId]); + } + } + + // Called under the router lock so state changes and notifications stay ordered. + private void UnsubscribeTopics(IEnumerable topicIds) + { + Dictionary peerMessages = []; + foreach (string topicId in topicIds) { if (!topicState.TryGetValue(topicId, out Topic? topic) || !topic.IsSubscribed) { - return; + continue; } topic.IsSubscribed = false; - if (mesh.TryRemove(topicId, out removedMesh)) + if (mesh.TryRemove(topicId, out HashSet? removedMesh)) { foreach (PeerId peerId in removedMesh) { RecordPeerLeaveMesh(peerId, topicId); + if (peerState.TryGetValue(peerId, out PubsubPeer? peer)) + { + DateTime backoffUntil = DateTime.Now.AddMilliseconds(_settings.UnsubscribeBackoff); + if (!peer.Backoff.TryGetValue(topicId, out DateTime existingBackoff) || existingBackoff < backoffUntil) + { + peer.Backoff[topicId] = backoffUntil; + } + } } } fanout.TryRemove(topicId, out _); fanoutLastPublished.TryRemove(topicId, out _); - foreach ((PeerId peerId, PubsubPeer peer) in peerState) + foreach (PeerId peerId in peerState.Keys) { - Rpc msg = new Rpc().WithTopics([], [topicId]); + if (!peerMessages.TryGetValue(peerId, out Rpc? msg)) + { + peerMessages[peerId] = msg = new Rpc(); + } + msg.WithTopics([], [topicId]); if (removedMesh?.Contains(peerId) is true) { - msg.Ensure(r => r.Control.Prune).Add(new ControlPrune { TopicID = topicId }); + msg.Ensure(r => r.Control.Prune).Add(new ControlPrune + { + TopicID = topicId, + Backoff = (ulong)Math.Ceiling(_settings.UnsubscribeBackoff / 1000.0), + }); } - - peer.Send(msg); } } + + foreach ((PeerId peerId, Rpc msg) in peerMessages) + { + peerState.GetValueOrDefault(peerId)?.Send(msg); + } } public void UnsubscribeAll() { - foreach (string topicId in topicState - .Where(pair => pair.Value.IsSubscribed) - .Select(pair => pair.Key) - .ToArray()) + lock (this) { - Unsubscribe(topicId); + UnsubscribeTopics(topicState.Keys); } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index ad234800..4dbbe5f7 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -64,13 +64,25 @@ public enum PubsubProtocol public void Send(Rpc rpc) { SendRpcQueue.Enqueue(rpc); + FlushSendQueue(); + } + + private void FlushSendQueue() + { if (SendRpc is not null) { lock (SendRpcQueue) { while (SendRpcQueue.TryDequeue(out Rpc? rpcToSend)) { - SendRpc.Invoke(rpcToSend); + try + { + SendRpc.Invoke(rpcToSend); + } + catch (Exception ex) + { + _logger?.LogWarning(ex, "Failed to send pubsub RPC to {peerId}", PeerId); + } } } } @@ -86,14 +98,7 @@ public Action? SendRpc { _logger?.LogDebug($"Set SENDRPC for {PeerId}: {value}"); _sendRpc = value; - if (_sendRpc is not null) - lock (SendRpcQueue) - { - while (SendRpcQueue.TryDequeue(out Rpc? rpcToSend)) - { - _sendRpc.Invoke(rpcToSend); - } - } + FlushSendQueue(); } } public CancellationTokenSource TokenSource { get; init; } @@ -324,7 +329,7 @@ public Task Heartbeat() if (meshPeers.Count < _settings.LowestDegree) { // Need to graft more peers - exclude peers with negative scores - PeerId[] peersToGraft = gPeers[topic] + PeerId[] peersToGraft = (gPeers.GetValueOrDefault(topic) ?? []) .Where(p => !meshPeers.Contains(p) && GetPeerScore(p) >= 0 // Only graft non-negative scoring peers && (peerState.GetValueOrDefault(p)?.Backoff.TryGetValue(topic, out DateTime backoff) != true || backoff < DateTime.Now)) @@ -411,7 +416,7 @@ public Task Heartbeat() int peerCountToAdd = _settings.Degree - fanout[fanoutTopic].Count; if (peerCountToAdd > 0) { - foreach (PeerId? peerId in gPeers[fanoutTopic].Where(p => !fanout[fanoutTopic].Contains(p)).Take(peerCountToAdd)) + foreach (PeerId? peerId in (gPeers.GetValueOrDefault(fanoutTopic) ?? []).Where(p => !fanout[fanoutTopic].Contains(p)).Take(peerCountToAdd)) { fanout[fanoutTopic].Add(peerId); } @@ -449,14 +454,31 @@ public Task Heartbeat() } } } + foreach (KeyValuePair peerMessage in peerMessages) + { + peerState.GetValueOrDefault(peerMessage.Key)?.Send(peerMessage.Value); + } } - foreach (KeyValuePair peerMessage in peerMessages) + return Task.CompletedTask; + } + + private void RemovePeer(PeerId peerId, Multiaddress addr) + { + PubsubPeer? removedPeer; + lock (this) { - peerState.GetValueOrDefault(peerMessage.Key)?.Send(peerMessage.Value); + peerState.TryRemove(peerId, out removedPeer); + foreach (var peersByTopic in new[] { fPeers, gPeers, fanout, mesh }) + { + foreach (HashSet topicPeers in peersByTopic.Values) + { + topicPeers.Remove(peerId); + } + } + reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); } - - return Task.CompletedTask; + removedPeer?.TokenSource.Cancel(); } internal CancellationToken OutboundConnection(Multiaddress addr, string protocolId, Task dialTask, Action sendRpc) @@ -490,42 +512,22 @@ internal CancellationToken OutboundConnection(Multiaddress addr, string protocol dialTask.ContinueWith(t => { - peerState.GetValueOrDefault(peerId)?.TokenSource.Cancel(); - peerState.TryRemove(peerId, out _); - foreach (KeyValuePair> topicPeers in fPeers) - { - topicPeers.Value.Remove(peerId); - } - foreach (KeyValuePair> topicPeers in gPeers) - { - topicPeers.Value.Remove(peerId); - } - foreach (KeyValuePair> topicPeers in fanout) - { - topicPeers.Value.Remove(peerId); - } - foreach (KeyValuePair> topicPeers in mesh) - { - topicPeers.Value.Remove(peerId); - } - reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); + RemovePeer(peerId, addr); }); - string[] topics; lock (this) { - topics = topicState + string[] topics = topicState .Where(pair => pair.Value.IsSubscribed) .Select(pair => pair.Key) .ToArray(); - } - - if (topics.Any()) - { - logger?.LogDebug("Topics sent to {peerId}: {topics}", peerId, string.Join(",", topics)); + if (topics.Any()) + { + logger?.LogDebug("Topics sent to {peerId}: {topics}", peerId, string.Join(",", topics)); - Rpc helloMessage = new Rpc().WithTopics(topics, []); - peer.Send(helloMessage); + Rpc helloMessage = new Rpc().WithTopics(topics, []); + peer.Send(helloMessage); + } } logger?.LogDebug("Outbound {peerId}", peerId); @@ -552,25 +554,7 @@ internal CancellationToken InboundConnection(Multiaddress addr, string protocolI logger?.LogDebug("Inbound, let's dial {peerId} via remotely initiated connection", peerId); listTask.ContinueWith(t => { - peerState.GetValueOrDefault(peerId)?.TokenSource.Cancel(); - peerState.TryRemove(peerId, out _); - foreach (KeyValuePair> topicPeers in fPeers) - { - topicPeers.Value.Remove(peerId); - } - foreach (KeyValuePair> topicPeers in gPeers) - { - topicPeers.Value.Remove(peerId); - } - foreach (KeyValuePair> topicPeers in fanout) - { - topicPeers.Value.Remove(peerId); - } - foreach (KeyValuePair> topicPeers in mesh) - { - topicPeers.Value.Remove(peerId); - } - reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); + RemovePeer(peerId, addr); }); subDial(); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs index 811b830d..8ff91f49 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs @@ -19,21 +19,22 @@ public Topic(PubsubRouter router, string topicName) private void OnRouterMessage(string topicName, PeerId peerId, byte[] message) { - lock (router) + if (!IsSubscribed || this.topicName != topicName) { - if (!IsSubscribed || this.topicName != topicName) - { - return; - } - - Action? onMessage = OnMessage; - onMessage?.Invoke(peerId, message); + return; } + + // User callbacks must not hold the routing lock: they may wait for work + // on another thread that needs the router. An in-flight delivery may finish + // concurrently with unsubscribe. + Action? onMessage = OnMessage; + onMessage?.Invoke(peerId, message); } public DateTime LastPublished { get; set; } - public bool IsSubscribed { get; internal set; } + private volatile bool isSubscribed; + public bool IsSubscribed { get => isSubscribed; internal set => isSubscribed = value; } public event Action? OnMessage; @@ -44,11 +45,11 @@ public void Publish(byte[] value) public void Unsubscribe() { - if (IsSubscribed) router.Unsubscribe(topicName); + router.Unsubscribe(topicName); } public void Subscribe() { - if (!IsSubscribed) router.Subscribe(topicName); + router.Subscribe(topicName); } } From 265a0b3e915b9ae11bb130caa14e5f34cd46b85b Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 18:13:09 +0300 Subject: [PATCH 38/45] Address independent review feedback --- .../PartialMessagesTests.cs | 189 +++++++++++++++++- .../PubsubRouter.Rpc.cs | 14 ++ .../PubsubRouter.Topics.cs | 30 +-- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 15 +- src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs | 11 +- 5 files changed, 231 insertions(+), 28 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs index 4ed39383..ecbcdc8e 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs @@ -127,6 +127,9 @@ public async Task PartialMessages_AdvertiseCapabilitiesAndGatePayloadsPerPeer() requestedPeerRpcs.Clear(); metadataOnlyPeerRpcs.Clear(); + await router.Heartbeat(); + requestedPeerRpcs.Clear(); + metadataOnlyPeerRpcs.Clear(); topic.PublishPartial([1, 2], partialMessage: [3], partsMetadata: [4]); PartialMessagesExtension requestedPartial = requestedPeerRpcs.Single().Partial; @@ -151,7 +154,7 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); IPartialMessagesTopic topic = router.GetPartialMessagesTopic( topicName, - new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }); + new PartialMessagesTopicOptions { RequestPartialMessages = true, SupportsSendingPartialMessages = true }); Multiaddress remoteAddress = TestPeers.Multiaddr(1); PeerId remotePeerId = remoteAddress.GetPeerId()!; TaskCompletionSource connection = new(); @@ -206,8 +209,9 @@ public void PartialMessages_AreForwardedAfterThePeerAdvertisesTheExtension() connection.SetResult(); } - [Test] - public void PartialMessages_HonorIndependentRemoteSubscriptionFlags() + [TestCase(false)] + [TestCase(true)] + public void PartialMessages_RequestingImpliesSendingSupport(bool omitSendingFlag) { const string topicName = "topic"; PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); @@ -219,7 +223,12 @@ public void PartialMessages_HonorIndependentRemoteSubscriptionFlags() TaskCompletionSource connection = new(); List sentRpcs = []; router.OutboundConnection(remoteAddress, PubsubRouter.GossipsubProtocolVersionV13, connection.Task, sentRpcs.Add); - router.OnRpc(remotePeerId, CreateSubscriptionRpc(topicName, requestsPartialMessages: true, supportsSendingPartialMessages: false)); + Rpc subscription = CreateSubscriptionRpc(topicName, requestsPartialMessages: true, supportsSendingPartialMessages: false); + if (omitSendingFlag) + { + subscription.Subscriptions.Single().ClearSupportsSendingPartial(); + } + router.OnRpc(remotePeerId, subscription); sentRpcs.Clear(); topic.SendPartial(remotePeerId, [1], partialMessage: [2], partsMetadata: [3]); @@ -228,7 +237,7 @@ public void PartialMessages_HonorIndependentRemoteSubscriptionFlags() Assert.Multiple(() => { Assert.That(sentPartial.PartialMessage.ToByteArray(), Is.EqualTo(new byte[] { 2 })); - Assert.That(sentPartial.HasPartsMetadata, Is.False); + Assert.That(sentPartial.PartsMetadata.ToByteArray(), Is.EqualTo(new byte[] { 3 })); }); connection.SetResult(); } @@ -257,8 +266,9 @@ public void PartialMessages_AreNeverAdvertisedOnOlderProtocols() connection.SetResult(); } - [Test] - public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave() + [TestCase(false)] + [TestCase(true)] + public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave(bool throwingHandler) { const string topicName = "topic"; PubsubRouter router = new(new PeerStore(), new PubsubSettings @@ -281,8 +291,14 @@ public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave Dictionary> sentRpcs = []; List? partialGossipRecipients = null; byte[]? partialGossipGroupId = null; + if (throwingHandler) + { + router.OnPartialGossip += (_, _, _) => throw new InvalidOperationException("Application failure"); + } + int notifications = 0; router.OnPartialGossip += (topicId, groupId, peers) => { + notifications++; Assert.That(topicId, Is.EqualTo(topicName)); partialGossipGroupId = groupId; partialGossipRecipients = peers.ToList(); @@ -322,6 +338,9 @@ public async Task PartialMessages_GossipUsesTheApplicationCallbackInsteadOfIhave Assert.That(partialGossipRecipients!.All(sentRpcs.ContainsKey), Is.True); Assert.That(sentRpcs.Values.SelectMany(rpcs => rpcs).SelectMany(rpc => rpc.Control?.Ihave ?? []), Is.Empty); + await router.Heartbeat(); + Assert.That(notifications, Is.EqualTo(2), "A failed handler must not interrupt this or subsequent heartbeats."); + cancellation.Cancel(); connection.SetResult(); } @@ -392,6 +411,162 @@ public async Task PartialMessages_RequestersDoNotReceiveFullPublishesOrRelays() connection.SetResult(); } + [Test] + public void PartialMessages_TopicDeliveryRequiresSubscriptionAndCapabilities() + { + using PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + IPartialMessagesTopic topic = router.GetPartialMessagesTopic("topic", + new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }); + TaskCompletionSource connection = new(); + PeerId peer = TestPeers.PeerId(1); + router.OutboundConnection(TestPeers.Multiaddr(1), PubsubRouter.GossipsubProtocolVersionV13, connection.Task, _ => { }); + router.OnRpc(peer, CreateSubscriptionRpc("topic", true, true)); + int deliveries = 0; + topic.OnPartialMessage += (_, _) => deliveries++; + Rpc metadata = new() + { + Partial = new PartialMessagesExtension + { + TopicID = ByteString.CopyFromUtf8("topic"), + GroupID = ByteString.CopyFrom([1]), + PartsMetadata = ByteString.CopyFrom([2]), + }, + }; + + router.OnRpc(peer, metadata); + Assert.That(deliveries, Is.EqualTo(1)); + topic.Unsubscribe(); + router.OnRpc(peer, metadata); + Assert.That(deliveries, Is.EqualTo(1)); + topic.Subscribe(); + router.OnRpc(peer, metadata); + Assert.That(deliveries, Is.EqualTo(2)); + router.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions()); + router.OnRpc(peer, metadata); + Assert.That(deliveries, Is.EqualTo(2)); + connection.SetResult(); + } + + [TestCase("metadata-only")] + [TestCase("ordinary")] + [TestCase("unknown")] + [TestCase("unsubscribed")] + public async Task PartialMessages_UnrequestedDataIsDroppedAndPenalized(string mode) + { + using PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); + if (mode == "ordinary") + { + router.GetTopic("topic"); + } + else if (mode != "unknown") + { + router.GetPartialMessagesTopic("topic", new PartialMessagesTopicOptions + { + RequestPartialMessages = mode == "unsubscribed", + SupportsSendingPartialMessages = true, + }, subscribe: mode != "unsubscribed"); + } + TaskCompletionSource connection = new(); + PeerId peer = TestPeers.PeerId(1); + router.OutboundConnection(TestPeers.Multiaddr(1), PubsubRouter.GossipsubProtocolVersionV13, connection.Task, _ => { }); + router.OnRpc(peer, CreateSubscriptionRpc("topic", true, true)); + IRoutingStateContainer state = router; + await router.Heartbeat(); + int deliveries = 0; + router.OnPartialMessage += (_, _, _) => deliveries++; + router.OnRpc(peer, new Rpc + { + Partial = new PartialMessagesExtension + { + TopicID = ByteString.CopyFromUtf8("topic"), + GroupID = ByteString.CopyFrom([1]), + PartialMessage = ByteString.CopyFrom([2]), + }, + }); + + Assert.That(deliveries, Is.Zero); + // A behavior penalty makes the peer ineligible for grafting. + router.GetTopic("topic"); + await router.Heartbeat(); + Assert.That(state.Mesh["topic"], Does.Not.Contain(peer)); + connection.SetResult(); + } + + [TestCase(true)] + [TestCase(false)] + public async Task PublishPartial_UsesFanoutOnlyWhenUnsubscribed(bool subscribed) + { + using PubsubRouter router = new(new PeerStore(), new PubsubSettings + { + EnablePartialMessages = true, + HeartbeatInterval = int.MaxValue, + }); + IPartialMessagesTopic topic = router.GetPartialMessagesTopic("topic", + new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }, subscribe: subscribed); + using CancellationTokenSource cancellation = new(); + await router.StartAsync(new LocalPeerStub(), cancellation.Token); + TaskCompletionSource connection = new(); + List sent = []; + router.OutboundConnection(TestPeers.Multiaddr(1), PubsubRouter.GossipsubProtocolVersionV13, connection.Task, sent.Add); + router.OnRpc(TestPeers.PeerId(1), CreateSubscriptionRpc("topic", true, false)); + sent.Clear(); + + topic.PublishPartial([1], partialMessage: [2]); + + IRoutingStateContainer state = router; + Assert.Multiple(() => + { + Assert.That(state.Fanout.ContainsKey("topic"), Is.EqualTo(!subscribed)); + Assert.That(state.FanoutLastPublished.ContainsKey("topic"), Is.EqualTo(!subscribed)); + Assert.That(sent.Count, Is.EqualTo(subscribed ? 0 : 1)); + }); + cancellation.Cancel(); + connection.SetResult(); + } + + [TestCase(true)] + [TestCase(false)] + public async Task PublishPartial_CanRunConcurrentlyWithHeartbeatAndSubscriptions(bool subscribed) + { + using PubsubRouter router = new(new PeerStore(), new PubsubSettings + { + EnablePartialMessages = true, + HeartbeatInterval = int.MaxValue, + }); + IPartialMessagesTopic topic = router.GetPartialMessagesTopic("topic", + new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }, subscribe: subscribed); + using CancellationTokenSource cancellation = new(); + await router.StartAsync(new LocalPeerStub(), cancellation.Token); + TaskCompletionSource connection = new(); + PeerId peer = TestPeers.PeerId(1); + router.OutboundConnection(TestPeers.Multiaddr(1), PubsubRouter.GossipsubProtocolVersionV13, connection.Task, _ => { }); + router.OnRpc(peer, CreateSubscriptionRpc("topic", true, true)); + using Barrier start = new(2); + Task publisher = Task.Run(() => + { + start.SignalAndWait(); + for (int i = 0; i < 500; i++) + { + topic.PublishPartial([1], partialMessage: [2]); + } + }); + Task heartbeat = Task.Run(async () => + { + start.SignalAndWait(); + for (int i = 0; i < 500; i++) + { + router.OnRpc(peer, new Rpc().WithTopics([], ["topic"])); + router.OnRpc(peer, new Rpc().WithTopics(["topic"], [])); + await router.Heartbeat(); + } + }); + + await Task.WhenAll(publisher, heartbeat).WaitAsync(TimeSpan.FromSeconds(15)); + Assert.That(((IRoutingStateContainer)router).Fanout.ContainsKey("topic"), Is.EqualTo(!subscribed)); + cancellation.Cancel(); + connection.SetResult(); + } + private static Rpc CreateSubscriptionRpc(string topicName, bool requestsPartialMessages, bool supportsSendingPartialMessages) { Rpc rpc = new() diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index fa58a023..fa97a523 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -146,6 +146,20 @@ private void HandlePartialMessage(PeerId peerId, PartialMessagesExtension partia return; } + topicState.TryGetValue(topicId, out Topic? topic); + if (partialMessage.HasPartialMessage && + (topic?.IsSubscribed is not true || !topic.RequestsPartialMessages)) + { + ApplyBehaviorPenalty(peerId, 1.0); + logger?.LogDebug("Ignoring unsolicited partial data from {peerId} for topic {topicId}", peerId, topicId); + return; + } + + if (topic?.SupportsSendingPartialMessages is not true) + { + return; + } + receivedPartialMessages.Add(( topicId, peerId, diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index d5bdd5ae..d105eb28 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -267,29 +267,29 @@ public void PublishPartial(string topicId, byte[] groupId, byte[]? partialMessag throw new InvalidOperationException("Router has not been started. Call StartAsync() first."); } + PeerId[] recipients; lock (this) { partialMessageGossip.Track(topicId, groupId); - } - PeerId[] recipients; - if (mesh.TryGetValue(topicId, out HashSet? meshPeers) && meshPeers.Count > 0) - { - recipients = meshPeers.Where(peerId => GetPeerScore(peerId) >= _settings.PublishThreshold).ToArray(); - } - else - { - fanoutLastPublished[topicId] = DateTime.Now; - HashSet fanoutPeers = fanout.GetOrAdd(topicId, _ => []); - if (fanoutPeers.Count == 0 && gPeers.TryGetValue(topicId, out HashSet? topicPeers)) + if (mesh.TryGetValue(topicId, out HashSet? meshPeers)) + { + recipients = meshPeers.Where(peerId => GetPeerScore(peerId) >= _settings.PublishThreshold).ToArray(); + } + else { - foreach (PeerId peerId in topicPeers.Where(peerId => GetPeerScore(peerId) >= 0).Take(_settings.Degree)) + fanoutLastPublished[topicId] = DateTime.Now; + HashSet fanoutPeers = fanout.GetOrAdd(topicId, _ => []); + if (fanoutPeers.Count == 0 && gPeers.TryGetValue(topicId, out HashSet? topicPeers)) { - fanoutPeers.Add(peerId); + foreach (PeerId peerId in topicPeers.Where(peerId => GetPeerScore(peerId) >= 0).Take(_settings.Degree)) + { + fanoutPeers.Add(peerId); + } } - } - recipients = fanoutPeers.Where(peerId => GetPeerScore(peerId) >= _settings.PublishThreshold).ToArray(); + recipients = fanoutPeers.Where(peerId => GetPeerScore(peerId) >= _settings.PublishThreshold).ToArray(); + } } foreach (PeerId peerId in recipients) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index e4b5beb6..7e3f8aa5 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -125,7 +125,8 @@ public bool NeedsExtensions public void UpdatePartialMessagesSubscription(string topicId, bool requestsPartialMessages, bool supportsSendingPartialMessages) { - _partialMessagesSubscriptions[topicId] = new(requestsPartialMessages, supportsSendingPartialMessages); + // The extension registry makes sending support implicit for requesters. + _partialMessagesSubscriptions[topicId] = new(requestsPartialMessages, requestsPartialMessages || supportsSendingPartialMessages); } public void RemovePartialMessagesSubscription(string topicId) @@ -605,7 +606,17 @@ onPartialGossip is not null && foreach ((string topic, byte[] groupId, PeerId[] peers) in partialGossipNotifications) { - onPartialGossip?.Invoke(topic, groupId, peers); + foreach (Action> handler in onPartialGossip!.GetInvocationList()) + { + try + { + handler(topic, groupId, peers); + } + catch (Exception ex) + { + logger?.LogError(ex, "Exception in partial gossip handler for topic {topic}", topic); + } + } } return Task.CompletedTask; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs index 62f1d817..4ac60d9d 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/Topic.cs @@ -111,11 +111,14 @@ public void SendPartial(PeerId peerId, byte[] groupId, byte[]? partialMessage = private void OnRouterPartialMessage(string topicName, PeerId peerId, PartialMessage message) { - if (topic.Name != topicName) + lock (topic.Router) { - return; - } + if (!IsSubscribed || !SupportsSendingPartialMessages || topic.Name != topicName) + { + return; + } - OnPartialMessage?.Invoke(peerId, message); + OnPartialMessage?.Invoke(peerId, message); + } } } From 64f118c236f0276874125cc9d26b7c6539f14b0d Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 18:24:55 +0300 Subject: [PATCH 39/45] Address independent review feedback --- .../TtlCacheTests.cs | 115 +++++++++++++----- .../Libp2p.Protocols.Pubsub/TtlCache.cs | 70 +++-------- 2 files changed, 101 insertions(+), 84 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs index 8328d26a..f65014bd 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/TtlCacheTests.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: MIT +using NSubstitute; + namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; [TestFixture] @@ -9,73 +11,128 @@ public class TtlCacheTests [Test] public void RemoveExpired_RemovesEntriesRegardlessOfKeyOrder() { - using TtlCache cache = new(500); + TestTimeProvider clock = new(); + using TtlCache cache = new(500, clock); MessageId expiredHigh = new([0xFF]); MessageId liveLow = new([0x01]); cache.Add(expiredHigh); - DateTimeOffset expiredAfter = DateTimeOffset.UtcNow.AddMilliseconds(500); - Assert.That(() => DateTimeOffset.UtcNow >= expiredAfter, Is.True.After(2_000, 25)); + clock.UtcNow = clock.UtcNow.AddMilliseconds(500); cache.Add(liveLow); - cache.RemoveExpired(DateTimeOffset.UtcNow); + cache.RemoveExpired(clock.UtcNow); Assert.Multiple(() => { Assert.That(cache.Count, Is.EqualTo(1)); - Assert.That(cache.EntryOrderCount, Is.EqualTo(1)); Assert.That(cache.Contains(expiredHigh), Is.False); Assert.That(cache.Contains(liveLow), Is.True); }); } - [Test] - public void ExpiredEntries_AreNotReturned() + [TestCase("Contains")] + [TestCase("TryGet")] + [TestCase("Get")] + [TestCase("ToList")] + public void ExpiredEntries_AreNotReturnedBeforeSweeping(string operation) { - using TtlCache cache = new(500); + TestTimeProvider clock = new(); + using TtlCache cache = new(500, clock); MessageId id = new([0x01]); cache.Add(id, "value"); + clock.UtcNow = clock.UtcNow.AddMilliseconds(500); - Assert.That(() => cache.Contains(id), Is.False.After(2_000, 25)); - - Assert.Multiple(() => + // No cache read or sweep may remove the expired entry before the operation under test. + Assert.That(cache.Count, Is.EqualTo(1)); + switch (operation) { - Assert.That(cache.Contains(id), Is.False); - Assert.That(cache.TryGet(id, out _), Is.False); - Assert.That(cache.ToList(), Is.Empty); - }); + case "Contains": + Assert.That(cache.Contains(id), Is.False); + break; + case "TryGet": + Assert.That(cache.TryGet(id, out string value), Is.False); + Assert.That(value, Is.Null); + break; + case "Get": + Assert.That(cache.Get(id), Is.Null); + break; + case "ToList": + Assert.That(cache.ToList(), Is.Empty); + break; + } + } + + [Test] + public void ToList_ReturnsOnlyLiveEntries() + { + TestTimeProvider clock = new(); + using TtlCache cache = new(500, clock); + cache.Add(new([0x01]), "expired"); + clock.UtcNow = clock.UtcNow.AddMilliseconds(500); + cache.Add(new([0x02]), "live"); + + Assert.That(cache.Count, Is.EqualTo(2)); + Assert.That(cache.ToList(), Is.EqualTo(new[] { "live" })); } [Test] public void Add_ReplacesAnExpiredEntry() { - using TtlCache cache = new(500); + TestTimeProvider clock = new(); + using TtlCache cache = new(500, clock); MessageId id = new([0x01]); cache.Add(id, "expired"); + clock.UtcNow = clock.UtcNow.AddMilliseconds(500); - Assert.That(() => cache.Contains(id), Is.False.After(2_000, 25)); + Assert.That(cache.Count, Is.EqualTo(1)); cache.Add(id, "replacement"); + Assert.That(cache.Count, Is.EqualTo(1)); Assert.That(cache.Get(id), Is.EqualTo("replacement")); } [Test] - public void Add_EvictsTheOldestLiveEntryAtCapacity() + public void Add_DoesNotReplaceOrRefreshALiveEntry() { - using TtlCache cache = new(ttl: 1_000, maxEntries: 2); + TestTimeProvider clock = new(); + using TtlCache cache = new(500, clock); + MessageId id = new([0x01]); + cache.Add(id, "original"); + clock.UtcNow = clock.UtcNow.AddMilliseconds(250); + cache.Add(id, "replacement"); + + Assert.That(cache.Get(id), Is.EqualTo("original")); + clock.UtcNow = clock.UtcNow.AddMilliseconds(250); + Assert.That(cache.Contains(id), Is.False); + } + + [Test] + public void RemoveExpired_RemovesLaterEntriesAfterClockMovesBackward() + { + TestTimeProvider clock = new(); + using TtlCache cache = new(500, clock); MessageId first = new([0x01]); MessageId second = new([0x02]); - MessageId third = new([0x03]); + cache.Add(first); + clock.UtcNow = clock.UtcNow.AddMilliseconds(-250); + cache.Add(second); + clock.UtcNow = clock.UtcNow.AddMilliseconds(500); - cache.Add(first, "first"); - cache.Add(second, "second"); - cache.Add(third, "third"); + cache.RemoveExpired(clock.UtcNow); - Assert.Multiple(() => - { - Assert.That(cache.Contains(first), Is.False); - Assert.That(cache.Get(second), Is.EqualTo("second")); - Assert.That(cache.Get(third), Is.EqualTo("third")); - }); + Assert.That(cache.Count, Is.EqualTo(1)); + Assert.That(cache.Contains(first), Is.True); + Assert.That(cache.Contains(second), Is.False); + } + + private sealed class TestTimeProvider : TimeProvider + { + public DateTimeOffset UtcNow { get; set; } = new(2026, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public override DateTimeOffset GetUtcNow() => UtcNow; + + // Keep the background sweeper dormant so each test controls expiry explicitly. + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + => Substitute.For(); } } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs index 00acdd00..8cbfc585 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/TtlCache.cs @@ -6,29 +6,27 @@ namespace Nethermind.Libp2p.Protocols.Pubsub; internal class TtlCache : IDisposable where TKey : notnull { private readonly int ttl; - private readonly int maxEntries; + private readonly TimeProvider timeProvider; private readonly object sync = new(); private readonly Dictionary items = []; - private readonly LinkedList insertionOrder = []; private readonly CancellationTokenSource sweeperCancellation = new(); private readonly Task sweeperTask; private int disposed; - private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill, LinkedListNode Node); + private readonly record struct CachedItem(TItem Item, DateTimeOffset ValidTill); - public TtlCache(int ttl, int maxEntries = int.MaxValue) + public TtlCache(int ttl, TimeProvider? timeProvider = null) { ArgumentOutOfRangeException.ThrowIfNegativeOrZero(ttl); - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(maxEntries); this.ttl = ttl; - this.maxEntries = maxEntries; + this.timeProvider = timeProvider ?? TimeProvider.System; sweeperTask = Task.Run(async () => { try { while (true) { - await Task.Delay(5_000, sweeperCancellation.Token); - RemoveExpired(DateTimeOffset.UtcNow); + await Task.Delay(TimeSpan.FromSeconds(5), this.timeProvider, sweeperCancellation.Token); + RemoveExpired(this.timeProvider.GetUtcNow()); } } catch (OperationCanceledException) when (sweeperCancellation.IsCancellationRequested) @@ -52,30 +50,19 @@ internal int Count } } - internal int EntryOrderCount - { - get - { - lock (sync) - { - return insertionOrder.Count; - } - } - } - public bool TryGet(TKey key, out TItem item) { lock (sync) { if (items.TryGetValue(key, out CachedItem cachedItem)) { - if (cachedItem.ValidTill > DateTimeOffset.UtcNow) + if (cachedItem.ValidTill > timeProvider.GetUtcNow()) { item = cachedItem.Item; return true; } - Remove(key, cachedItem); + items.Remove(key); } } @@ -95,7 +82,7 @@ public void Add(TKey key, TItem item) { lock (sync) { - DateTimeOffset now = DateTimeOffset.UtcNow; + DateTimeOffset now = timeProvider.GetUtcNow(); if (items.TryGetValue(key, out CachedItem cachedItem)) { if (cachedItem.ValidTill > now) @@ -103,21 +90,16 @@ public void Add(TKey key, TItem item) return; } - Remove(key, cachedItem); - } - - while (items.Count >= maxEntries) - { - EvictOldest(); + items.Remove(key); } - LinkedListNode node = insertionOrder.AddLast(key); - items.Add(key, new CachedItem(item, now.AddMilliseconds(ttl), node)); + items.Add(key, new CachedItem(item, now.AddMilliseconds(ttl))); } } private void RemoveExpiredLocked(DateTimeOffset now) { + // Wall-clock adjustments can make expiration order differ from insertion order. List? expired = null; foreach ((TKey key, CachedItem item) in items) { @@ -131,32 +113,11 @@ private void RemoveExpiredLocked(DateTimeOffset now) { foreach (TKey key in expired) { - Remove(key, items[key]); + items.Remove(key); } } } - private void EvictOldest() - { - LinkedListNode? oldest = insertionOrder.First; - if (oldest is null) - { - throw new InvalidOperationException("TTL cache insertion order was unexpectedly empty."); - } - - insertionOrder.RemoveFirst(); - if (!items.Remove(oldest.Value)) - { - throw new InvalidOperationException("TTL cache insertion order was out of sync with its entries."); - } - } - - private void Remove(TKey key, CachedItem item) - { - items.Remove(key); - insertionOrder.Remove(item.Node); - } - public void Dispose() { if (Interlocked.Exchange(ref disposed, 1) != 0) @@ -171,15 +132,14 @@ public void Dispose() lock (sync) { items.Clear(); - insertionOrder.Clear(); } } internal IList ToList() { - DateTimeOffset now = DateTimeOffset.UtcNow; lock (sync) { + DateTimeOffset now = timeProvider.GetUtcNow(); return items.Values .Where(item => item.ValidTill > now) .Select(item => item.Item) @@ -188,7 +148,7 @@ internal IList ToList() } } -internal class TtlCache(int ttl, int maxEntries = int.MaxValue) : TtlCache(ttl, maxEntries) where TKey : notnull +internal class TtlCache(int ttl, TimeProvider? timeProvider = null) : TtlCache(ttl, timeProvider) where TKey : notnull { public void Add(TKey key) => Add(key, true); } From 10a37ddfc84f63a281fd60ee821e9bd5cb3b0ccd Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 18:32:54 +0300 Subject: [PATCH 40/45] Address independent review feedback --- .../PubsubFrameLimitTests.cs | 11 +- .../PubsubSessionFailureTests.cs | 110 ++++++++++++++++++ .../Libp2p.Protocols.Pubsub/PubSubSettings.cs | 2 +- .../Libp2p.Protocols.Pubsub/PubsubProtocol.cs | 35 +++--- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 21 +++- 5 files changed, 155 insertions(+), 24 deletions(-) create mode 100644 src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubSessionFailureTests.cs diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs index 3b93306b..bc2e3e71 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubFrameLimitTests.cs @@ -8,6 +8,7 @@ namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; [TestFixture] +[CancelAfter(5_000)] public class PubsubFrameLimitTests { [Test] @@ -37,11 +38,12 @@ public void MaxRpcBytes_RejectsNonPositiveValues() public async Task ReadPrefixedProtobufAsync_RejectsFramesLargerThanLimit(ulong messageLength) { TestChannel channel = new(); - Task write = channel.Reverse().WriteAsync(new ReadOnlySequence(EncodeVarint(messageLength))).AsTask(); + CancellationToken token = TestContext.CurrentContext.CancellationToken; + Task write = channel.Reverse().WriteAsync(new ReadOnlySequence(EncodeVarint(messageLength)), token).AsTask(); IChannel reader = channel; InvalidDataException? exception = Assert.ThrowsAsync(async () => - await reader.ReadPrefixedProtobufAsync(Rpc.Parser, 1_024)); + await reader.ReadPrefixedProtobufAsync(Rpc.Parser, 1_024, token)); Assert.That(exception!.Message, Does.Contain(messageLength.ToString())); await write; @@ -52,10 +54,11 @@ public async Task ReadPrefixedProtobufAsync_RejectsOverflowingVarint() { byte[] overflowedLength = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02]; TestChannel channel = new(); - Task write = channel.Reverse().WriteAsync(new ReadOnlySequence(overflowedLength)).AsTask(); + CancellationToken token = TestContext.CurrentContext.CancellationToken; + Task write = channel.Reverse().WriteAsync(new ReadOnlySequence(overflowedLength), token).AsTask(); IChannel reader = channel; - Assert.ThrowsAsync(async () => await reader.ReadPrefixedProtobufAsync(Rpc.Parser, 1_024)); + Assert.ThrowsAsync(async () => await reader.ReadPrefixedProtobufAsync(Rpc.Parser, 1_024, token)); await write; } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubSessionFailureTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubSessionFailureTests.cs new file mode 100644 index 00000000..ea3edd55 --- /dev/null +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PubsubSessionFailureTests.cs @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: MIT + +using Multiformats.Address; +using Nethermind.Libp2p.Core.Discovery; +using System.Buffers; +using System.Diagnostics; + +namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; + +[TestFixture] +[CancelAfter(5_000)] +public class PubsubSessionFailureTests +{ + [TestCase(false)] + [TestCase(true)] + public async Task ListenAsync_Eof_DoesNotMarkActivityAsErrorAndReconnects(bool outboundFirst) + { + await CheckSessionFailure([], outboundFirst, protocolViolation: false); + } + + [TestCase(false, new byte[] { 1, 0 })] // Invalid protobuf tag in a small frame. + [TestCase(true, new byte[] { 1, 0 })] + [TestCase(false, new byte[] { 0x81, 0x08 })] // 1,025 bytes, exceeding the configured limit. + [TestCase(true, new byte[] { 0x81, 0x08 })] + [TestCase(false, new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 2 })] + [TestCase(true, new byte[] { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 2 })] + public async Task ListenAsync_InvalidFrame_DisconnectsWithoutReconnecting(bool outboundFirst, byte[] frame) + { + await CheckSessionFailure(frame, outboundFirst, protocolViolation: true); + } + + private static async Task CheckSessionFailure(byte[] frame, bool outboundFirst, bool protocolViolation) + { + using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.CurrentContext.CancellationToken); + using PubsubRouter router = new(new PeerStore(), new PubsubSettings { MaxRpcBytes = 1_024, ReconnectionPeriod = 10 }); + ILocalPeer localPeer = Substitute.For(); + localPeer.Identity.Returns(TestPeers.Identity(1)); + localPeer.ListenAddresses.Returns([TestPeers.Multiaddr(1)]); + Multiaddress remoteAddress = TestPeers.Multiaddr(2); + TaskCompletionSource redial = new(TaskCreationOptions.RunContinuationsAsynchronously); + localPeer.DialAsync(Arg.Any(), Arg.Any()).Returns(_ => + { + redial.TrySetResult(); + return new TestRemotePeer(remoteAddress); + }); + + await router.StartAsync(localPeer, cts.Token); + using Activity activity = new Activity("pubsub-listen").Start(); + ISessionContext context = Substitute.For(); + context.State.Returns(new State { RemoteAddress = remoteAddress }); + context.Activity.Returns(activity); + TestChannel channel = new(); + TaskCompletionSource outboundClosed = new(TaskCreationOptions.RunContinuationsAsynchronously); + void OpenOutbound() => router.OutboundConnection(remoteAddress, PubsubRouter.FloodsubProtocolVersion, outboundClosed.Task, _ => { }); + if (outboundFirst) + { + OpenOutbound(); + } + else + { + context.DialAsync(Arg.Any()).Returns(_ => + { + OpenOutbound(); + return Task.CompletedTask; + }); + } + context.DisconnectAsync().Returns(async _ => + { + outboundClosed.TrySetResult(); + await channel.CloseAsync(); + }); + + try + { + Task listen = new FloodsubProtocol(router).ListenAsync(channel, context); + if (frame.Length == 0) + { + await channel.Reverse().WriteEofAsync(cts.Token); + } + else + { + await channel.Reverse().WriteAsync(new ReadOnlySequence(frame), cts.Token); + } + await listen.WaitAsync(cts.Token); + outboundClosed.TrySetResult(); + + if (protocolViolation) + { + await context.Received(1).DisconnectAsync(); + Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Error)); + await Task.Delay(200, cts.Token); + Assert.That(((IRoutingStateContainer)router).ConnectedPeers, Is.Empty); + Assert.That(redial.Task.IsCompleted, Is.False, "A protocol violator must not be redialed by either stream's cleanup."); + } + else + { + await context.DidNotReceive().DisconnectAsync(); + Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Unset)); + await redial.Task.WaitAsync(cts.Token); + } + } + finally + { + await cts.CancelAsync(); + await channel.CloseAsync(); + outboundClosed.TrySetResult(); + } + } +} diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index cae2662d..acc44d9d 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -27,7 +27,7 @@ public class PubsubSettings public int mcache_len { get; set; } = 5; // Number of history windows in message cache 5 public int mcache_gossip { get; set; } = 3; // Number of history windows to use when emitting gossip 3 public int MessageCacheTtl { get; set; } = 2 * 60 * 1000; // Expiry time for cache of seen message ids 2 minutes - // Maximum incoming RPC frame size 1 MiB + // Maximum incoming RPC frame size in bytes (default: 1 MiB). public int MaxRpcBytes { get => maxRpcBytes; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs index 1ff2628b..396d158a 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubProtocol.cs @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: MIT +using Google.Protobuf; using Microsoft.Extensions.Logging; using Nethermind.Libp2p.Core; +using Nethermind.Libp2p.Core.Exceptions; using Nethermind.Libp2p.Protocols.Pubsub; using Nethermind.Libp2p.Protocols.Pubsub.Dto; using System.Diagnostics; @@ -75,26 +77,23 @@ public async Task ListenAsync(IChannel channel, ISessionContext context) { while (!token.IsCancellationRequested) { - Rpc? rpc = await channel.ReadPrefixedProtobufAsync(Rpc.Parser, router.MaxRpcBytes, token); - if (rpc is null) - { - _logger?.LogDebug("Received a broken message or EOF from {remotePeerId}", remotePeerId); - context.Activity?.AddEvent(new ActivityEvent($"Received a broken message or EOF from {remotePeerId}")); - break; - } - else - { - _logger?.LogTrace("Received message from {remotePeerId}: {rpc}", remotePeerId, rpc); - context.Activity?.AddEvent(new ActivityEvent($"Received message from {remotePeerId}: {rpc}")); - router.OnRpc(remotePeerId, rpc); - } + Rpc rpc = await channel.ReadPrefixedProtobufAsync(Rpc.Parser, router.MaxRpcBytes, token); + _logger?.LogTrace("Received message from {remotePeerId}: {rpc}", remotePeerId, rpc); + context.Activity?.AddEvent(new ActivityEvent($"Received message from {remotePeerId}: {rpc}")); + router.OnRpc(remotePeerId, rpc); } } - catch (Exception e) when (e is InvalidDataException or FormatException) + catch (ChannelClosedException) + { + _logger?.LogDebug("RPC channel closed by {remotePeerId}", remotePeerId); + context.Activity?.AddEvent(new ActivityEvent($"RPC channel closed by {remotePeerId}")); + } + catch (Exception e) when (e is InvalidDataException or FormatException or InvalidProtocolBufferException) { _logger?.LogDebug(e, "Invalid RPC from {remotePeerId}: {message}", remotePeerId, e.Message); context.Activity?.AddEvent(new ActivityEvent($"Invalid RPC from {remotePeerId}: {e.Message}")); context.Activity?.SetStatus(ActivityStatusCode.Error); + router.SuppressReconnection(remotePeerId); await context.DisconnectAsync(); } catch (Exception e) @@ -102,9 +101,11 @@ public async Task ListenAsync(IChannel channel, ISessionContext context) context.Activity?.AddEvent(new ActivityEvent($"Exception: {e.Message}")); context.Activity?.SetStatus(ActivityStatusCode.Error); } - - listTcs.SetResult(); - context.Activity?.AddEvent(new ActivityEvent($"Finished({context.Id}) list {context.State.RemoteAddress}")); + finally + { + listTcs.SetResult(); + context.Activity?.AddEvent(new ActivityEvent($"Finished({context.Id}) list {context.State.RemoteAddress}")); + } } public override string ToString() => Id; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index 4a3f573d..ed9c0115 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -106,6 +106,9 @@ public Action? SendRpc public ConnectionInitiation InitiatedBy { get; internal set; } public Multiaddress Address { get; internal set; } = null!; + // Shared by both streams; set before disconnecting a peer for a protocol violation. + public volatile bool SuppressReconnection; + // Peer scoring (Gossipsub v1.1) public PeerScore Score { get; internal set; } } @@ -455,6 +458,14 @@ public Task Heartbeat() return Task.CompletedTask; } + internal void SuppressReconnection(PeerId peerId) + { + if (peerState.TryGetValue(peerId, out PubsubPeer? peer)) + { + peer.SuppressReconnection = true; + } + } + internal CancellationToken OutboundConnection(Multiaddress addr, string protocolId, Task dialTask, Action sendRpc) { PeerId? peerId = addr.GetPeerId(); @@ -504,7 +515,10 @@ internal CancellationToken OutboundConnection(Multiaddress addr, string protocol { topicPeers.Value.Remove(peerId); } - reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); + if (!peer.SuppressReconnection) + { + reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); + } }); string[] topics = topicState.Keys.ToArray(); @@ -559,7 +573,10 @@ internal CancellationToken InboundConnection(Multiaddress addr, string protocolI { topicPeers.Value.Remove(peerId); } - reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); + if (!existingPeer.SuppressReconnection) + { + reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts)); + } }); subDial(); From cc08c0175b1ed99addff4062743336419783144a Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 18:49:11 +0300 Subject: [PATCH 41/45] Address independent review feedback --- .../StrictNoSignTests.cs | 58 +++++++++++++++++++ .../PubsubRouter.Topics.cs | 16 ++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs index c72572eb..cdef106e 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/StrictNoSignTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT using Google.Protobuf; +using Multiformats.Address; using Nethermind.Libp2p.Core.Discovery; using Nethermind.Libp2p.Protocols.Pubsub; using Nethermind.Libp2p.Protocols.Pubsub.Dto; @@ -75,6 +76,63 @@ public void StrictNoSign_DeliversDistinctMessagesWithACustomMessageId() Assert.That(deliveries, Is.EqualTo(2)); } + [TestCase(PubsubSettings.SignaturePolicy.StrictNoSign, PubsubRouter.FloodsubProtocolVersion)] + [TestCase(PubsubSettings.SignaturePolicy.StrictNoSign, PubsubRouter.GossipsubProtocolVersionV11)] + [TestCase(PubsubSettings.SignaturePolicy.StrictSign, PubsubRouter.FloodsubProtocolVersion)] + [TestCase(PubsubSettings.SignaturePolicy.StrictSign, PubsubRouter.GossipsubProtocolVersionV11)] + public async Task Publish_RespectsSignaturePolicy(PubsubSettings.SignaturePolicy policy, string protocol) + { + const string topicName = "publish-policy"; + PubsubSettings settings = new() + { + DefaultSignaturePolicy = policy, + GetMessageId = message => new(message.Data.ToByteArray()), + }; + using PubsubRouter sender = new(new PeerStore(), settings); + using PubsubRouter receiver = new(new PeerStore(), settings); + LocalPeerStub localPeer = new(); + using CancellationTokenSource stopped = new(); + stopped.Cancel(); + await sender.StartAsync(localPeer, stopped.Token); + + List deliveries = []; + receiver.GetTopic(topicName).OnMessage += (_, payload) => deliveries.Add(payload); + Multiaddress receiverAddress = TestPeers.Multiaddr(1); + List sent = []; + TaskCompletionSource connectionClosed = new(); + sender.OutboundConnection(receiverAddress, protocol, connectionClosed.Task, sent.Add); + sender.OnRpc(receiverAddress.GetPeerId()!, new Rpc().WithTopics([topicName], [])); + + byte[][] payloads = [[1, 2, 3], [4, 5, 6]]; + foreach (byte[] payload in payloads) + { + sender.Publish(topicName, payload); + } + + Rpc[] published = sent.Where(rpc => rpc.Publish.Count > 0) + .Select(rpc => Rpc.Parser.ParseFrom(rpc.ToByteArray())).ToArray(); + Assert.That(published, Has.Length.EqualTo(payloads.Length)); + for (int i = 0; i < published.Length; i++) + { + Message message = published[i].Publish.Single(); + bool signed = policy == PubsubSettings.SignaturePolicy.StrictSign; + Assert.Multiple(() => + { + Assert.That(message.Topic, Is.EqualTo(topicName)); + Assert.That(message.Data.ToByteArray(), Is.EqualTo(payloads[i])); + Assert.That(message.HasFrom, Is.EqualTo(signed)); + Assert.That(message.HasSeqno, Is.EqualTo(signed)); + Assert.That(message.HasSignature, Is.EqualTo(signed)); + Assert.That(message.HasKey, Is.False); + Assert.That(message.VerifySignature(policy), Is.True); + }); + receiver.OnRpc(localPeer.Identity.PeerId, published[i]); + } + + Assert.That(deliveries, Is.EqualTo(payloads)); + connectionClosed.SetResult(); + } + private static Rpc CreateUnsignedMessage(string payload) { Rpc rpc = new(); diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index c99b09f0..5a15cc41 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited // SPDX-License-Identifier: MIT +using Google.Protobuf; using Nethermind.Libp2p.Core; using Nethermind.Libp2p.Protocols.Pubsub.Dto; using System.Collections.Concurrent; @@ -156,8 +157,19 @@ public void Publish(string topicId, byte[] message) { topicState.GetOrAdd(topicId, (id) => new Topic(this, topicId)); - ulong seqNo = this.seqNo++; - Rpc rpc = new Rpc().WithMessages(topicId, seqNo, localPeer.Identity.PeerId.Bytes, message, localPeer.Identity); + Rpc rpc = new(); + if (_settings.DefaultSignaturePolicy is PubsubSettings.SignaturePolicy.StrictNoSign) + { + rpc.Publish.Add(new Message + { + Topic = topicId, + Data = ByteString.CopyFrom(message), + }); + } + else + { + rpc.WithMessages(topicId, seqNo++, localPeer.Identity.PeerId.Bytes, message, localPeer.Identity); + } // Floodsub peers always get the message foreach (PeerId peerId in fPeers.GetValueOrDefault(topicId) ?? []) From a8e70fe1a4a4e65de38a5ed1cb56c88ca6e5b1aa Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 18:56:37 +0300 Subject: [PATCH 42/45] Address independent review feedback --- .../GossipsubV13ProtocolTests.cs | 78 +++++++++++++++++++ .../PubsubRouter.Rpc.cs | 1 - 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs index ba9a9bc0..abd00db5 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/GossipsubV13ProtocolTests.cs @@ -7,12 +7,90 @@ using Nethermind.Libp2p.Protocols; using Nethermind.Libp2p.Protocols.Pubsub.Dto; using System.Collections.ObjectModel; +using System.Reflection; namespace Nethermind.Libp2p.Protocols.Pubsub.Tests; [TestFixture] public class GossipsubV13ProtocolTests { + [TestCase(false, TestName = "Router_PenalizesExtensionsInSecondRpc")] + [TestCase(true, TestName = "Router_PenalizesRepeatedExtensions")] + public void Router_PenalizesExtensionsAfterFirstRpc(bool firstRpcHasExtensions) + { + using PubsubRouter router = CreateConnectedRouter(PubsubRouter.GossipsubProtocolVersionV13); + PeerId peerId = TestPeers.PeerId(1); + double initialScore = GetPeerScore(router, peerId); + + router.OnRpc(peerId, firstRpcHasExtensions ? CreateExtensionsRpc() : new Rpc()); + Assert.That(GetPeerScore(router, peerId), Is.EqualTo(initialScore)); + + router.OnRpc(peerId, CreateExtensionsRpc()); + double penalizedScore = GetPeerScore(router, peerId); + Assert.That(penalizedScore, Is.LessThan(initialScore)); + + router.OnRpc(peerId, new Rpc()); + Assert.That(GetPeerScore(router, peerId), Is.EqualTo(penalizedScore), + "Subsequent RPCs without extensions must not incur a penalty."); + + router.OnRpc(peerId, CreateExtensionsRpc()); + Assert.That(GetPeerScore(router, peerId), Is.LessThan(penalizedScore), + "Each additional extensions advertisement must incur a penalty."); + } + + [TestCase(false)] + [TestCase(true)] + public void Router_AllowsFirstRpcWithOrWithoutExtensions(bool firstRpcHasExtensions) + { + using PubsubRouter router = CreateConnectedRouter(PubsubRouter.GossipsubProtocolVersionV13); + PeerId peerId = TestPeers.PeerId(1); + double initialScore = GetPeerScore(router, peerId); + + router.OnRpc(peerId, firstRpcHasExtensions ? CreateExtensionsRpc() : new Rpc()); + router.OnRpc(peerId, new Rpc { Control = new ControlMessage() }); + + Assert.That(GetPeerScore(router, peerId), Is.EqualTo(initialScore)); + } + + [TestCase(PubsubRouter.FloodsubProtocolVersion)] + [TestCase(PubsubRouter.GossipsubProtocolVersionV10)] + [TestCase(PubsubRouter.GossipsubProtocolVersionV11)] + [TestCase(PubsubRouter.GossipsubProtocolVersionV12)] + public void Router_IgnoresExtensionsOnOlderProtocols(string protocolId) + { + using PubsubRouter router = CreateConnectedRouter(protocolId); + PeerId peerId = TestPeers.PeerId(1); + double initialScore = GetPeerScore(router, peerId); + + router.OnRpc(peerId, CreateExtensionsRpc()); + Assert.That(GetPeerScore(router, peerId), Is.EqualTo(initialScore)); + + router.OnRpc(peerId, CreateExtensionsRpc()); + Assert.That(GetPeerScore(router, peerId), Is.EqualTo(initialScore)); + } + + private static PubsubRouter CreateConnectedRouter(string protocolId) + { + PubsubRouter router = new(new PeerStore()); + TaskCompletionSource connectionLifetime = new(); + router.OutboundConnection(TestPeers.Multiaddr(1), protocolId, connectionLifetime.Task, _ => { }); + Assert.That(((IRoutingStateContainer)router).ConnectedPeers, Does.Contain(TestPeers.PeerId(1))); + return router; + } + + private static Rpc CreateExtensionsRpc() => new() + { + Control = new ControlMessage + { + Extensions = new ControlExtensions { PartialMessages = true }, + }, + }; + + private static double GetPeerScore(PubsubRouter router, PeerId peerId) => + (double)typeof(PubsubRouter) + .GetMethod("GetPeerScore", BindingFlags.Instance | BindingFlags.NonPublic)! + .Invoke(router, [peerId])!; + [Test] public void Protocol_UsesTheV13ProtocolId() { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs index 56fc4244..c498268a 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Rpc.cs @@ -87,7 +87,6 @@ private void HandleExtensions(PeerId peerId, Rpc rpc) { if (extensions is not null) { - ApplyBehaviorPenalty(peerId, 1.0); logger?.LogDebug("Ignoring Gossipsub v1.3 extensions from {peerId} on {protocol}", peerId, peer.Protocol); } From 5e3d4465cbbda680ffa1e5cdf4939866a500e5d8 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 19:02:38 +0300 Subject: [PATCH 43/45] Address independent review feedback --- .../DirectPeersTests.cs | 122 +++++++++++++++++- .../Libp2p.Protocols.Pubsub/PubSubSettings.cs | 2 +- .../PubsubRouter.Topics.cs | 4 +- .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 23 ++-- 4 files changed, 136 insertions(+), 15 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs index 8d999010..b9bbe1a7 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs @@ -114,13 +114,18 @@ public void DirectPeerGrafts_AreRejectedWithPrune() connection.SetResult(); } - [Test] - public async Task Router_ConnectsConfiguredDirectPeersAtStartup() + [TestCase(false)] + [TestCase(true)] + public async Task Router_ConnectsConfiguredDirectPeersAtStartup(bool addressesAlreadyKnown) { PeerStore peerStore = new(); Multiaddress directAddress = TestPeers.Multiaddr(1); PeerId directPeerId = directAddress.GetPeerId()!; peerStore.GetPeerInfo(directPeerId).SupportedProtocols = [PubsubRouter.GossipsubProtocolVersionV12]; + if (addressesAlreadyKnown) + { + peerStore.Discover([directAddress]); + } PubsubRouter router = new(peerStore, new PubsubSettings { DirectPeers = [directAddress] }); TaskCompletionSource protocolDialed = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -147,6 +152,119 @@ public async Task Router_ConnectsConfiguredDirectPeersAtStartup() cancellation.Cancel(); } + [Test] + public async Task PublishPartial_ExcludesDirectPeersFromFanoutAndMesh() + { + const string topic = "topic"; + Multiaddress directAddress = TestPeers.Multiaddr(1); + Multiaddress otherAddress = TestPeers.Multiaddr(2); + using PubsubRouter router = new(new PeerStore(), new PubsubSettings + { + DirectPeers = [directAddress], + EnablePartialMessages = true, + }); + IRoutingStateContainer state = router; + router.GetPartialMessagesTopic(topic, + new PartialMessagesTopicOptions { SupportsSendingPartialMessages = true }, subscribe: false); + ILocalPeer localPeer = Substitute.For(); + localPeer.Identity.Returns(TestPeers.Identity(3)); + localPeer.ListenAddresses.Returns(new ObservableCollection()); + using CancellationTokenSource cancellation = new(); + TaskCompletionSource connection = new(); + try + { + await router.StartAsync(localPeer, cancellation.Token); + foreach (Multiaddress address in new[] { directAddress, otherAddress }) + { + router.OutboundConnection(address, PubsubRouter.GossipsubProtocolVersionV13, connection.Task, _ => { }); + router.OnRpc(address.GetPeerId()!, new Rpc().WithTopics([topic], [])); + } + + router.PublishPartial(topic, [1], partialMessage: [2]); + Assert.That(state.Fanout[topic], Is.EquivalentTo(new[] { otherAddress.GetPeerId() })); + + router.Subscribe(topic); + Assert.That(state.Mesh[topic], Is.EquivalentTo(new[] { otherAddress.GetPeerId() })); + } + finally + { + cancellation.Cancel(); + connection.TrySetResult(); + } + } + + [Test] + public void Subscribe_ExcludesDirectPeersWhenMovingFanoutToMesh() + { + const string topic = "topic"; + Multiaddress directAddress = TestPeers.Multiaddr(1); + PeerId otherPeerId = TestPeers.Multiaddr(2).GetPeerId()!; + using PubsubRouter router = new(new PeerStore(), new PubsubSettings { DirectPeers = [directAddress] }); + IRoutingStateContainer state = router; + state.Fanout[topic] = [directAddress.GetPeerId()!, otherPeerId]; + + router.Subscribe(topic); + + Assert.That(state.Mesh[topic], Is.EquivalentTo(new[] { otherPeerId })); + Assert.That(state.Fanout.ContainsKey(topic), Is.False); + } + + [Test] + public async Task Router_ReconnectsDisconnectedDirectPeersIndependentlyOfOtherIntervals() + { + Multiaddress directAddress = TestPeers.Multiaddr(1); + PeerStore peerStore = new(); + peerStore.GetPeerInfo(directAddress.GetPeerId()!).SupportedProtocols = [PubsubRouter.GossipsubProtocolVersionV12]; + using PubsubRouter router = new(peerStore, new PubsubSettings + { + DirectPeers = [directAddress], + DirectConnectPeriod = 50, + ReconnectionPeriod = 60_000, + HeartbeatInterval = 60_000, + }); + TaskCompletionSource firstConnection = new(); + TaskCompletionSource secondConnection = new(); + TaskCompletionSource redialed = new(TaskCreationOptions.RunContinuationsAsynchronously); + int protocolDials = 0; + ISession session = Substitute.For(); + session.RemoteAddress.Returns(directAddress); + session.DialAsync(Arg.Any()).Returns(_ => + { + int attempt = Interlocked.Increment(ref protocolDials); + router.OutboundConnection(directAddress, PubsubRouter.GossipsubProtocolVersionV12, + attempt == 1 ? firstConnection.Task : secondConnection.Task, _ => { }); + if (attempt > 1) + { + redialed.TrySetResult(); + } + return Task.CompletedTask; + }); + ILocalPeer localPeer = Substitute.For(); + localPeer.Identity.Returns(TestPeers.Identity(2)); + localPeer.ListenAddresses.Returns(new ObservableCollection()); + localPeer.DialAsync(Arg.Any(), Arg.Any()).Returns(session); + using CancellationTokenSource cancellation = new(); + try + { + await router.StartAsync(localPeer, cancellation.Token); + await Task.Delay(200); + _ = localPeer.Received(1).DialAsync(Arg.Any(), Arg.Any()); + + firstConnection.SetResult(); + await redialed.Task.WaitAsync(TimeSpan.FromSeconds(3)); + + await Task.Delay(200); + _ = localPeer.Received(2).DialAsync(Arg.Any(), Arg.Any()); + _ = session.Received(2).DialAsync(Arg.Any()); + } + finally + { + cancellation.Cancel(); + firstConnection.TrySetResult(); + secondConnection.TrySetResult(); + } + } + [Test] public void DirectPeers_RequirePeerIdsInTheirAddresses() { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs index 1d8782d7..541074b9 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubSubSettings.cs @@ -30,7 +30,7 @@ public class PubsubSettings public Multiaddress[] DirectPeers { get; set; } = []; /// - /// Interval for reconnecting disconnected direct peers. Gossipsub recommends + /// Interval in milliseconds for reconnecting disconnected direct peers. Gossipsub recommends /// five minutes. /// public int DirectConnectPeriod { get; set; } = 5 * 60 * 1000; diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs index 13a2fff7..e5b6ed32 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.Topics.cs @@ -111,7 +111,7 @@ public void Subscribe(string topicId) { foreach (PeerId peerId in fanoutPeers.ToList()) { - if (meshPeers.Add(peerId)) + if (!IsDirectPeer(peerId) && meshPeers.Add(peerId)) { RecordPeerJoinMesh(peerId, topicId); } @@ -297,7 +297,7 @@ public void PublishPartial(string topicId, byte[] groupId, byte[]? partialMessag HashSet fanoutPeers = fanout.GetOrAdd(topicId, _ => []); if (fanoutPeers.Count == 0 && gPeers.TryGetValue(topicId, out HashSet? topicPeers)) { - foreach (PeerId peerId in topicPeers.Where(peerId => GetPeerScore(peerId) >= 0).Take(_settings.Degree)) + foreach (PeerId peerId in topicPeers.Where(peerId => !IsDirectPeer(peerId) && GetPeerScore(peerId) >= 0).Take(_settings.Degree)) { fanoutPeers.Add(peerId); } diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index f4102335..3d7743ff 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -255,7 +255,6 @@ public Action? SendRpc private readonly ConcurrentBag reconnections = []; private readonly PeerStore _peerStore; private readonly IReadOnlyDictionary directPeers; - private DateTime nextDirectConnectionAttempt; private ulong seqNo = 1; private record Reconnection(Multiaddress[] Addresses, int Attempts); @@ -314,10 +313,13 @@ public Task StartAsync(ILocalPeer localPeer, CancellationToken token = default) { _peerStore.Discover(directPeerAddresses); } - nextDirectConnectionAttempt = DateTime.UtcNow.AddMilliseconds(_settings.DirectConnectPeriod); _ = Task.Run(LoopHeartbeat, token); _ = Task.Run(LoopReconnect, token); + if (directPeers.Count > 0) + { + _ = Task.Run(LoopReconnectDirectPeers, token); + } logger?.LogInformation("Started"); return Task.CompletedTask; @@ -340,6 +342,15 @@ async Task LoopReconnect() Reconnect(token); } } + + async Task LoopReconnectDirectPeers() + { + while (!token.IsCancellationRequested) + { + await Task.Delay(_settings.DirectConnectPeriod, token); + ReconnectDirectPeers(token); + } + } } private async Task Connect(Multiaddress[] addrs, CancellationToken token, bool reconnect = false) @@ -412,8 +423,6 @@ private void Reconnect(CancellationToken token) } }, token); } - - ReconnectDirectPeers(token); } private static IReadOnlyDictionary CreateDirectPeers(IEnumerable? configuredPeers) @@ -426,12 +435,6 @@ private static IReadOnlyDictionary CreateDirectPeers(IEn private void ReconnectDirectPeers(CancellationToken token) { - if (directPeers.Count == 0 || DateTime.UtcNow < nextDirectConnectionAttempt) - { - return; - } - - nextDirectConnectionAttempt = DateTime.UtcNow.AddMilliseconds(_settings.DirectConnectPeriod); foreach ((PeerId peerId, Multiaddress[] addresses) in directPeers) { if (!peerState.ContainsKey(peerId)) From 93a518e3ec06d6e05efb118118b9603f3f9533b0 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Tue, 22 Sep 2026 19:41:13 +0300 Subject: [PATCH 44/45] Use spelling-friendly partial message test name --- .../Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs index ecbcdc8e..2ce68539 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/PartialMessagesTests.cs @@ -451,7 +451,7 @@ public void PartialMessages_TopicDeliveryRequiresSubscriptionAndCapabilities() [TestCase("ordinary")] [TestCase("unknown")] [TestCase("unsubscribed")] - public async Task PartialMessages_UnrequestedDataIsDroppedAndPenalized(string mode) + public async Task PartialMessages_DataWithoutARequestIsDroppedAndPenalized(string mode) { using PubsubRouter router = new(new PeerStore(), new PubsubSettings { EnablePartialMessages = true }); if (mode == "ordinary") From 1a92986557451a566abe41a8efa38fa0554ac395 Mon Sep 17 00:00:00 2001 From: Alexey Osipov Date: Wed, 23 Sep 2026 18:23:39 +0300 Subject: [PATCH 45/45] Dial direct peers from a single deduplicated path Startup discovery, the direct-peer loop and the generic reconnection queue could all dial the same direct peer. A dial slower than DirectConnectPeriod was repeated, which also made the reconnect test timing-sensitive on busy runners. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../DirectPeersTests.cs | 32 +++++++++++++++++++ .../Libp2p.Protocols.Pubsub/PubsubRouter.cs | 20 ++++++++---- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs index 212989c3..691b01e2 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub.Tests/DirectPeersTests.cs @@ -276,6 +276,38 @@ public async Task Router_ReconnectsDisconnectedDirectPeersIndependentlyOfOtherIn } } + [Test] + public async Task Router_DoesNotRepeatInFlightDirectPeerDials() + { + Multiaddress directAddress = TestPeers.Multiaddr(1); + PeerStore peerStore = new(); + peerStore.Discover([directAddress]); + await using PubsubRouter router = new(peerStore, new PubsubSettings + { + DirectPeers = [directAddress], + DirectConnectPeriod = 20, + ReconnectionPeriod = 60_000, + HeartbeatInterval = 60_000, + }); + TaskCompletionSource slowDial = new(TaskCreationOptions.RunContinuationsAsynchronously); + ILocalPeer localPeer = Substitute.For(); + localPeer.Identity.Returns(TestPeers.Identity(2)); + localPeer.ListenAddresses.Returns(new ObservableCollection()); + localPeer.DialAsync(Arg.Any(), Arg.Any()).Returns(slowDial.Task); + + try + { + await router.StartAsync(localPeer); + await Task.Delay(200); + + _ = localPeer.Received(1).DialAsync(Arg.Any(), Arg.Any()); + } + finally + { + slowDial.TrySetCanceled(); + } + } + [Test] public void DirectPeers_RequirePeerIdsInTheirAddresses() { diff --git a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs index 61da6a14..853ab244 100644 --- a/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs +++ b/src/libp2p/Libp2p.Protocols.Pubsub/PubsubRouter.cs @@ -279,6 +279,7 @@ public Action? SendRpc private readonly ConcurrentBag reconnections = []; private readonly PeerStore _peerStore; private readonly IReadOnlyDictionary directPeers; + private readonly ConcurrentDictionary pendingDirectDials = new(); private ulong seqNo = 1; // Lifetime of work owned by the router: background loops and dials it starts itself. @@ -385,10 +386,12 @@ public Task StartAsync(ILocalPeer localPeer, CancellationToken token = default) { _peerStore.Discover(directPeerAddresses); } + ConnectDirectPeers(lifetime); _onNewPeer = (addrs) => { - if (addrs.Any(a => a.GetPeerId()! == localPeer.Identity.PeerId)) + // Direct peers are dialed by ConnectDirectPeers. + if (addrs.Any(a => a.GetPeerId() is PeerId peerId && (peerId == localPeer.Identity.PeerId || IsDirectPeer(peerId)))) { return; } @@ -426,7 +429,7 @@ private async Task LoopReconnectDirectPeers(CancellationToken token) while (!token.IsCancellationRequested) { await Task.Delay(_settings.DirectConnectPeriod, token); - ReconnectDirectPeers(token); + ConnectDirectPeers(token); } } @@ -581,14 +584,19 @@ private static IReadOnlyDictionary CreateDirectPeers(IEn .ToDictionary(group => group.Key, group => group.Select(entry => entry.Address).ToArray()); } - private void ReconnectDirectPeers(CancellationToken token) + private void ConnectDirectPeers(CancellationToken token) { foreach ((PeerId peerId, Multiaddress[] addresses) in directPeers) { - if (!peerState.ContainsKey(peerId)) + // The direct-peer loop owns retries, so a dial that is still in flight is not repeated. + if (peerState.ContainsKey(peerId) || !pendingDirectDials.TryAdd(peerId, 0)) { - Track(Connect(addresses, token, reconnect: true)); + continue; } + + Task connect = Connect(addresses, token); + Track(connect); + _ = connect.ContinueWith(t => pendingDirectDials.TryRemove(peerId, out byte _), TaskScheduler.Default); } } @@ -842,7 +850,7 @@ private void RemovePeer(PeerId peerId, Multiaddress addr, PubsubPeer peer) topicPeers.Remove(peerId); } } - if (!peer.ReconnectionPolicy.Suppressed && !_stopped.IsCancellationRequested) + if (!peer.ReconnectionPolicy.Suppressed && !_stopped.IsCancellationRequested && !IsDirectPeer(peerId)) { reconnections.Add(new Reconnection([addr], _settings.ReconnectionAttempts, peer.ReconnectionPolicy)); }