diff --git a/src/Persistence/MySql/MySqlTests/Transport/scheduled_retry_is_atomic.cs b/src/Persistence/MySql/MySqlTests/Transport/scheduled_retry_is_atomic.cs new file mode 100644 index 000000000..716fccbf1 --- /dev/null +++ b/src/Persistence/MySql/MySqlTests/Transport/scheduled_retry_is_atomic.cs @@ -0,0 +1,124 @@ +using IntegrationTests; +using MySqlConnector; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Wolverine; +using Wolverine.Configuration; +using Wolverine.MySql; +using Wolverine.MySql.Transport; +using Wolverine.Tracking; + +namespace MySqlTests.Transport; + +/// +/// A scheduled retry of a message received on a MySQL queue deletes its inbox row and writes it to the +/// queue's scheduled table. Those were two autocommitted statements in one command, so a scheduled write that failed -- a command +/// timeout or a deadlock, found by the durability chaos rig -- left the message in neither table, alive only in +/// the receiver's in-memory retry, and a node that died or shut down before that retry succeeded lost it. +/// +public class scheduled_retry_is_atomic : IAsyncLifetime +{ + private const string SchemaName = "wolverine_schedretry"; + private const string QueueName = "schedretry"; + + private IHost _host = null!; + private MySqlQueue _queue = null!; + + public async ValueTask InitializeAsync() + { + await using (var conn = new MySqlConnection(Servers.MySqlConnectionString)) + { + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = $"DROP DATABASE IF EXISTS {SchemaName}; CREATE DATABASE {SchemaName}"; + await cmd.ExecuteNonQueryAsync(); + } + + _host = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Discovery.DisableConventionalDiscovery(); + opts.Durability.Mode = DurabilityMode.Solo; + opts.UseMySqlPersistenceAndTransport(Servers.MySqlConnectionString, SchemaName, + transportSchema: SchemaName); + + // Declared without a listener, so nothing but the test touches the queue + opts.PublishMessage().ToMySqlQueue(QueueName); + }).StartAsync(); + + _queue = _host.GetRuntime().Options.Transports.GetOrCreate().Queues[QueueName]; + _queue.Mode = EndpointMode.Durable; + await _queue.PurgeAsync(NullLogger.Instance); + await _host.GetRuntime().Storage.Admin.ClearAllAsync(); + } + + public async ValueTask DisposeAsync() + { + await _host.StopAsync(); + _host.Dispose(); + } + + /// + /// What a durable listener has done by the time the handler fails: the envelope sits in the inbox + /// + private async Task receivedIntoTheInboxAsync() + { + var runtime = _host.GetRuntime(); + var envelope = runtime.RoutingFor(typeof(MySqlScheduledRetryMessage)) + .RouteForSend(new MySqlScheduledRetryMessage(Guid.NewGuid()), null).Single(); + envelope.Destination = _queue.Uri; + await runtime.Storage.Inbox.StoreIncomingAsync(envelope); + + envelope.ScheduledTime = DateTimeOffset.UtcNow.AddHours(1); + return envelope; + } + + private async Task isInTheInboxAsync(Envelope envelope) => + (await _host.GetRuntime().Storage.Admin.AllIncomingAsync()).Any(x => x.Id == envelope.Id); + + private async Task executeAsync(string sql) + { + await using var conn = new MySqlConnection(Servers.MySqlConnectionString); + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(); + } + + [Fact] + public async Task a_scheduled_retry_moves_the_inbox_row_to_the_scheduled_table() + { + var envelope = await receivedIntoTheInboxAsync(); + + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } + + [Fact] + public async Task a_failed_scheduled_write_leaves_the_inbox_row_in_place() + { + var envelope = await receivedIntoTheInboxAsync(); + + await executeAsync( + $"CREATE TRIGGER {SchemaName}.fail_scheduled_writes BEFORE INSERT ON {_queue.ScheduledTable.Identifier} " + + "FOR EACH ROW SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'the scheduled write fails'"); + + await Should.ThrowAsync(() => _queue.ScheduleRetryAsync(envelope, CancellationToken.None)); + + // Still in the inbox, so recovery runs it again if this node goes away + (await isInTheInboxAsync(envelope)).ShouldBeTrue(); + (await _queue.ScheduledCountAsync()).ShouldBe(0); + + // and the receiver's retry completes the move once the write can succeed + await executeAsync($"DROP TRIGGER {SchemaName}.fail_scheduled_writes"); + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } +} + +public record MySqlScheduledRetryMessage(Guid Id); diff --git a/src/Persistence/MySql/Wolverine.MySql/Transport/MySqlQueueSender.cs b/src/Persistence/MySql/Wolverine.MySql/Transport/MySqlQueueSender.cs index e13158646..9cee31991 100644 --- a/src/Persistence/MySql/Wolverine.MySql/Transport/MySqlQueueSender.cs +++ b/src/Persistence/MySql/Wolverine.MySql/Transport/MySqlQueueSender.cs @@ -114,13 +114,18 @@ public async Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancel await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); try { - await conn.CreateCommand($"DELETE FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.IncomingTable} WHERE id = @id;" + _writeDirectlyToTheScheduledTable) + // The inbox delete and the scheduled write commit together or not at all: autocommitted one by one, a + // failed scheduled write left the inbox row deleted with nothing scheduled + await using var tx = await conn.BeginTransactionAsync(cancellationToken); + var cmd = conn.CreateCommand($"DELETE FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.IncomingTable} WHERE id = @id;" + _writeDirectlyToTheScheduledTable) .With("id", envelope.Id) .With("body", EnvelopeSerializer.Serialize(envelope)) .With("type", envelope.MessageType) .With("expires", envelope.DeliverBy) - .With("time", envelope.ScheduledTime) - .ExecuteNonQueryAsync(cancellationToken); + .With("time", envelope.ScheduledTime); + cmd.Transaction = tx; + await cmd.ExecuteNonQueryAsync(cancellationToken); + await tx.CommitAsync(cancellationToken); } finally { diff --git a/src/Persistence/Oracle/OracleTests/Transport/scheduled_retry_is_atomic.cs b/src/Persistence/Oracle/OracleTests/Transport/scheduled_retry_is_atomic.cs new file mode 100644 index 000000000..0a22ec650 --- /dev/null +++ b/src/Persistence/Oracle/OracleTests/Transport/scheduled_retry_is_atomic.cs @@ -0,0 +1,107 @@ +using IntegrationTests; +using Oracle.ManagedDataAccess.Client; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Wolverine; +using Wolverine.Configuration; +using Wolverine.Oracle; +using Wolverine.Oracle.Transport; +using Wolverine.Tracking; + +namespace OracleTests.Transport; + +/// +/// A scheduled retry of a message received on an Oracle queue deletes its inbox row and writes it to the +/// queue's scheduled table. Those were two separately committed commands, so a scheduled write that failed -- a command +/// timeout or a deadlock, found by the durability chaos rig -- left the message in neither table, alive only in +/// the receiver's in-memory retry, and a node that died or shut down before that retry succeeded lost it. +/// +public class scheduled_retry_is_atomic : IAsyncLifetime +{ + private const string QueueName = "schedretry"; + + private IHost _host = null!; + private OracleQueue _queue = null!; + + public async ValueTask InitializeAsync() + { + _host = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Discovery.DisableConventionalDiscovery(); + opts.Durability.Mode = DurabilityMode.Solo; + opts.PersistMessagesWithOracle(Servers.OracleConnectionString, "WOLVERINE") + .EnableMessageTransport(); + + // Declared without a listener, so nothing but the test touches the queue + opts.PublishMessage().ToOracleQueue(QueueName); + }).StartAsync(); + + _queue = _host.GetRuntime().Options.Transports.GetOrCreate().Queues[QueueName]; + _queue.Mode = EndpointMode.Durable; + await _queue.PurgeAsync(NullLogger.Instance); + await _host.GetRuntime().Storage.Admin.ClearAllAsync(); + } + + public async ValueTask DisposeAsync() + { + await _host.StopAsync(); + _host.Dispose(); + } + + /// + /// What a durable listener has done by the time the handler fails: the envelope sits in the inbox + /// + private async Task receivedIntoTheInboxAsync() + { + var runtime = _host.GetRuntime(); + var envelope = runtime.RoutingFor(typeof(OracleScheduledRetryMessage)) + .RouteForSend(new OracleScheduledRetryMessage(Guid.NewGuid()), null).Single(); + envelope.Destination = _queue.Uri; + await runtime.Storage.Inbox.StoreIncomingAsync(envelope); + + envelope.ScheduledTime = DateTimeOffset.UtcNow.AddHours(1); + return envelope; + } + + private async Task isInTheInboxAsync(Envelope envelope) => + (await _host.GetRuntime().Storage.Admin.AllIncomingAsync()).Any(x => x.Id == envelope.Id); + + [Fact] + public async Task a_scheduled_retry_moves_the_inbox_row_to_the_scheduled_table() + { + var envelope = await receivedIntoTheInboxAsync(); + + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } + + [Fact] + public async Task a_failed_scheduled_write_leaves_the_inbox_row_in_place() + { + var envelope = await receivedIntoTheInboxAsync(); + + // Too long for the scheduled table's VARCHAR2(500) message_type, so the write fails with ORA-12899 after the + // delete -- the test user may not create a trigger in the shared WOLVERINE schema + var messageType = envelope.MessageType; + envelope.MessageType = new string('x', 600); + + await Should.ThrowAsync(() => _queue.ScheduleRetryAsync(envelope, CancellationToken.None)); + + // Still in the inbox, so recovery runs it again if this node goes away + (await isInTheInboxAsync(envelope)).ShouldBeTrue(); + (await _queue.ScheduledCountAsync()).ShouldBe(0); + + // and the receiver's retry completes the move once the write can succeed + envelope.MessageType = messageType; + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } +} + +public record OracleScheduledRetryMessage(Guid Id); diff --git a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs index 2df07bfe5..d09c71c67 100644 --- a/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs +++ b/src/Persistence/Oracle/Wolverine.Oracle/Transport/OracleQueueSender.cs @@ -62,13 +62,19 @@ public async Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancel await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken); try { - // Delete from incoming, write to scheduled + // Delete from incoming, write to scheduled -- in one transaction. Committed one by one, a failed + // scheduled write left the inbox row deleted with nothing scheduled + await using var tx = (OracleTransaction)await conn.BeginTransactionAsync(cancellationToken); + await using var deleteCmd = conn.CreateCommand( $"DELETE FROM {_queue.Parent.MessageStorageSchemaName}.{DatabaseConstants.IncomingTable} WHERE id = :id"); + deleteCmd.Transaction = tx; deleteCmd.With("id", envelope.Id); await deleteCmd.ExecuteNonQueryAsync(cancellationToken); - await writeToScheduledTableAsync(envelope, cancellationToken, conn); + await writeToScheduledTableAsync(envelope, cancellationToken, conn, tx); + + await tx.CommitAsync(cancellationToken); } finally { diff --git a/src/Persistence/SqlServerTests/Transport/scheduled_retry_is_atomic.cs b/src/Persistence/SqlServerTests/Transport/scheduled_retry_is_atomic.cs new file mode 100644 index 000000000..a1996cb0e --- /dev/null +++ b/src/Persistence/SqlServerTests/Transport/scheduled_retry_is_atomic.cs @@ -0,0 +1,123 @@ +using IntegrationTests; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Weasel.SqlServer; +using Wolverine; +using Wolverine.Configuration; +using Wolverine.SqlServer; +using Wolverine.SqlServer.Transport; +using Wolverine.Tracking; + +namespace SqlServerTests.Transport; + +/// +/// A scheduled retry of a message received on a SQL Server queue deletes its inbox row and writes it to the +/// queue's scheduled table. Those were two autocommitted statements, so a scheduled write that failed -- a command +/// timeout or a deadlock, found by the durability chaos rig -- left the message in neither table, alive only in +/// the receiver's in-memory retry, and a node that died or shut down before that retry succeeded lost it. +/// +public class scheduled_retry_is_atomic : IAsyncLifetime +{ + private const string SchemaName = "schedretry"; + private const string QueueName = "schedretry"; + + private IHost _host = null!; + private SqlServerQueue _queue = null!; + + public async ValueTask InitializeAsync() + { + await using (var conn = new SqlConnection(Servers.SqlServerConnectionString)) + { + await conn.OpenAsync(); + await conn.DropSchemaAsync(SchemaName); + } + + _host = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Discovery.DisableConventionalDiscovery(); + opts.Durability.Mode = DurabilityMode.Solo; + opts.UseSqlServerPersistenceAndTransport(Servers.SqlServerConnectionString, SchemaName, + transportSchema: SchemaName).AutoProvision(); + + // Declared without a listener, so nothing but the test touches the queue + opts.PublishMessage().ToSqlServerQueue(QueueName); + }).StartAsync(); + + _queue = _host.GetRuntime().Options.Transports.OfType().Single().Queues[QueueName]; + _queue.Mode = EndpointMode.Durable; + await _queue.PurgeAsync(NullLogger.Instance); + await _host.GetRuntime().Storage.Admin.ClearAllAsync(); + } + + public async ValueTask DisposeAsync() + { + await _host.StopAsync(); + _host.Dispose(); + } + + /// + /// What a durable listener has done by the time the handler fails: the envelope sits in the inbox + /// + private async Task receivedIntoTheInboxAsync() + { + var runtime = _host.GetRuntime(); + var envelope = runtime.RoutingFor(typeof(ScheduledRetryMessage)) + .RouteForSend(new ScheduledRetryMessage(Guid.NewGuid()), null).Single(); + envelope.Destination = _queue.Uri; + await runtime.Storage.Inbox.StoreIncomingAsync(envelope); + + envelope.ScheduledTime = DateTimeOffset.UtcNow.AddHours(1); + return envelope; + } + + private async Task isInTheInboxAsync(Envelope envelope) => + (await _host.GetRuntime().Storage.Admin.AllIncomingAsync()).Any(x => x.Id == envelope.Id); + + private async Task executeAsync(string sql) + { + await using var conn = new SqlConnection(Servers.SqlServerConnectionString); + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(); + } + + [Fact] + public async Task a_scheduled_retry_moves_the_inbox_row_to_the_scheduled_table() + { + var envelope = await receivedIntoTheInboxAsync(); + + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } + + [Fact] + public async Task a_failed_scheduled_write_leaves_the_inbox_row_in_place() + { + var envelope = await receivedIntoTheInboxAsync(); + + await executeAsync( + $"create trigger {SchemaName}.fail_scheduled_writes on {_queue.ScheduledTable.Identifier} after insert, update " + + "as throw 50000, 'the scheduled write fails', 1;"); + + await Should.ThrowAsync(() => _queue.ScheduleRetryAsync(envelope, CancellationToken.None)); + + // Still in the inbox, so recovery runs it again if this node goes away + (await isInTheInboxAsync(envelope)).ShouldBeTrue(); + (await _queue.ScheduledCountAsync()).ShouldBe(0); + + // and the receiver's retry completes the move once the write can succeed + await executeAsync($"drop trigger {SchemaName}.fail_scheduled_writes"); + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } +} + +public record ScheduledRetryMessage(Guid Id); diff --git a/src/Persistence/SqliteTests/Transport/scheduled_retry_is_atomic.cs b/src/Persistence/SqliteTests/Transport/scheduled_retry_is_atomic.cs new file mode 100644 index 000000000..a38347aae --- /dev/null +++ b/src/Persistence/SqliteTests/Transport/scheduled_retry_is_atomic.cs @@ -0,0 +1,115 @@ +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; +using Wolverine; +using Wolverine.Configuration; +using Wolverine.Sqlite; +using Wolverine.Sqlite.Transport; +using Wolverine.Tracking; + +namespace SqliteTests.Transport; + +/// +/// A scheduled retry of a message received on a SQLite queue deletes its inbox row and writes it to the +/// queue's scheduled table. Those were two autocommitted statements in one command, so a scheduled write that failed -- a command +/// timeout or a deadlock, found by the durability chaos rig -- left the message in neither table, alive only in +/// the receiver's in-memory retry, and a node that died or shut down before that retry succeeded lost it. +/// +public class scheduled_retry_is_atomic : IAsyncLifetime +{ + private const string QueueName = "schedretry"; + + private readonly SqliteTestDatabase _database = Servers.CreateDatabase(nameof(scheduled_retry_is_atomic)); + private IHost _host = null!; + private SqliteQueue _queue = null!; + + public async ValueTask InitializeAsync() + { + _host = await Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Discovery.DisableConventionalDiscovery(); + opts.Durability.Mode = DurabilityMode.Solo; + opts.UseSqlitePersistenceAndTransport(_database.ConnectionString).AutoProvision(); + + // Declared without a listener, so nothing but the test touches the queue + opts.PublishMessage().ToSqliteQueue(QueueName); + }).StartAsync(); + + _queue = _host.GetRuntime().Options.Transports.GetOrCreate().Queues[QueueName]; + _queue.Mode = EndpointMode.Durable; + await _queue.PurgeAsync(NullLogger.Instance); + await _host.GetRuntime().Storage.Admin.ClearAllAsync(); + } + + public async ValueTask DisposeAsync() + { + await _host.StopAsync(); + _host.Dispose(); + _database.Dispose(); + } + + /// + /// What a durable listener has done by the time the handler fails: the envelope sits in the inbox + /// + private async Task receivedIntoTheInboxAsync() + { + var runtime = _host.GetRuntime(); + var envelope = runtime.RoutingFor(typeof(SqliteScheduledRetryMessage)) + .RouteForSend(new SqliteScheduledRetryMessage(Guid.NewGuid()), null).Single(); + envelope.Destination = _queue.Uri; + await runtime.Storage.Inbox.StoreIncomingAsync(envelope); + + envelope.ScheduledTime = DateTimeOffset.UtcNow.AddHours(1); + return envelope; + } + + private async Task isInTheInboxAsync(Envelope envelope) => + (await _host.GetRuntime().Storage.Admin.AllIncomingAsync()).Any(x => x.Id == envelope.Id); + + private async Task executeAsync(string sql) + { + await using var conn = new SqliteConnection(_database.ConnectionString); + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(); + } + + [Fact] + public async Task a_scheduled_retry_moves_the_inbox_row_to_the_scheduled_table() + { + var envelope = await receivedIntoTheInboxAsync(); + + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } + + [Fact] + public async Task a_failed_scheduled_write_leaves_the_inbox_row_in_place() + { + var envelope = await receivedIntoTheInboxAsync(); + + await executeAsync( + $"CREATE TRIGGER fail_scheduled_writes BEFORE INSERT ON {_queue.ScheduledTable.Identifier} " + + "BEGIN SELECT RAISE(ABORT, 'the scheduled write fails'); END;"); + + await Should.ThrowAsync(() => _queue.ScheduleRetryAsync(envelope, CancellationToken.None)); + + // Still in the inbox, so recovery runs it again if this node goes away + (await isInTheInboxAsync(envelope)).ShouldBeTrue(); + (await _queue.ScheduledCountAsync()).ShouldBe(0); + + // and the receiver's retry completes the move once the write can succeed + await executeAsync("DROP TRIGGER fail_scheduled_writes"); + await _queue.ScheduleRetryAsync(envelope, CancellationToken.None); + + (await isInTheInboxAsync(envelope)).ShouldBeFalse(); + (await _queue.ScheduledCountAsync()).ShouldBe(1); + } +} + +public record SqliteScheduledRetryMessage(Guid Id); diff --git a/src/Persistence/Wolverine.SqlServer/Transport/SqlServerQueueSender.cs b/src/Persistence/Wolverine.SqlServer/Transport/SqlServerQueueSender.cs index d0fe6f3e9..74a43ba32 100644 --- a/src/Persistence/Wolverine.SqlServer/Transport/SqlServerQueueSender.cs +++ b/src/Persistence/Wolverine.SqlServer/Transport/SqlServerQueueSender.cs @@ -120,13 +120,21 @@ public async Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancel await conn.OpenAsync(cancellationToken); try { + // The inbox delete and the scheduled write commit together or not at all. Run as two autocommitted + // statements, a scheduled write that failed (a timeout, a deadlock) left the inbox row deleted with + // nothing scheduled, and the message lived on only in this node's memory until the retry block tried + // again -- a node that died or shut down first lost it. A client-side transaction, rather than + // BEGIN TRANSACTION in the batch, so a command timeout's attention rolls it back with the dispose. + await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(cancellationToken); await using var cmd = conn.CreateCommand(_deleteFromIncomingAndScheduleSql) .With("id", envelope.Id) .With("body", EnvelopeSerializer.Serialize(envelope)) .With("type", envelope.MessageType!) .With("expires", envelope.DeliverBy!) .With("time", envelope.ScheduledTime!); + cmd.Transaction = tx; await cmd.ExecuteNonQueryAsync(cancellationToken); + await tx.CommitAsync(cancellationToken); } finally { diff --git a/src/Persistence/Wolverine.Sqlite/Transport/SqliteQueueSender.cs b/src/Persistence/Wolverine.Sqlite/Transport/SqliteQueueSender.cs index 76bb2562e..e9a15d026 100644 --- a/src/Persistence/Wolverine.Sqlite/Transport/SqliteQueueSender.cs +++ b/src/Persistence/Wolverine.Sqlite/Transport/SqliteQueueSender.cs @@ -145,12 +145,17 @@ public async Task ScheduleRetryAsync(Envelope envelope, CancellationToken cancel { await using var conn = await _dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); - await conn.CreateCommand(_deleteFromIncomingAndScheduleSql) + // The inbox delete and the scheduled write commit together or not at all: autocommitted one by one, a + // failed scheduled write left the inbox row deleted with nothing scheduled + await using var tx = await conn.BeginTransactionAsync(cancellationToken); + var cmd = conn.CreateCommand(_deleteFromIncomingAndScheduleSql) .With("id", $"{envelope.Id:D}") .With("body", EnvelopeSerializer.Serialize(envelope)) .With("type", envelope.MessageType!) - .With("time", (envelope.ScheduledTime ?? DateTimeOffset.UtcNow).UtcDateTime.ToString(SqliteDateTimeFormat)) - .ExecuteNonQueryAsync(cancellationToken); + .With("time", (envelope.ScheduledTime ?? DateTimeOffset.UtcNow).UtcDateTime.ToString(SqliteDateTimeFormat)); + cmd.Transaction = tx; + await cmd.ExecuteNonQueryAsync(cancellationToken); + await tx.CommitAsync(cancellationToken); } public async ValueTask SendAsync(Envelope envelope)