From 7542fbfcbef3fd4fcd9eaa9c6fc1c9d9897b0447 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:07:43 -0600 Subject: [PATCH] Honor `Timeout` on `StaFact`/`StaTheory` and other UI test attributes (#237) * Initial plan * Honor Timeout on UI/Sta facts and theories Co-authored-by: AArnott <3548+AArnott@users.noreply.github.com> * Cancel timeout timer when test completes; document timeout semantics Co-authored-by: AArnott <3548+AArnott@users.noreply.github.com> * Refresh CI after retargeting #237 to v4.0 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Enforce UI test timeouts across the full lifecycle Keep timeout reporting independent of blocked UI threads and add runner-level regression tests for lifecycle, fact, and theory timeout failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: AArnott <3548+AArnott@users.noreply.github.com> Co-authored-by: Andrew Arnott Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docfx/docs/features.md | 6 + src/Xunit.StaFact/Sdk/UITestCaseRunner.cs | 9 +- src/Xunit.StaFact/Sdk/UITestRunner.cs | 65 +++++ test/Xunit.StaFact.Tests/TimeoutTests.cs | 296 ++++++++++++++++++++++ test/Xunit.StaFact.Tests/UIFactTests.cs | 28 ++ test/Xunit.StaFact.Tests/UITheoryTests.cs | 33 +++ 6 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 test/Xunit.StaFact.Tests/TimeoutTests.cs diff --git a/docfx/docs/features.md b/docfx/docs/features.md index 728810de..36a4516b 100644 --- a/docfx/docs/features.md +++ b/docfx/docs/features.md @@ -13,6 +13,12 @@ Xunit test attributes | Supported OS's | SynchronizationContext We also offer a @Xunit.UISettingsAttribute that can be applied to individual test methods or test classes to control the behavior of the various UI test attributes. This attribute offers a means to add automated retries to a test's execution for unstable tests. +UI fact and theory attributes support a per-test timeout in milliseconds, +covering construction, initialization, invocation, and disposal. A timeout fails the test and +cancels its test cancellation token without waiting for the UI thread to finish. Code that does +not observe cancellation may continue running after the failure is reported. Timeouts are +disabled while a debugger is attached. + [^1]: This is a private @System.Threading.SynchronizationContext that works cross-platform and effectively keeps code running on the test's starting thread the way a GUI application's main thread would do. diff --git a/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs b/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs index f8c09771..14c3bca2 100644 --- a/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs @@ -1,6 +1,7 @@ // Copyright (c) Andrew Arnott. All rights reserved. // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. +using System.Diagnostics; using System.Text; namespace Xunit.Sdk; @@ -106,7 +107,13 @@ internal static ValueTask Run( async () => { using ThreadRental threadRental = await ThreadRental.CreateAsync(adapter, testCase.TestMethod); - await threadRental.SynchronizationContext; + + // Keep timeout reporting independent of a blocked UI thread. + if (testCase.Timeout <= 0 || Debugger.IsAttached) + { + await threadRental.SynchronizationContext; + } + var runner = new UITestCaseRunner(settings, threadRental); return await runner.Run( testCase, diff --git a/src/Xunit.StaFact/Sdk/UITestRunner.cs b/src/Xunit.StaFact/Sdk/UITestRunner.cs index 00317ea6..60d8d09d 100644 --- a/src/Xunit.StaFact/Sdk/UITestRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITestRunner.cs @@ -1,6 +1,8 @@ // Copyright (c) Andrew Arnott. All rights reserved. // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. +using System.Diagnostics; + namespace Xunit.Sdk; public class UITestRunner : XunitTestRunnerBase @@ -72,6 +74,69 @@ protected async override ValueTask RunTest(UITestRunnerContext ctxt) uiSyncContext.SetExceptionAggregator(ctxt.Aggregator); } + int timeout = ctxt.Test.Timeout; + if (ctxt.Aggregator.HasExceptions || timeout <= 0 || Debugger.IsAttached) + { + return await this.RunTestLifecycle(ctxt); + } + + Stopwatch stopwatch = Stopwatch.StartNew(); + TaskCompletionSource finished = new(TaskCreationOptions.RunContinuationsAsynchronously); + ExecutionContext? executionContext = ExecutionContext.Capture(); + ctxt.ThreadRental.SynchronizationContext.Post( + _ => + { + if (executionContext is not null) + { + ExecutionContext.Run(executionContext, RunLifecycle, null); + } + else + { + RunLifecycle(null); + } + }, + null); + + using CancellationTokenSource delayCancellation = new(); + Task completedTask = await Task.WhenAny(finished.Task, Task.Delay(timeout, delayCancellation.Token)).ConfigureAwait(false); + delayCancellation.Cancel(); + if (completedTask == finished.Task) + { + return await finished.Task.ConfigureAwait(false); + } + + // Like xunit, report the timeout without waiting for the lifecycle to finish. + // The UI thread continues until the test completes or observes cancellation. + try + { + TestTimeoutException timeoutException = TestTimeoutException.ForTimedOutTest(timeout); + ctxt.Aggregator.Add(timeoutException); + this.UpdateTestContext(null, TestResultState.FromException((decimal)stopwatch.Elapsed.TotalSeconds, timeoutException)); + } + finally + { + TestContext.Current.CancelCurrentTest(); + } + + return stopwatch.Elapsed; + + async void RunLifecycle(object? state) + { + try + { + SynchronizationContext.SetSynchronizationContext( + ctxt.ThreadRental.SyncContextAdapter.ShouldSetAsCurrent ? ctxt.ThreadRental.SynchronizationContext : null); + finished.TrySetResult(await this.RunTestLifecycle(ctxt)); + } + catch (Exception ex) + { + finished.TrySetException(ex); + } + } + } + + private async ValueTask RunTestLifecycle(UITestRunnerContext ctxt) + { object? testClassInstance = null; TimeSpan elapsedTime = TimeSpan.Zero; diff --git a/test/Xunit.StaFact.Tests/TimeoutTests.cs b/test/Xunit.StaFact.Tests/TimeoutTests.cs new file mode 100644 index 00000000..d7e3db88 --- /dev/null +++ b/test/Xunit.StaFact.Tests/TimeoutTests.cs @@ -0,0 +1,296 @@ +// Copyright (c) Andrew Arnott. All rights reserved. +// Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; +using System.Reflection; +using Xunit.Runner.Common; +using Xunit.Sdk; +using Xunit.v3; + +/// +/// Verifies timeout failures emitted by the UI test runner. +/// +public class TimeoutTests +{ + private const int TimeoutMilliseconds = 500; + + /// + /// Verifies that a blocked lifecycle fails before the blocked operation is released. + /// + /// The fixture method to execute. + /// The lifecycle stage to block. + [Theory] + [InlineData(nameof(TimeoutFixture.FactSync), "Constructor")] + [InlineData(nameof(TimeoutFixture.FactSync), "InitializeSync")] + [InlineData(nameof(TimeoutFixture.FactSync), "InitializeAsync")] + [InlineData(nameof(TimeoutFixture.FactSync), "DisposeSync")] + [InlineData(nameof(TimeoutFixture.FactSync), "DisposeAsync")] + [InlineData(nameof(TimeoutFixture.FactSync), "Test")] + [InlineData(nameof(TimeoutFixture.FactAsync), "Test")] + [InlineData(nameof(TimeoutFixture.TheorySync), "Test")] + [InlineData(nameof(TimeoutFixture.TheoryAsync), "Test")] + [InlineData(nameof(TimeoutFixture.DelayedTheory), "Test")] + public async Task Timeout_ReportsFailure(string methodName, string stage) + { + MethodInfo method = typeof(TimeoutFixture).GetMethod(methodName)!; + XunitTestAssembly assembly = new(typeof(TimeoutFixture).Assembly, configFilePath: null, assemblyName: null); + XunitTestCollection collection = new(assembly, null, disableParallelization: true, displayName: nameof(TimeoutTests)); + XunitTestClass testClass = new(typeof(TimeoutFixture), collection); + XunitTestMethod testMethod = new(testClass, method, []); + IFactAttribute attribute = Assert.Single(testMethod.FactAttributes); + ITestFrameworkDiscoveryOptions options = TestFrameworkOptions.ForDiscovery(new TestAssemblyConfiguration()); + options.SetPreEnumerateTheories(true); + IXunitTestCase testCase = Assert.Single(await (attribute is ITheoryAttribute + ? new UITheoryDiscoverer().Discover(options, testMethod, attribute) + : new UIFactDiscoverer().Discover(options, testMethod, attribute))); + + using BlockingState state = new(stage); + using RecordingMessageBus messageBus = new(); + using CancellationTokenSource cancellation = new(); + await using ExecutionScheduler scheduler = ExecutionScheduler.CreateUnlimited(); + await using FixtureMappingManager fixtures = new("Method"); + Task run = Task.Run( + async () => await Assert.IsAssignableFrom(testCase).Run( + ExplicitOption.On, + messageBus, + [state], + new ExceptionAggregator(), + cancellation, + ParallelMode.None, + scheduler, + fixtures), + TestContext.Current.CancellationToken); + + try + { + Task completed = await Task.WhenAny(run, Task.Delay(10_000, TestContext.Current.CancellationToken)); + Assert.Same(run, completed); + RunSummary summary = await run; + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + ITestFailed failure = Assert.Single(messageBus.Failures); + Assert.Equal(typeof(TestTimeoutException).FullName, Assert.Single(failure.ExceptionTypes)); + Assert.Equal($"Test execution timed out after {TimeoutMilliseconds} milliseconds", Assert.Single(failure.Messages)); + Assert.True(state.Entered); + Assert.True(state.CancellationToken.IsCancellationRequested); + Assert.False(TestContext.Current.CancellationToken.IsCancellationRequested); + } + finally + { + state.Release(); + if (state.Entered) + { + Assert.Same(state.Unblocked, await Task.WhenAny(state.Unblocked, Task.Delay(10_000, TestContext.Current.CancellationToken))); + } + + await run; + } + } + + /// + /// An explicitly run fixture with independently blockable lifecycle stages. + /// + public sealed class TimeoutFixture : IAsyncLifetime + { + private readonly BlockingState state; + + /// + /// Initializes a new instance of the class. + /// + /// The state controlling the blocked stage. + public TimeoutFixture(BlockingState state) + { + this.state = state; + if (state.Stage == "Constructor") + { + state.Block(); + } + } + + /// + public async ValueTask InitializeAsync() + { + if (this.state.Stage == "InitializeSync") + { + this.state.Block(); + } + else if (this.state.Stage == "InitializeAsync") + { + await this.state.BlockAsync(); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (this.state.Stage == "DisposeSync") + { + this.state.Block(); + } + else if (this.state.Stage == "DisposeAsync") + { + await this.state.BlockAsync(); + } + } + +#pragma warning disable xUnit1069 // Cancellation is recorded by BlockingState but deliberately not honored. + /// + /// Blocks a synchronous fact when invocation is the selected stage. + /// + [UIFact(Explicit = true, Timeout = TimeoutMilliseconds)] + public void FactSync() + { + if (this.state.Stage == "Test") + { + this.state.Block(); + } + } + + /// + /// Blocks an asynchronous fact. + /// + [UIFact(Explicit = true, Timeout = TimeoutMilliseconds)] + public async Task FactAsync() + { + await this.state.BlockAsync(); + } + + /// + /// Blocks a pre-enumerated synchronous theory row. + /// + /// The discovered row argument. + [UITheory(Explicit = true, Timeout = TimeoutMilliseconds)] + [InlineData(0)] + public void TheorySync(int arg) + { + Assert.Equal(0, arg); + this.state.Block(); + } + + /// + /// Blocks a pre-enumerated asynchronous theory row. + /// + /// The discovered row argument. + [UITheory(Explicit = true, Timeout = TimeoutMilliseconds)] + [InlineData(0)] + public async Task TheoryAsync(int arg) + { + Assert.Equal(0, arg); + await this.state.BlockAsync(); + } + + /// + /// Blocks a theory row enumerated at execution time. + /// + /// The discovered row argument. + [UITheory(Explicit = true, Timeout = TimeoutMilliseconds, DisableDiscoveryEnumeration = true)] + [InlineData(0)] + public async Task DelayedTheory(int arg) + { + Assert.Equal(0, arg); + await this.state.BlockAsync(); + } +#pragma warning restore xUnit1069 + } + + /// + /// Controls a blocked operation without relying on sleeps or unconditional failures. + /// + public sealed class BlockingState : IDisposable + { + private readonly ManualResetEventSlim release = new(); + private readonly TaskCompletionSource releaseAsync = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource unblocked = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Initializes a new instance of the class. + /// + /// The lifecycle stage to block. + public BlockingState(string stage) + { + this.Stage = stage; + } + + /// + /// Gets the stage to block. + /// + public string Stage { get; } + + /// + /// Gets a value indicating whether the selected stage was entered. + /// + public bool Entered { get; private set; } + + /// + /// Gets the cancellation token of the nested test. + /// + public CancellationToken CancellationToken { get; private set; } + + /// + /// Gets a task that completes when the blocked operation exits. + /// + public Task Unblocked => this.unblocked.Task; + + /// + /// Blocks the UI thread until the test releases it. + /// + public void Block() + { + this.CancellationToken = TestContext.Current.CancellationToken; + this.Entered = true; + this.release.Wait(); + this.unblocked.TrySetResult(null); + } + + /// + /// Asynchronously blocks until the test releases it. + /// + /// A task that completes when released. + public async Task BlockAsync() + { + this.CancellationToken = TestContext.Current.CancellationToken; + this.Entered = true; + await this.releaseAsync.Task.ConfigureAwait(false); + this.unblocked.TrySetResult(null); + } + + /// + /// Releases either kind of blocked operation. + /// + public void Release() + { + this.release.Set(); + this.releaseAsync.TrySetResult(null); + } + + /// + public void Dispose() + { + this.release.Dispose(); + } + } + + private sealed class RecordingMessageBus : IMessageBus + { + /// + /// Gets the failure messages emitted by the nested runner. + /// + internal ConcurrentQueue Failures { get; } = new(); + + /// + public bool QueueMessage(IMessageSinkMessage message) + { + if (message is ITestFailed failure) + { + this.Failures.Enqueue(failure); + } + + return true; + } + + /// + public void Dispose() + { + } + } +} diff --git a/test/Xunit.StaFact.Tests/UIFactTests.cs b/test/Xunit.StaFact.Tests/UIFactTests.cs index 24a09e2a..d3fcd527 100644 --- a/test/Xunit.StaFact.Tests/UIFactTests.cs +++ b/test/Xunit.StaFact.Tests/UIFactTests.cs @@ -69,6 +69,34 @@ public async Task PassAfterDelay() await Task.Delay(10); } + [DesktopFact(Timeout = 30_000)] + public async Task Timeout_NotExceeded() + { + Assert.Equal(30_000, Assert.IsAssignableFrom(TestContext.Current.TestCase).Timeout); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + await Task.Yield(); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + TestContext.Current.CancellationToken.ThrowIfCancellationRequested(); + } + +#pragma warning disable xUnit1069 // Deliberately ignore the CancellationToken to simulate a blocked UI thread. + [DesktopFact(Timeout = 100), Trait("TestCategory", "FailureExpected")] + public void Timeout_Exceeded_Sync() + { + // Deliberately block the UI thread without honoring the CancellationToken. + Thread.Sleep(2000); + Assert.Fail("The test should have timed out."); + } +#pragma warning restore xUnit1069 + + [DesktopFact(Timeout = 100), Trait("TestCategory", "FailureExpected")] + public async Task Timeout_Exceeded_Async() + { + await Task.Delay(2000, TestContext.Current.CancellationToken); + } + [DesktopFact, Trait("TestCategory", "FailureExpected")] public async Task FailAfterYield() { diff --git a/test/Xunit.StaFact.Tests/UITheoryTests.cs b/test/Xunit.StaFact.Tests/UITheoryTests.cs index c4251fdc..098a3551 100644 --- a/test/Xunit.StaFact.Tests/UITheoryTests.cs +++ b/test/Xunit.StaFact.Tests/UITheoryTests.cs @@ -83,6 +83,39 @@ public async Task PassAfterDelay(int arg) Assert.Equal(0, arg); } + [DesktopTheory(Timeout = 30_000)] + [InlineData(0)] + public async Task Timeout_NotExceeded(int arg) + { + Assert.Equal(30_000, Assert.IsAssignableFrom(TestContext.Current.TestCase).Timeout); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + await Task.Yield(); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + TestContext.Current.CancellationToken.ThrowIfCancellationRequested(); + Assert.Equal(0, arg); + } + +#pragma warning disable xUnit1069 // Deliberately ignore the CancellationToken to simulate a blocked UI thread. + [DesktopTheory(Timeout = 100), Trait("TestCategory", "FailureExpected")] + [InlineData(0)] + public void Timeout_Exceeded_Sync(int arg) + { + // Deliberately block the UI thread without honoring the CancellationToken. + Thread.Sleep(2000); + Assert.Fail($"The test should have timed out. Arg: {arg}"); + } +#pragma warning restore xUnit1069 + + [DesktopTheory(Timeout = 100), Trait("TestCategory", "FailureExpected")] + [InlineData(0)] + public async Task Timeout_Exceeded_Async(int arg) + { + await Task.Delay(2000, TestContext.Current.CancellationToken); + Assert.Equal(0, arg); + } + [DesktopTheory, Trait("TestCategory", "FailureExpected")] [InlineData(0)] public async Task FailAfterYield(int arg)