diff --git a/docfx/docs/features.md b/docfx/docs/features.md index 7690da97..23c9f2ff 100644 --- a/docfx/docs/features.md +++ b/docfx/docs/features.md @@ -98,6 +98,12 @@ The STA fixture intentionally does not install one, matching @Xunit.StaFactAttri - Do not dispose a fixture directly when xUnit owns it. xUnit disposes it at the end of its class or collection lifetime. - Put only state that is intentionally shared into the fixture. Test class instances are still created separately for each test. +UI fact and theory attributes support a per-test timeout in milliseconds, +covering construction, initialization, invocation, and disposal. A timeout fails the test and +cancels its test cancellation token without waiting for the UI thread to finish. Code that does +not observe cancellation may continue running after the failure is reported. Timeouts are +disabled while a debugger is attached. + [^1]: This is a private @System.Threading.SynchronizationContext that works cross-platform and effectively keeps code running on the test's starting thread the way a GUI application's main thread would do. diff --git a/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs b/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs index c28af947..a5514d69 100644 --- a/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITestCaseRunner.cs @@ -1,6 +1,7 @@ // Copyright (c) Andrew Arnott. All rights reserved. // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. +using System.Diagnostics; using System.Globalization; using System.Text; @@ -112,7 +113,13 @@ internal static ValueTask Run( { ThreadRental threadRental = sharedThreadFixture?.ThreadRental ?? (ownedThreadRental = await ThreadRental.CreateAsync(adapter, testCase.TestMethod)); - await threadRental.SynchronizationContext; + + // Keep timeout reporting independent of a blocked UI thread. + if (testCase.Timeout <= 0 || Debugger.IsAttached) + { + await threadRental.SynchronizationContext; + } + var runner = new UITestCaseRunner(settings, threadRental); return await runner.Run( testCase, @@ -129,7 +136,42 @@ internal static ValueTask Run( } finally { - ownedThreadRental?.Dispose(); + if (ownedThreadRental is not null) + { + if (testCase.Timeout > 0 && !Debugger.IsAttached) + { + // Cleanup must run on the UI thread without delaying timeout reporting. + ownedThreadRental.SynchronizationContext.Post( + _ => + { + try + { + ownedThreadRental.Dispose(); + } + catch (Exception ex) + { + IXunitTestMethod testMethod = testCase.TestMethod; + IXunitTestClass testClass = testMethod.TestClass; + IXunitTestCollection testCollection = testClass.TestCollection; + if (!messageBus.QueueMessage(TestCaseCleanupFailure.FromException( + ex, + testCollection.TestAssembly.UniqueID, + testCollection.UniqueID, + testClass.UniqueID, + testMethod.UniqueID, + testCase.UniqueID))) + { + cancellationTokenSource.Cancel(); + } + } + }, + null); + } + else + { + ownedThreadRental.Dispose(); + } + } } }, cancellationTokenSource.Token); @@ -146,6 +188,11 @@ protected async override ValueTask RunTest(UITestCaseRunnerContext c throw new ArgumentNullException(nameof(ctxt)); } + if (test is null) + { + throw new ArgumentNullException(nameof(test)); + } + RunSummary result = default; for (int i = 0; i < this.settings.MaxAttempts; i++) { @@ -181,7 +228,12 @@ protected async override ValueTask RunTest(UITestCaseRunnerContext c private async ValueTask RunTestAttempt(UITestCaseRunnerContext ctxt, IXunitTest test, bool finalAttempt) { - await ctxt.ThreadRental.SynchronizationContext; + bool useUIThread = test.Timeout <= 0 || Debugger.IsAttached; + if (useUIThread) + { + await ctxt.ThreadRental.SynchronizationContext; + } + CultureInfo? originalCulture = null; CultureInfo? originalUICulture = null; try @@ -213,7 +265,11 @@ private async ValueTask RunTestAttempt(UITestCaseRunnerContext ctxt, { if (originalCulture is not null && originalUICulture is not null) { - await ctxt.ThreadRental.SynchronizationContext; + if (useUIThread) + { + await ctxt.ThreadRental.SynchronizationContext; + } + CultureInfo.CurrentCulture = originalCulture; CultureInfo.CurrentUICulture = originalUICulture; } diff --git a/src/Xunit.StaFact/Sdk/UITestRunner.cs b/src/Xunit.StaFact/Sdk/UITestRunner.cs index 00317ea6..60d8d09d 100644 --- a/src/Xunit.StaFact/Sdk/UITestRunner.cs +++ b/src/Xunit.StaFact/Sdk/UITestRunner.cs @@ -1,6 +1,8 @@ // Copyright (c) Andrew Arnott. All rights reserved. // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. +using System.Diagnostics; + namespace Xunit.Sdk; public class UITestRunner : XunitTestRunnerBase @@ -72,6 +74,69 @@ protected async override ValueTask RunTest(UITestRunnerContext ctxt) uiSyncContext.SetExceptionAggregator(ctxt.Aggregator); } + int timeout = ctxt.Test.Timeout; + if (ctxt.Aggregator.HasExceptions || timeout <= 0 || Debugger.IsAttached) + { + return await this.RunTestLifecycle(ctxt); + } + + Stopwatch stopwatch = Stopwatch.StartNew(); + TaskCompletionSource finished = new(TaskCreationOptions.RunContinuationsAsynchronously); + ExecutionContext? executionContext = ExecutionContext.Capture(); + ctxt.ThreadRental.SynchronizationContext.Post( + _ => + { + if (executionContext is not null) + { + ExecutionContext.Run(executionContext, RunLifecycle, null); + } + else + { + RunLifecycle(null); + } + }, + null); + + using CancellationTokenSource delayCancellation = new(); + Task completedTask = await Task.WhenAny(finished.Task, Task.Delay(timeout, delayCancellation.Token)).ConfigureAwait(false); + delayCancellation.Cancel(); + if (completedTask == finished.Task) + { + return await finished.Task.ConfigureAwait(false); + } + + // Like xunit, report the timeout without waiting for the lifecycle to finish. + // The UI thread continues until the test completes or observes cancellation. + try + { + TestTimeoutException timeoutException = TestTimeoutException.ForTimedOutTest(timeout); + ctxt.Aggregator.Add(timeoutException); + this.UpdateTestContext(null, TestResultState.FromException((decimal)stopwatch.Elapsed.TotalSeconds, timeoutException)); + } + finally + { + TestContext.Current.CancelCurrentTest(); + } + + return stopwatch.Elapsed; + + async void RunLifecycle(object? state) + { + try + { + SynchronizationContext.SetSynchronizationContext( + ctxt.ThreadRental.SyncContextAdapter.ShouldSetAsCurrent ? ctxt.ThreadRental.SynchronizationContext : null); + finished.TrySetResult(await this.RunTestLifecycle(ctxt)); + } + catch (Exception ex) + { + finished.TrySetException(ex); + } + } + } + + private async ValueTask RunTestLifecycle(UITestRunnerContext ctxt) + { object? testClassInstance = null; TimeSpan elapsedTime = TimeSpan.Zero; diff --git a/src/Xunit.StaFact/Sdk/Utilities.cs b/src/Xunit.StaFact/Sdk/Utilities.cs index bc4bc8f3..502e0e44 100644 --- a/src/Xunit.StaFact/Sdk/Utilities.cs +++ b/src/Xunit.StaFact/Sdk/Utilities.cs @@ -46,7 +46,8 @@ internal static IReadOnlyCollection CreateTestCasesForFact( details.SkipWhen, GetCultureTraits(traits, culture), sourceFilePath: details.SourceFilePath, - sourceLineNumber: details.SourceLineNumber)); + sourceLineNumber: details.SourceLineNumber, + timeout: details.Timeout)); } internal static IReadOnlyCollection CreateTestCasesForDataRow( diff --git a/test/Xunit.StaFact.Tests/SharedUIThreadFixtureTests.cs b/test/Xunit.StaFact.Tests/SharedUIThreadFixtureTests.cs index 9b0db9a5..a0c8c3ed 100644 --- a/test/Xunit.StaFact.Tests/SharedUIThreadFixtureTests.cs +++ b/test/Xunit.StaFact.Tests/SharedUIThreadFixtureTests.cs @@ -12,9 +12,10 @@ public SharedUIThreadFixtureTests(TrackingUIThreadFixture fixture) this.fixture = fixture; } - [UIFact] + [UIFact(Timeout = 30_000)] public void FactUsesFixtureThread() { + TestContext.Current.CancellationToken.ThrowIfCancellationRequested(); Assert.True(this.fixture.Initialized); Assert.Equal(this.fixture.ThreadId, Environment.CurrentManagedThreadId); Assert.Same(this.fixture.Context, SynchronizationContext.Current); diff --git a/test/Xunit.StaFact.Tests/TimeoutTests.cs b/test/Xunit.StaFact.Tests/TimeoutTests.cs new file mode 100644 index 00000000..5cc018f4 --- /dev/null +++ b/test/Xunit.StaFact.Tests/TimeoutTests.cs @@ -0,0 +1,326 @@ +// Copyright (c) Andrew Arnott. All rights reserved. +// Licensed under the Ms-PL license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; +using System.Globalization; +using System.Reflection; +using Xunit.Runner.Common; +using Xunit.Sdk; +using Xunit.v3; + +/// +/// Verifies timeout failures emitted by the UI test runner. +/// +public class TimeoutTests +{ + private const int TimeoutMilliseconds = 500; + + /// + /// Verifies that a blocked lifecycle fails before the blocked operation is released. + /// + /// The fixture method to execute. + /// The lifecycle stage to block. + [Theory] + [InlineData(nameof(TimeoutFixture.FactSync), "Constructor")] + [InlineData(nameof(TimeoutFixture.FactSync), "InitializeSync")] + [InlineData(nameof(TimeoutFixture.FactSync), "InitializeAsync")] + [InlineData(nameof(TimeoutFixture.FactSync), "DisposeSync")] + [InlineData(nameof(TimeoutFixture.FactSync), "DisposeAsync")] + [InlineData(nameof(TimeoutFixture.FactSync), "Test")] + [InlineData(nameof(TimeoutFixture.FactAsync), "Test")] + [InlineData(nameof(TimeoutFixture.TheorySync), "Test")] + [InlineData(nameof(TimeoutFixture.TheoryAsync), "Test")] + [InlineData(nameof(TimeoutFixture.DelayedTheory), "Test")] + [InlineData(nameof(TimeoutFixture.FactSyncInCulture), "Constructor")] + [InlineData(nameof(TimeoutFixture.FactSyncInCulture), "Test")] + public async Task Timeout_ReportsFailure(string methodName, string stage) + { + MethodInfo method = typeof(TimeoutFixture).GetMethod(methodName)!; + XunitTestAssembly assembly = new(typeof(TimeoutFixture).Assembly, configFilePath: null, assemblyName: null); + XunitTestCollection collection = new(assembly, null, disableParallelization: true, displayName: nameof(TimeoutTests)); + XunitTestClass testClass = new(typeof(TimeoutFixture), collection); + XunitTestMethod testMethod = new(testClass, method, []); + IFactAttribute attribute = Assert.Single(testMethod.FactAttributes); + ITestFrameworkDiscoveryOptions options = TestFrameworkOptions.ForDiscovery(new TestAssemblyConfiguration()); + options.SetPreEnumerateTheories(true); + IXunitTestCase testCase = Assert.Single(await (attribute is ITheoryAttribute + ? new UITheoryDiscoverer().Discover(options, testMethod, attribute) + : new UIFactDiscoverer().Discover(options, testMethod, attribute))); + + using BlockingState state = new(stage); + using RecordingMessageBus messageBus = new(); + using CancellationTokenSource cancellation = new(); + await using ExecutionScheduler scheduler = ExecutionScheduler.CreateUnlimited(); + await using FixtureMappingManager fixtures = new("Method"); + Task run = Task.Run( + async () => await Assert.IsAssignableFrom(testCase).Run( + ExplicitOption.On, + messageBus, + [state], + new ExceptionAggregator(), + cancellation, + ParallelMode.None, + scheduler, + fixtures), + TestContext.Current.CancellationToken); + + try + { + Task completed = await Task.WhenAny(run, Task.Delay(10_000, TestContext.Current.CancellationToken)); + Assert.Same(run, completed); + RunSummary summary = await run; + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + ITestFailed failure = Assert.Single(messageBus.Failures); + Assert.Equal(typeof(TestTimeoutException).FullName, Assert.Single(failure.ExceptionTypes)); + Assert.Equal($"Test execution timed out after {TimeoutMilliseconds} milliseconds", Assert.Single(failure.Messages)); + Assert.True(state.Entered); + Assert.True(state.CancellationToken.IsCancellationRequested); + Assert.False(TestContext.Current.CancellationToken.IsCancellationRequested); + if (methodName == nameof(TimeoutFixture.FactSyncInCulture)) + { + Assert.Equal("fr-FR", state.Culture); + Assert.Equal("fr-FR", state.UICulture); + } + } + finally + { + state.Release(); + if (state.Entered) + { + Assert.Same(state.Unblocked, await Task.WhenAny(state.Unblocked, Task.Delay(10_000, TestContext.Current.CancellationToken))); + } + + await run; + } + } + + /// + /// An explicitly run fixture with independently blockable lifecycle stages. + /// + public sealed class TimeoutFixture : IAsyncLifetime + { + private readonly BlockingState state; + + /// + /// Initializes a new instance of the class. + /// + /// The state controlling the blocked stage. + public TimeoutFixture(BlockingState state) + { + this.state = state; + if (state.Stage == "Constructor") + { + state.Block(); + } + } + + /// + public async ValueTask InitializeAsync() + { + if (this.state.Stage == "InitializeSync") + { + this.state.Block(); + } + else if (this.state.Stage == "InitializeAsync") + { + await this.state.BlockAsync(); + } + } + + /// + public async ValueTask DisposeAsync() + { + if (this.state.Stage == "DisposeSync") + { + this.state.Block(); + } + else if (this.state.Stage == "DisposeAsync") + { + await this.state.BlockAsync(); + } + } + +#pragma warning disable xUnit1069 // Cancellation is recorded by BlockingState but deliberately not honored. + /// + /// Blocks a synchronous fact when invocation is the selected stage. + /// + [UIFact(Explicit = true, Timeout = TimeoutMilliseconds)] + public void FactSync() + { + if (this.state.Stage == "Test") + { + this.state.Block(); + } + } + + /// + /// Blocks a synchronous fact under a configured culture. + /// + [UIFact(Explicit = true, Timeout = TimeoutMilliseconds)] + [UISettings(Cultures = new[] { "fr-FR" })] + public void FactSyncInCulture() + { + this.FactSync(); + } + + /// + /// Blocks an asynchronous fact. + /// + [UIFact(Explicit = true, Timeout = TimeoutMilliseconds)] + public async Task FactAsync() + { + await this.state.BlockAsync(); + } + + /// + /// Blocks a pre-enumerated synchronous theory row. + /// + /// The discovered row argument. + [UITheory(Explicit = true, Timeout = TimeoutMilliseconds)] + [InlineData(0)] + public void TheorySync(int arg) + { + Assert.Equal(0, arg); + this.state.Block(); + } + + /// + /// Blocks a pre-enumerated asynchronous theory row. + /// + /// The discovered row argument. + [UITheory(Explicit = true, Timeout = TimeoutMilliseconds)] + [InlineData(0)] + public async Task TheoryAsync(int arg) + { + Assert.Equal(0, arg); + await this.state.BlockAsync(); + } + + /// + /// Blocks a theory row enumerated at execution time. + /// + /// The discovered row argument. + [UITheory(Explicit = true, Timeout = TimeoutMilliseconds, DisableDiscoveryEnumeration = true)] + [InlineData(0)] + public async Task DelayedTheory(int arg) + { + Assert.Equal(0, arg); + await this.state.BlockAsync(); + } +#pragma warning restore xUnit1069 + } + + /// + /// Controls a blocked operation without relying on sleeps or unconditional failures. + /// + public sealed class BlockingState : IDisposable + { + private readonly ManualResetEventSlim release = new(); + private readonly TaskCompletionSource releaseAsync = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource unblocked = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Initializes a new instance of the class. + /// + /// The lifecycle stage to block. + public BlockingState(string stage) + { + this.Stage = stage; + } + + /// + /// Gets the stage to block. + /// + public string Stage { get; } + + /// + /// Gets a value indicating whether the selected stage was entered. + /// + public bool Entered { get; private set; } + + /// + /// Gets the cancellation token of the nested test. + /// + public CancellationToken CancellationToken { get; private set; } + + /// + /// Gets the culture of the blocked operation. + /// + public string? Culture { get; private set; } + + /// + /// Gets the UI culture of the blocked operation. + /// + public string? UICulture { get; private set; } + + /// + /// Gets a task that completes when the blocked operation exits. + /// + public Task Unblocked => this.unblocked.Task; + + /// + /// Blocks the UI thread until the test releases it. + /// + public void Block() + { + this.CancellationToken = TestContext.Current.CancellationToken; + this.Culture = CultureInfo.CurrentCulture.Name; + this.UICulture = CultureInfo.CurrentUICulture.Name; + this.Entered = true; + this.release.Wait(); + this.unblocked.TrySetResult(null); + } + + /// + /// Asynchronously blocks until the test releases it. + /// + /// A task that completes when released. + public async Task BlockAsync() + { + this.CancellationToken = TestContext.Current.CancellationToken; + this.Entered = true; + await this.releaseAsync.Task.ConfigureAwait(false); + this.unblocked.TrySetResult(null); + } + + /// + /// Releases either kind of blocked operation. + /// + public void Release() + { + this.release.Set(); + this.releaseAsync.TrySetResult(null); + } + + /// + public void Dispose() + { + this.release.Dispose(); + } + } + + private sealed class RecordingMessageBus : IMessageBus + { + /// + /// Gets the failure messages emitted by the nested runner. + /// + internal ConcurrentQueue Failures { get; } = new(); + + /// + public bool QueueMessage(IMessageSinkMessage message) + { + if (message is ITestFailed failure) + { + this.Failures.Enqueue(failure); + } + + return true; + } + + /// + public void Dispose() + { + } + } +} diff --git a/test/Xunit.StaFact.Tests/UIFactTests.cs b/test/Xunit.StaFact.Tests/UIFactTests.cs index b475b15f..521105bc 100644 --- a/test/Xunit.StaFact.Tests/UIFactTests.cs +++ b/test/Xunit.StaFact.Tests/UIFactTests.cs @@ -71,6 +71,34 @@ public async Task PassAfterDelay() await Task.Delay(10); } + [DesktopFact(Timeout = 30_000)] + public async Task Timeout_NotExceeded() + { + Assert.Equal(30_000, Assert.IsAssignableFrom(TestContext.Current.TestCase).Timeout); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + await Task.Yield(); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + TestContext.Current.CancellationToken.ThrowIfCancellationRequested(); + } + +#pragma warning disable xUnit1069 // Deliberately ignore the CancellationToken to simulate a blocked UI thread. + [DesktopFact(Timeout = 100), Trait("TestCategory", "FailureExpected")] + public void Timeout_Exceeded_Sync() + { + // Deliberately block the UI thread without honoring the CancellationToken. + Thread.Sleep(2000); + Assert.Fail("The test should have timed out."); + } +#pragma warning restore xUnit1069 + + [DesktopFact(Timeout = 100), Trait("TestCategory", "FailureExpected")] + public async Task Timeout_Exceeded_Async() + { + await Task.Delay(2000, TestContext.Current.CancellationToken); + } + [DesktopFact, Trait("TestCategory", "FailureExpected")] public async Task FailAfterYield() { @@ -109,10 +137,11 @@ public async Task UIFact_OnSingleThreadedSyncContext() Assert.Same(syncContext, SynchronizationContext.Current); } - [DesktopFact] + [DesktopFact(Timeout = 30_000)] [UISettings(Cultures = new[] { "en-US", "fr-FR" })] public async Task ExecutesUnderEachConfiguredCulture() { + TestContext.Current.CancellationToken.ThrowIfCancellationRequested(); string culture = CultureInfo.CurrentCulture.Name; Assert.Equal(culture, CultureInfo.CurrentUICulture.Name); Assert.Contains(culture, new[] { "en-US", "fr-FR" }); diff --git a/test/Xunit.StaFact.Tests/UITheoryTests.cs b/test/Xunit.StaFact.Tests/UITheoryTests.cs index b42958bd..7bb073b2 100644 --- a/test/Xunit.StaFact.Tests/UITheoryTests.cs +++ b/test/Xunit.StaFact.Tests/UITheoryTests.cs @@ -83,6 +83,39 @@ public async Task PassAfterDelay(int arg) Assert.Equal(0, arg); } + [DesktopTheory(Timeout = 30_000)] + [InlineData(0)] + public async Task Timeout_NotExceeded(int arg) + { + Assert.Equal(30_000, Assert.IsAssignableFrom(TestContext.Current.TestCase).Timeout); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + await Task.Yield(); + Assert.Equal(this.ctorThreadId, Environment.CurrentManagedThreadId); + Assert.Same(this.ctorSyncContext, SynchronizationContext.Current); + TestContext.Current.CancellationToken.ThrowIfCancellationRequested(); + Assert.Equal(0, arg); + } + +#pragma warning disable xUnit1069 // Deliberately ignore the CancellationToken to simulate a blocked UI thread. + [DesktopTheory(Timeout = 100), Trait("TestCategory", "FailureExpected")] + [InlineData(0)] + public void Timeout_Exceeded_Sync(int arg) + { + // Deliberately block the UI thread without honoring the CancellationToken. + Thread.Sleep(2000); + Assert.Fail($"The test should have timed out. Arg: {arg}"); + } +#pragma warning restore xUnit1069 + + [DesktopTheory(Timeout = 100), Trait("TestCategory", "FailureExpected")] + [InlineData(0)] + public async Task Timeout_Exceeded_Async(int arg) + { + await Task.Delay(2000, TestContext.Current.CancellationToken); + Assert.Equal(0, arg); + } + [DesktopTheory, Trait("TestCategory", "FailureExpected")] [InlineData(0)] public async Task FailAfterYield(int arg)