From fa8fa0d4efbb895b90222adb0cf476aa5caae9ee Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 08:09:18 -0400 Subject: [PATCH 01/14] Fix approve-all permission test event subscription race Subscribe before sending the permission E2E prompt so an ephemeral session.idle cannot be lost. Exercise the shared scenario against fake RPC with idle before and after the send reply, reusing the abort regression event fence and preserving the 120-second E2E timeout. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- dotnet/test/E2E/PermissionE2ETests.cs | 10 +- .../test/Unit/ClientSessionLifetimeTests.cs | 97 +++++++++++++++---- 2 files changed, 87 insertions(+), 20 deletions(-) diff --git a/dotnet/test/E2E/PermissionE2ETests.cs b/dotnet/test/E2E/PermissionE2ETests.cs index 2225dcba89..4e999bb623 100644 --- a/dotnet/test/E2E/PermissionE2ETests.cs +++ b/dotnet/test/E2E/PermissionE2ETests.cs @@ -159,13 +159,17 @@ await session.SendAndWaitAsync(new MessageOptions public async Task Should_Work_With_Approve_All_Permission_Handler() { var session = await CreateSessionAsync(new SessionConfig()); + await AssertApproveAllPermissionHandlerAsync(session, TimeSpan.FromSeconds(120)); + } - await session.SendAsync(new MessageOptions + internal static async Task AssertApproveAllPermissionHandlerAsync(CopilotSession session, TimeSpan timeout) + { + // Subscribe before sending: session.idle is ephemeral and cannot be backfilled. + var message = await session.SendAndWaitAsync(new MessageOptions { Prompt = "What is 2+2?" - }); + }, timeout); - var message = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.Contains("4", message?.Data.Content ?? string.Empty); } diff --git a/dotnet/test/Unit/ClientSessionLifetimeTests.cs b/dotnet/test/Unit/ClientSessionLifetimeTests.cs index dd7fdc2bbb..6f06d107d3 100644 --- a/dotnet/test/Unit/ClientSessionLifetimeTests.cs +++ b/dotnet/test/Unit/ClientSessionLifetimeTests.cs @@ -1717,6 +1717,64 @@ private static void AssertMessageSource(JsonElement request, string? source) Assert.False(request.TryGetProperty("wait", out _)); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Approve_All_Permission_Handler_Observes_Early_Events(bool completesBeforeReply) + { + await using var server = await FakeCopilotServer.StartAsync(); + await using var client = new CopilotClient(new CopilotClientOptions { Connection = RuntimeConnection.ForUri(server.Url) }); + await using var session = await client.CreateSessionAsync(new SessionConfig + { + OnPermissionRequest = PermissionHandler.ApproveAll + }); + var timeout = TimeSpan.FromSeconds(5); + var sendReplied = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + server.BeforeResponseAsync = async (request, cancellationToken) => + { + if (request.Method == "session.send") + { + Assert.Equal("What is 2+2?", request.Params.GetProperty("prompt").GetString()); + await server.SendSessionEventAsync(session.SessionId, "user.message", new() + { + ["content"] = request.Params.GetProperty("prompt").GetString() + }); + await server.SendAndDrainSessionEventAsync(session, "assistant.message", new() + { + ["messageId"] = "permission-message", + ["content"] = "4" + }, timeout, cancellationToken); + if (completesBeforeReply) + { + await server.SendAndDrainSessionEventAsync(session, "session.idle", new(), timeout, cancellationToken); + } + } + }; + server.AfterResponseAsync = (request, _) => + { + if (request.Method == "session.send") + { + sendReplied.TrySetResult(); + } + return Task.CompletedTask; + }; + + // Exercise the E2E test's actual ordering and assertion, without launching a CLI. + var scenario = E2E.PermissionE2ETests.AssertApproveAllPermissionHandlerAsync(session, timeout); + await sendReplied.Task.WaitAsync(timeout); + if (!completesBeforeReply) + { + Assert.False(scenario.IsCompleted); + await server.SendAndDrainSessionEventAsync(session, "session.idle", new(), timeout); + } + await scenario; + + Assert.Single(server.Requests, request => request.Method == "session.send"); + var history = await session.GetEventsAsync(); + Assert.DoesNotContain(history, evt => evt is SessionIdleEvent); + Assert.Equal("4", Assert.Single(history.OfType()).Data.Content); + } + [Theory] [InlineData(true)] [InlineData(false)] @@ -1738,23 +1796,23 @@ public async Task Abort_Recovery_Observes_Early_Events(bool recoveryCompletesBef }); if (sendCount == 1) { - await SendAndDrainAsync("tool.execution_start", new() + await server.SendAndDrainSessionEventAsync(session, "tool.execution_start", new() { ["toolCallId"] = "slow-tool", ["toolName"] = "shell" - }, cancellationToken); + }, timeout, cancellationToken); } else { Assert.Equal(2, sendCount); - await SendAndDrainAsync("assistant.message", new() + await server.SendAndDrainSessionEventAsync(session, "assistant.message", new() { ["messageId"] = "recovery-message", ["content"] = "4" - }, cancellationToken); + }, timeout, cancellationToken); if (recoveryCompletesBeforeReply) { - await SendAndDrainAsync("session.idle", new(), cancellationToken); + await server.SendAndDrainSessionEventAsync(session, "session.idle", new(), timeout, cancellationToken); } } } @@ -1765,14 +1823,14 @@ public async Task Abort_Recovery_Observes_Early_Events(bool recoveryCompletesBef { ["reason"] = "user" }); - await SendAndDrainAsync("session.idle", new() { ["aborted"] = true }, cancellationToken); + await server.SendAndDrainSessionEventAsync(session, "session.idle", new() { ["aborted"] = true }, timeout, cancellationToken); } }; server.AfterResponseAsync = async (request, cancellationToken) => { if (request.Method == "session.send" && sendCount == 2 && !recoveryCompletesBeforeReply) { - await SendAndDrainAsync("session.idle", new(), cancellationToken); + await server.SendAndDrainSessionEventAsync(session, "session.idle", new(), timeout, cancellationToken); } }; @@ -1786,16 +1844,6 @@ public async Task Abort_Recovery_Observes_Early_Events(bool recoveryCompletesBef var history = await session.GetEventsAsync(); Assert.DoesNotContain(history, evt => evt is SessionIdleEvent); Assert.Equal("4", Assert.Single(history.OfType()).Data.Content); - - async Task SendAndDrainAsync(string type, Dictionary data, CancellationToken cancellationToken) - { - var drained = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - using var subscription = session.On(_ => drained.TrySetResult()); - await server.SendSessionEventAsync(session.SessionId, type, data); - // A later event is a fence: every subscriber has finished handling the target event. - await server.SendSessionEventAsync(session.SessionId, "session.title_changed", new() { ["title"] = "fence" }); - await drained.Task.WaitAsync(timeout, cancellationToken); - } } [Fact] @@ -2411,6 +2459,21 @@ public Task SendSessionEventAsync(string sessionId, string type, Dictionary data, + TimeSpan timeout, + CancellationToken cancellationToken = default) + { + var drained = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var subscription = session.On(_ => drained.TrySetResult()); + await SendSessionEventAsync(session.SessionId, type, data); + // A later event is a fence: every subscriber has finished handling the target event. + await SendSessionEventAsync(session.SessionId, "session.title_changed", new() { ["title"] = "fence" }); + await drained.Task.WaitAsync(timeout, cancellationToken); + } + public async ValueTask DisposeAsync() { _allowDestroy.TrySetResult(); From c39bab19064e7b4e62236bb005d1839f8c0e656f Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 09:15:17 -0400 Subject: [PATCH 02/14] Eliminate remaining .NET test completion subscription races Replace post-send history-backfill waits with a send-and-wait helper that preserves the 120-second budget and requires a current-turn assistant message. Keep SendAsync under test with pre-armed completion observation. Exercise the shared helper with early events and reject missing or previous-turn-only answers. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- dotnet/test/E2E/HooksE2ETests.cs | 16 +-- dotnet/test/E2E/PermissionE2ETests.cs | 15 +-- dotnet/test/E2E/SessionE2ETests.cs | 27 ++-- .../test/E2E/SystemMessageSectionsE2ETests.cs | 6 +- .../E2E/SystemMessageTransformE2ETests.cs | 12 +- dotnet/test/E2E/TelemetryExportE2ETests.cs | 3 +- dotnet/test/E2E/ToolResultsE2ETests.cs | 9 +- dotnet/test/E2E/ToolsE2ETests.cs | 36 ++---- dotnet/test/Harness/TestHelper.cs | 120 +----------------- .../test/Unit/ClientSessionLifetimeTests.cs | 42 ++++++ 10 files changed, 92 insertions(+), 194 deletions(-) diff --git a/dotnet/test/E2E/HooksE2ETests.cs b/dotnet/test/E2E/HooksE2ETests.cs index 0d9155fbc7..08366a87a4 100644 --- a/dotnet/test/E2E/HooksE2ETests.cs +++ b/dotnet/test/E2E/HooksE2ETests.cs @@ -32,13 +32,11 @@ public async Task Should_Invoke_PreToolUse_Hook_When_Model_Runs_A_Tool() // Create a file for the model to read await File.WriteAllTextAsync(Path.Join(Ctx.WorkDir, "hello.txt"), "Hello from the test!"); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Read the contents of hello.txt and tell me what it says" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - // Should have received at least one preToolUse hook call Assert.NotEmpty(preToolUseInputs); @@ -68,13 +66,11 @@ public async Task Should_Invoke_PostToolUse_Hook_After_Model_Runs_A_Tool() // Create a file for the model to read await File.WriteAllTextAsync(Path.Join(Ctx.WorkDir, "world.txt"), "World from the test!"); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Read the contents of world.txt and tell me what it says" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - // Should have received at least one postToolUse hook call Assert.NotEmpty(postToolUseInputs); @@ -109,13 +105,11 @@ public async Task Should_Invoke_Both_PreToolUse_And_PostToolUse_Hooks_For_Single await File.WriteAllTextAsync(Path.Join(Ctx.WorkDir, "both.txt"), "Testing both hooks!"); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Read the contents of both.txt" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - // Both hooks should have been called Assert.NotEmpty(preToolUseInputs); Assert.NotEmpty(postToolUseInputs); @@ -149,13 +143,11 @@ public async Task Should_Deny_Tool_Execution_When_PreToolUse_Returns_Deny() var originalContent = "Original content that should not be modified"; await File.WriteAllTextAsync(Path.Join(Ctx.WorkDir, "protected.txt"), originalContent); - await session.SendAsync(new MessageOptions + var response = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Edit protected.txt and replace 'Original' with 'Modified'" }); - var response = await TestHelper.GetFinalAssistantMessageAsync(session); - // The hook should have been called Assert.NotEmpty(preToolUseInputs); diff --git a/dotnet/test/E2E/PermissionE2ETests.cs b/dotnet/test/E2E/PermissionE2ETests.cs index 4e999bb623..1de100dee1 100644 --- a/dotnet/test/E2E/PermissionE2ETests.cs +++ b/dotnet/test/E2E/PermissionE2ETests.cs @@ -111,13 +111,11 @@ public async Task Should_Deny_Permission_When_Handler_Returns_Denied() var testFilePath = Path.Combine(Ctx.WorkDir, "protected.txt"); await File.WriteAllTextAsync(testFilePath, "protected content"); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Edit protected.txt and replace 'protected' with 'hacked'." }); - await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.True( userRejectedToolCall, "Expected a tool.execution_complete event whose error indicates the user rejected the call."); @@ -164,8 +162,7 @@ public async Task Should_Work_With_Approve_All_Permission_Handler() internal static async Task AssertApproveAllPermissionHandlerAsync(CopilotSession session, TimeSpan timeout) { - // Subscribe before sending: session.idle is ephemeral and cannot be backfilled. - var message = await session.SendAndWaitAsync(new MessageOptions + var message = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "What is 2+2?" }, timeout); @@ -187,13 +184,11 @@ public async Task Should_Handle_Async_Permission_Handler() } }); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Run 'echo test' and tell me what happens" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.True(permissionRequestReceived, "Permission request should have been received"); } @@ -325,13 +320,11 @@ public async Task Should_Receive_ToolCallId_In_Permission_Requests() } }); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Run 'echo test'" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.True(receivedToolCallId, "Should have received toolCallId in permission request"); } diff --git a/dotnet/test/E2E/SessionE2ETests.cs b/dotnet/test/E2E/SessionE2ETests.cs index 08781ce209..37c25095fc 100644 --- a/dotnet/test/E2E/SessionE2ETests.cs +++ b/dotnet/test/E2E/SessionE2ETests.cs @@ -54,8 +54,7 @@ public async Task Should_Create_A_Session_With_Appended_SystemMessage_Config() SystemMessage = new SystemMessageConfig { Mode = SystemMessageMode.Append, Content = systemMessageSuffix } }); - await session.SendAsync(new MessageOptions { Prompt = "What is your full name?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "What is your full name?" }); Assert.NotNull(assistantMessage); var content = assistantMessage!.Data.Content ?? string.Empty; @@ -78,8 +77,7 @@ public async Task Should_Create_A_Session_With_Replaced_SystemMessage_Config() SystemMessage = new SystemMessageConfig { Mode = SystemMessageMode.Replace, Content = testSystemMessage } }); - await session.SendAsync(new MessageOptions { Prompt = "What is your full name?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "What is your full name?" }); Assert.NotNull(assistantMessage); var content = assistantMessage!.Data.Content ?? string.Empty; @@ -219,8 +217,7 @@ public async Task Should_Create_Session_With_Custom_Tool() ] }); - await session.SendAsync(new MessageOptions { Prompt = "What is the secret number for key ALPHA?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "What is the secret number for key ALPHA?" }); Assert.NotNull(assistantMessage); Assert.Contains("54321", assistantMessage!.Data.Content ?? string.Empty); } @@ -463,10 +460,7 @@ public async Task Should_Receive_Session_Events() // Events must be dispatched serially — never more than one handler invocation at a time. Assert.Equal(1, maxConcurrent); - // Verify the assistant response contains the expected answer. - // session.idle is ephemeral and not in getEvents(), but we already - // confirmed idle via the live event handler above. - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session, alreadyIdle: true); + var assistantMessage = observedEvents.OfType().LastOrDefault(); Assert.NotNull(assistantMessage); Assert.Contains("300", assistantMessage!.Data.Content); @@ -481,8 +475,17 @@ public async Task Send_Returns_Immediately_While_Events_Stream_In_Background() OnPermissionRequest = PermissionHandler.ApproveAll, }); var events = new ConcurrentQueue(); + AssistantMessageEvent? message = null; - session.On(evt => events.Enqueue(evt.Type)); + session.On(evt => + { + events.Enqueue(evt.Type); + if (evt is AssistantMessageEvent assistantMessage) + { + message = assistantMessage; + } + }); + var idle = TestHelper.GetNextEventOfTypeAsync(session); // Use a slow command so we can verify SendAsync() returns before completion await session.SendAsync(new MessageOptions { Prompt = "Run 'sleep 2 && echo done'" }); @@ -491,7 +494,7 @@ public async Task Send_Returns_Immediately_While_Events_Stream_In_Background() Assert.DoesNotContain("session.idle", events); // Wait for turn to complete - var message = await TestHelper.GetFinalAssistantMessageAsync(session); + await idle; Assert.Contains("done", message?.Data.Content ?? string.Empty); Assert.Contains("session.idle", events); diff --git a/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs b/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs index 41c46d3b9d..8e9f670c3e 100644 --- a/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs +++ b/dotnet/test/E2E/SystemMessageSectionsE2ETests.cs @@ -31,8 +31,7 @@ public async Task Should_Use_Replaced_Identity_Section_In_Response() } }); - await session.SendAsync(new MessageOptions { Prompt = "Who are you?" }); - var response = await TestHelper.GetFinalAssistantMessageAsync(session); + var response = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Who are you?" }); Assert.NotNull(response); var content = response.Data.Content.ToLowerInvariant(); @@ -61,8 +60,7 @@ public async Task Should_Use_Replaced_Preamble_Section_In_Response() } }); - await session.SendAsync(new MessageOptions { Prompt = "Who are you?" }); - var response = await TestHelper.GetFinalAssistantMessageAsync(session); + var response = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Who are you?" }); Assert.NotNull(response); var content = response.Data.Content.ToLowerInvariant(); diff --git a/dotnet/test/E2E/SystemMessageTransformE2ETests.cs b/dotnet/test/E2E/SystemMessageTransformE2ETests.cs index 79210e61b3..9a8e687071 100644 --- a/dotnet/test/E2E/SystemMessageTransformE2ETests.cs +++ b/dotnet/test/E2E/SystemMessageTransformE2ETests.cs @@ -48,13 +48,11 @@ public async Task Should_Invoke_Transform_Callbacks_With_Section_Content() await File.WriteAllTextAsync(Path.Combine(Ctx.WorkDir, "test.txt"), "Hello transform!"); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Read the contents of test.txt and tell me what it says" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.True(identityCallbackInvoked, "Expected identity transform callback to be invoked"); Assert.True(toneCallbackInvoked, "Expected tone transform callback to be invoked"); } @@ -83,13 +81,11 @@ public async Task Should_Apply_Transform_Modifications_To_Section_Content() await File.WriteAllTextAsync(Path.Combine(Ctx.WorkDir, "hello.txt"), "Hello!"); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Read the contents of hello.txt" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - // Verify the transform result was actually applied to the system message var traffic = await Ctx.GetExchangesAsync(); Assert.NotEmpty(traffic); @@ -128,13 +124,11 @@ public async Task Should_Work_With_Static_Overrides_And_Transforms_Together() await File.WriteAllTextAsync(Path.Combine(Ctx.WorkDir, "combo.txt"), "Combo test!"); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Read the contents of combo.txt and tell me what it says" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.True(transformCallbackInvoked, "Expected identity transform callback to be invoked"); } } diff --git a/dotnet/test/E2E/TelemetryExportE2ETests.cs b/dotnet/test/E2E/TelemetryExportE2ETests.cs index e2ad447e26..f6914688fe 100644 --- a/dotnet/test/E2E/TelemetryExportE2ETests.cs +++ b/dotnet/test/E2E/TelemetryExportE2ETests.cs @@ -40,8 +40,7 @@ public async Task Should_Export_File_Telemetry_For_Sdk_Interactions() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions { Prompt = prompt }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = prompt }); Assert.NotNull(assistantMessage); Assert.Contains("TELEMETRY_E2E_DONE", assistantMessage!.Data.Content ?? string.Empty, StringComparison.Ordinal); diff --git a/dotnet/test/E2E/ToolResultsE2ETests.cs b/dotnet/test/E2E/ToolResultsE2ETests.cs index 103c7ffe2a..34aada586f 100644 --- a/dotnet/test/E2E/ToolResultsE2ETests.cs +++ b/dotnet/test/E2E/ToolResultsE2ETests.cs @@ -29,12 +29,11 @@ public async Task Should_Handle_Structured_ToolResultObject_From_Custom_Tool() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "What's the weather in Paris?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Matches("(?i)sunny|72", assistantMessage!.Data.Content ?? string.Empty); @@ -56,12 +55,11 @@ public async Task Should_Handle_Tool_Result_With_Failure_ResultType() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Check the status of the service using check_status. If it fails, say 'service is down'." }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("service is down", assistantMessage!.Data.Content?.ToLowerInvariant() ?? string.Empty); @@ -84,12 +82,11 @@ public async Task Should_Preserve_ToolTelemetry_And_Not_Stringify_Structured_Res OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Analyze the file main.ts for issues." }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("no issues", assistantMessage!.Data.Content?.ToLowerInvariant() ?? string.Empty); diff --git a/dotnet/test/E2E/ToolsE2ETests.cs b/dotnet/test/E2E/ToolsE2ETests.cs index 8a786f3927..943e49f45f 100644 --- a/dotnet/test/E2E/ToolsE2ETests.cs +++ b/dotnet/test/E2E/ToolsE2ETests.cs @@ -35,12 +35,11 @@ await File.WriteAllTextAsync( OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "What's the first line of README.md in this directory?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("ELIZA", assistantMessage!.Data.Content ?? string.Empty); } @@ -54,12 +53,11 @@ public async Task Invokes_Custom_Tool() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Use encrypt_string to encrypt this string: Hello" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("HELLO", assistantMessage!.Data.Content ?? string.Empty); @@ -92,13 +90,11 @@ public async Task Low_Level_Tool_Definition() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "First, set the current phase to 'analyzing'. Then search for items with keyword 'copilot'. Report the phase and search results." }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); - Assert.NotNull(assistantMessage); var content = assistantMessage!.Data.Content ?? string.Empty; Assert.NotEmpty(content); @@ -133,8 +129,7 @@ public async Task Handles_Tool_Calling_Errors() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions { Prompt = "What is my location? If you can't find out, just say 'unknown'." }); - var answer = await TestHelper.GetFinalAssistantMessageAsync(session); + var answer = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "What is my location? If you can't find out, just say 'unknown'." }); // Check the underlying traffic var traffic = await Ctx.GetExchangesAsync(); @@ -175,14 +170,13 @@ public async Task Can_Receive_And_Return_Complex_Types() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Perform a DB query for the 'cities' table using IDs 12 and 19, sorting ascending. " + "Reply only with lines of the form: [cityname] [population]" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); var responseContent = assistantMessage?.Data.Content!; Assert.NotNull(assistantMessage); Assert.NotEmpty(responseContent); @@ -227,12 +221,11 @@ public async Task Overrides_Built_In_Tool_With_Custom_Tool() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Use grep to search for the word 'hello'" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("CUSTOM_GREP_RESULT", assistantMessage!.Data.Content ?? string.Empty); @@ -351,12 +344,11 @@ static string SafeLookup([Description("Lookup ID")] string id) } }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Use safe_lookup to look up 'test123'" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("RESULT", assistantMessage!.Data.Content ?? string.Empty); Assert.False(didRunPermissionRequest); @@ -371,12 +363,11 @@ public async Task Can_Return_Binary_Result() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Use get_image. What color is the square in the image?" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("yellow", assistantMessage!.Data.Content?.ToLowerInvariant() ?? string.Empty); @@ -408,12 +399,11 @@ public async Task Invokes_Custom_Tool_With_Permission_Handler() }, }); - await session.SendAsync(new MessageOptions + var assistantMessage = await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Use encrypt_string to encrypt this string: Hello" }); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); Assert.NotNull(assistantMessage); Assert.Contains("HELLO", assistantMessage!.Data.Content ?? string.Empty); @@ -438,13 +428,11 @@ public async Task Denies_Custom_Tool_When_Permission_Denied() OnPermissionRequest = async (request, invocation) => PermissionDecision.Reject(), }); - await session.SendAsync(new MessageOptions + await TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Use encrypt_string to encrypt this string: Hello" }); - await TestHelper.GetFinalAssistantMessageAsync(session); - // The tool handler should NOT have been called since permission was denied Assert.False(toolHandlerCalled); @@ -472,7 +460,7 @@ public async Task Should_Execute_Multiple_Custom_Tools_In_Parallel_Single_Turn() OnPermissionRequest = PermissionHandler.ApproveAll, }); - await session.SendAsync(new MessageOptions + var assistantMessageTask = TestHelper.SendAndGetFinalAssistantMessageAsync(session, new MessageOptions { Prompt = "Use lookup_city with 'Paris' and lookup_country with 'France' at the same time, then combine both results in your reply." }); @@ -483,7 +471,7 @@ await session.SendAsync(new MessageOptions Assert.Equal("Paris", cityResult); Assert.Equal("France", countryResult); - var assistantMessage = await TestHelper.GetFinalAssistantMessageAsync(session); + var assistantMessage = await assistantMessageTask; Assert.NotNull(assistantMessage); var content = assistantMessage!.Data.Content ?? string.Empty; Assert.Contains("CITY_PARIS", content); diff --git a/dotnet/test/Harness/TestHelper.cs b/dotnet/test/Harness/TestHelper.cs index a230ddb81e..76045bfbb0 100644 --- a/dotnet/test/Harness/TestHelper.cs +++ b/dotnet/test/Harness/TestHelper.cs @@ -13,122 +13,14 @@ public static class TestHelper private static readonly TimeSpan DefaultEventTimeout = TimeSpan.FromSeconds(120); private static readonly TimeSpan DefaultPollInterval = TimeSpan.FromMilliseconds(100); - public static async Task GetFinalAssistantMessageAsync( + public static async Task SendAndGetFinalAssistantMessageAsync( CopilotSession session, - TimeSpan? timeout = null, - bool alreadyIdle = false) - { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - using var cts = new CancellationTokenSource(timeout ?? DefaultEventTimeout); - - // Both `finalAssistantMessage` and `sawIdle` are set from two threads — the - // subscription callback (CLI read loop) and CheckExistingMessagesAsync (RPC reply). - // We complete only once we've observed both, regardless of which path saw which. - var stateLock = new object(); - AssistantMessageEvent? finalAssistantMessage = null; - bool sawIdle = false; - - void TryComplete() - { - AssistantMessageEvent? snapshot; - bool idle; - lock (stateLock) - { - snapshot = finalAssistantMessage; - idle = sawIdle; - } - if (snapshot != null && idle) tcs.TrySetResult(snapshot); - } - - using var subscription = session.On(evt => - { - switch (evt) - { - case AssistantMessageEvent msg: - lock (stateLock) { finalAssistantMessage = msg; } - TryComplete(); - break; - case SessionIdleEvent: - lock (stateLock) { sawIdle = true; } - TryComplete(); - break; - case SessionErrorEvent error: - tcs.TrySetException(new Exception(error.Data.Message ?? "session error")); - break; - } - }); - - // Backfill from already-delivered messages so we don't lose events that arrived - // between SendAsync returning and the subscription being installed. Run it - // concurrently with the live subscription, but keep the Task observable so any - // exception is propagated through tcs (not the unobserved-task handler) and so - // we can drain it deterministically below. Pass cts.Token so the backfill is - // bounded by the same timeout as the wait itself, and so a hung GetEventsAsync - // can't block the drain in `finally`. - var backfill = CheckExistingMessagesAsync(cts.Token); - - using var registration = cts.Token.Register( - static state => ((TaskCompletionSource)state!).TrySetException( - new TimeoutException("Timeout waiting for assistant message")), - tcs); - - try - { - return await tcs.Task; - } - finally - { - // Drain the backfill before our `using` scopes (cts, subscription) dispose. - // Any exception was already routed through tcs above, so swallow here. - try { await backfill.ConfigureAwait(false); } - catch (Exception) { /* intentionally ignored: already propagated via tcs */ } - } - - async Task CheckExistingMessagesAsync(CancellationToken cancellationToken) - { - try - { - var (existingFinal, existingIdle) = await GetExistingMessagesAsync(session, alreadyIdle, cancellationToken); - lock (stateLock) - { - // Preserve a newer message captured by the subscription in the meantime. - if (existingFinal != null && finalAssistantMessage == null) - { - finalAssistantMessage = existingFinal; - } - if (existingIdle) sawIdle = true; - } - TryComplete(); - } - catch (Exception ex) - { - tcs.TrySetException(ex); - } - } - } - - private static async Task<(AssistantMessageEvent? Final, bool SawIdle)> GetExistingMessagesAsync(CopilotSession session, bool alreadyIdle, CancellationToken cancellationToken = default) + MessageOptions options, + TimeSpan? timeout = null) { - var messages = (await session.GetEventsAsync(cancellationToken)).ToList(); - - var lastUserIdx = messages.FindLastIndex(m => m is UserMessageEvent); - var currentTurn = lastUserIdx < 0 ? messages : messages.Skip(lastUserIdx).ToList(); - - var error = currentTurn.OfType().FirstOrDefault(); - if (error != null) throw new Exception(error.Data.Message ?? "session error"); - - var idleIdx = alreadyIdle ? currentTurn.Count : currentTurn.FindIndex(m => m is SessionIdleEvent); - var sawIdle = alreadyIdle || idleIdx >= 0; - - // Find the most recent assistant message in the turn (whether idle has arrived or not). - var searchEnd = idleIdx >= 0 ? idleIdx : currentTurn.Count; - for (var i = searchEnd - 1; i >= 0; i--) - { - if (currentTurn[i] is AssistantMessageEvent msg) - return (msg, sawIdle); - } - - return (null, sawIdle); + // Subscribe before sending: session.idle is ephemeral and cannot be backfilled. + return await session.SendAndWaitAsync(options, timeout ?? DefaultEventTimeout) + ?? throw new InvalidOperationException("Session became idle without an assistant message."); } public static async Task GetNextEventOfTypeAsync( diff --git a/dotnet/test/Unit/ClientSessionLifetimeTests.cs b/dotnet/test/Unit/ClientSessionLifetimeTests.cs index 6f06d107d3..88de6abd0d 100644 --- a/dotnet/test/Unit/ClientSessionLifetimeTests.cs +++ b/dotnet/test/Unit/ClientSessionLifetimeTests.cs @@ -12,6 +12,7 @@ using System.Text; using System.Text.Json; using GitHub.Copilot.Rpc; +using GitHub.Copilot.Test.Harness; using Microsoft.Extensions.AI; using Xunit; @@ -1775,6 +1776,47 @@ public async Task Approve_All_Permission_Handler_Observes_Early_Events(bool comp Assert.Equal("4", Assert.Single(history.OfType()).Data.Content); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task SendAndGetFinalAssistantMessage_Requires_Current_Turn_Message(bool hasPreviousTurn) + { + await using var server = await FakeCopilotServer.StartAsync(); + await using var client = new CopilotClient(new CopilotClientOptions { Connection = RuntimeConnection.ForUri(server.Url) }); + await using var session = await client.CreateSessionAsync(new SessionConfig()); + var timeout = TimeSpan.FromSeconds(5); + server.BeforeResponseAsync = async (request, cancellationToken) => + { + if (request.Method == "session.send") + { + var prompt = request.Params.GetProperty("prompt").GetString(); + await server.SendSessionEventAsync(session.SessionId, "user.message", new() { ["content"] = prompt }); + if (prompt == "previous turn") + { + await server.SendAndDrainSessionEventAsync(session, "assistant.message", new() + { + ["messageId"] = "previous-message", + ["content"] = "previous answer" + }, timeout, cancellationToken); + } + await server.SendAndDrainSessionEventAsync(session, "session.idle", new(), timeout, cancellationToken); + } + }; + + if (hasPreviousTurn) + { + var previous = await TestHelper.SendAndGetFinalAssistantMessageAsync( + session, new MessageOptions { Prompt = "previous turn" }, timeout); + Assert.Equal("previous answer", previous.Data.Content); + } + + var error = await Assert.ThrowsAsync(() => + TestHelper.SendAndGetFinalAssistantMessageAsync( + session, new MessageOptions { Prompt = "no assistant message" }, timeout)); + Assert.Equal("Session became idle without an assistant message.", error.Message); + Assert.DoesNotContain(await session.GetEventsAsync(), evt => evt is SessionIdleEvent); + } + [Theory] [InlineData(true)] [InlineData(false)] From 0de0fc6872db6334031ef81d1f92bc87b78092a7 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 09:25:09 -0400 Subject: [PATCH 03/14] Make Node test completion subscriptions race-free Replace unsafe post-send waits with sendAndWait or a trigger-based helper that subscribes before work starts. Remove ephemeral-idle history fallback, retain send behavior assertions and test budgets, and add deterministic early-event and cleanup regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- nodejs/test/e2e/harness/sdkTestHelper.ts | 104 ++++------ nodejs/test/e2e/permissions.e2e.test.ts | 8 +- nodejs/test/e2e/session.e2e.test.ts | 29 ++- nodejs/test/e2e/session_lifecycle.e2e.test.ts | 2 +- nodejs/test/e2e/telemetry.e2e.test.ts | 6 +- nodejs/test/session-send-and-wait.test.ts | 184 ++++++++++++++++-- 6 files changed, 228 insertions(+), 105 deletions(-) diff --git a/nodejs/test/e2e/harness/sdkTestHelper.ts b/nodejs/test/e2e/harness/sdkTestHelper.ts index c30aa9ea6f..9cca10388b 100644 --- a/nodejs/test/e2e/harness/sdkTestHelper.ts +++ b/nodejs/test/e2e/harness/sdkTestHelper.ts @@ -53,73 +53,51 @@ function waitForChildExit(child: ChildProcess, timeoutMs: number): Promise Promise ): Promise { - // Install the live subscription (via getFutureFinalResponse) before issuing the - // existing-messages RPC so we don't miss events that arrive while that RPC is in flight. - const futurePromise = getFutureFinalResponse(session); - // We may end up returning from the existing-messages path; attach a noop handler so - // the unawaited future-response rejection doesn't surface as an unhandled rejection. - futurePromise.catch(() => {}); - - const existing = await getExistingFinalResponse(session, alreadyIdle); - if (existing) { - return existing; - } - return futurePromise; -} - -async function getExistingFinalResponse( - session: CopilotSession, - alreadyIdle: boolean = false -): Promise { - const messages = await session.getEvents(); - const finalUserMessageIndex = messages.findLastIndex((m) => m.type === "user.message"); - const currentTurnMessages = - finalUserMessageIndex < 0 ? messages : messages.slice(finalUserMessageIndex); - - const currentTurnError = currentTurnMessages.find((m) => m.type === "session.error"); - if (currentTurnError) { - const error = new Error(currentTurnError.data.message); - error.stack = currentTurnError.data.stack; - throw error; - } + type Outcome = { message: AssistantMessageEvent } | { error: Error }; + let resolveOutcome!: (outcome: Outcome) => void; + const outcomePromise = new Promise((resolve) => { + resolveOutcome = resolve; + }); + let finalAssistantMessage: AssistantMessageEvent | undefined; + + // session.idle is ephemeral: subscribe before triggering work, not after an RPC + // reply or a history lookup. Keep errors as values while the trigger is in flight. + const unsubscribe = session.on((event) => { + if (event.type === "assistant.message") { + finalAssistantMessage = event; + } else if (event.type === "session.idle" && event.data.mode !== "autopilot") { + unsubscribe(); + resolveOutcome( + finalAssistantMessage + ? { message: finalAssistantMessage } + : { + error: new Error( + "Received session.idle without a preceding assistant.message" + ), + } + ); + } else if (event.type === "session.error") { + unsubscribe(); + const error = new Error(event.data.message); + error.stack = event.data.stack; + resolveOutcome({ error }); + } + }); - const sessionIdleMessageIndex = alreadyIdle - ? currentTurnMessages.length - : currentTurnMessages.findIndex((m) => m.type === "session.idle"); - if (sessionIdleMessageIndex !== -1) { - return currentTurnMessages - .slice(0, sessionIdleMessageIndex) - .findLast((m) => m.type === "assistant.message") as AssistantMessageEvent | undefined; + try { + await trigger(); + const outcome = await outcomePromise; + if ("error" in outcome) { + throw outcome.error; + } + return outcome.message; + } finally { + unsubscribe(); } - - return undefined; -} - -function getFutureFinalResponse(session: CopilotSession): Promise { - return new Promise((resolve, reject) => { - let finalAssistantMessage: AssistantMessageEvent | undefined; - session.on((event) => { - if (event.type === "assistant.message") { - finalAssistantMessage = event; - } else if (event.type === "session.idle") { - if (!finalAssistantMessage) { - reject( - new Error("Received session.idle without a preceding assistant.message") - ); - } else { - resolve(finalAssistantMessage); - } - } else if (event.type === "session.error") { - const error = new Error(event.data.message); - error.stack = event.data.stack; - reject(error); - } - }); - }); } export async function retry( diff --git a/nodejs/test/e2e/permissions.e2e.test.ts b/nodejs/test/e2e/permissions.e2e.test.ts index d7600ed165..638ea12a33 100644 --- a/nodejs/test/e2e/permissions.e2e.test.ts +++ b/nodejs/test/e2e/permissions.e2e.test.ts @@ -15,7 +15,7 @@ import type { } from "../../src/index.js"; import { approveAll, defineTool, createAttributedPermissionResult } from "../../src/index.js"; import { createSdkTestContext, isInProcessTransport } from "./harness/sdkTestContext.js"; -import { getFinalAssistantMessage, getNextEventOfType } from "./harness/sdkTestHelper.js"; +import { withFinalAssistantMessage, getNextEventOfType } from "./harness/sdkTestHelper.js"; const isWindows = process.platform === "win32"; @@ -342,9 +342,9 @@ describe("Permission callbacks", async () => { } }); - const sessionDone = getFinalAssistantMessage(session); - - void session.send({ prompt: "Run 'echo slow_handler_test'" }); + const sessionDone = withFinalAssistantMessage(session, () => + session.send({ prompt: "Run 'echo slow_handler_test'" }) + ); // Wait for permission handler to be invoked await handlerStarted; diff --git a/nodejs/test/e2e/session.e2e.test.ts b/nodejs/test/e2e/session.e2e.test.ts index 77a1dec408..900ca4e859 100644 --- a/nodejs/test/e2e/session.e2e.test.ts +++ b/nodejs/test/e2e/session.e2e.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it, onTestFinished, vi } from "vitest"; import { ParsedHttpExchange } from "../../../test/harness/replayingCapiProxy.js"; import { CopilotClient, approveAll, defineTool, RuntimeConnection } from "../../src/index.js"; import { createSdkTestContext, DEFAULT_GITHUB_TOKEN, isCI } from "./harness/sdkTestContext.js"; -import { getFinalAssistantMessage, getNextEventOfType, retry } from "./harness/sdkTestHelper.js"; +import { withFinalAssistantMessage, getNextEventOfType, retry } from "./harness/sdkTestHelper.js"; const { copilotClient: client, @@ -440,12 +440,12 @@ describe("Sessions", () => { }); expect(session2.sessionId).toBe(sessionId); - // session.idle is ephemeral and not persisted, so use alreadyIdle - // to find the assistant message from the completed session. - const answer2 = await getFinalAssistantMessage(session2, { alreadyIdle: true }); + // sendAndWait already observed idle on session1; only durable messages + // are needed to verify the completed turn survived resumption. + const messages = await session2.getEvents(); + const answer2 = messages.findLast((m) => m.type === "assistant.message"); expect(answer2?.data.content).toContain("2"); - const messages = await session2.getEvents(); expect(messages).toContainEqual(expect.objectContaining({ type: "user.message" })); expect(messages).toContainEqual(expect.objectContaining({ type: "session.resume" })); @@ -671,9 +671,9 @@ describe("Sessions", () => { expect(session.sessionId).toMatch(/^[a-f0-9-]+$/); // Session should work normally with custom config dir - await session.send({ prompt: "What is 1+1?" }); - const assistantMessage = await getFinalAssistantMessage(session); - expect(assistantMessage.data.content).toContain("2"); + const assistantMessage = await session.sendAndWait({ prompt: "What is 1+1?" }); + expect(assistantMessage).toBeDefined(); + expect(assistantMessage?.data.content).toContain("2"); }); it("should log messages at all levels and emit matching session events", async () => { @@ -969,14 +969,13 @@ describe("Send Blocking Behavior", async () => { events.push(event.type); }); - // Use a slow command so we can verify send() returns before completion - await session.send({ prompt: "Run 'sleep 2 && echo done'" }); - - // send() should return before turn completes (no session.idle yet) - expect(events).not.toContain("session.idle"); + const message = await withFinalAssistantMessage(session, async () => { + // Use a slow command so we can verify send() returns before completion. + await session.send({ prompt: "Run 'sleep 2 && echo done'" }); - // Wait for turn to complete - const message = await getFinalAssistantMessage(session); + // send() should return before turn completes (no session.idle yet). + expect(events).not.toContain("session.idle"); + }); expect(message.data.content).toContain("done"); expect(events).toContain("session.idle"); diff --git a/nodejs/test/e2e/session_lifecycle.e2e.test.ts b/nodejs/test/e2e/session_lifecycle.e2e.test.ts index fae8782736..fe7ba645e0 100644 --- a/nodejs/test/e2e/session_lifecycle.e2e.test.ts +++ b/nodejs/test/e2e/session_lifecycle.e2e.test.ts @@ -85,7 +85,7 @@ describe("Session Lifecycle", async () => { const messages = await session.getEvents(); expect(messages.length).toBeGreaterThan(0); - // Should have at least session.start, user.message, assistant.message, session.idle + // History contains durable messages, not the ephemeral session.idle event. const types = messages.map((m: SessionEvent) => m.type); expect(types).toContain("session.start"); expect(types).toContain("user.message"); diff --git a/nodejs/test/e2e/telemetry.e2e.test.ts b/nodejs/test/e2e/telemetry.e2e.test.ts index 66a0bb8cef..9fb89fc0d7 100644 --- a/nodejs/test/e2e/telemetry.e2e.test.ts +++ b/nodejs/test/e2e/telemetry.e2e.test.ts @@ -8,7 +8,6 @@ import { describe, expect, it } from "vitest"; import { z } from "zod"; import { approveAll, defineTool, RuntimeConnection } from "../../src/index.js"; import { createSdkTestContext } from "./harness/sdkTestContext.js"; -import { getFinalAssistantMessage } from "./harness/sdkTestHelper.js"; interface TelemetryEntry { type?: string; @@ -85,10 +84,9 @@ describe("Telemetry export", async () => { ], }); - await session.send({ prompt }); - const assistantMessage = await getFinalAssistantMessage(session); + const assistantMessage = await session.sendAndWait({ prompt }, 90_000); expect(assistantMessage).toBeDefined(); - expect(assistantMessage.data.content ?? "").toContain("TELEMETRY_E2E_DONE"); + expect(assistantMessage?.data.content ?? "").toContain("TELEMETRY_E2E_DONE"); await session.disconnect(); await client.stop(); diff --git a/nodejs/test/session-send-and-wait.test.ts b/nodejs/test/session-send-and-wait.test.ts index 4ee2e8eebc..07d67fe9fe 100644 --- a/nodejs/test/session-send-and-wait.test.ts +++ b/nodejs/test/session-send-and-wait.test.ts @@ -2,10 +2,11 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import { describe, expect, it, onTestFinished } from "vitest"; +import { describe, expect, it, onTestFinished, vi } from "vitest"; import type { MessageConnection } from "vscode-jsonrpc/node.js"; import { CopilotSession } from "../src/session.js"; -import type { SessionEvent } from "../src/generated/session-events.js"; +import type { AssistantMessageEvent, SessionEvent } from "../src/generated/session-events.js"; +import { withFinalAssistantMessage } from "./e2e/harness/sdkTestHelper.js"; function sessionEvent( type: "session.idle", @@ -32,32 +33,49 @@ function errorEvent(message: string): SessionEvent { } as SessionEvent; } -function controlledSession(): { - session: CopilotSession; - sendStarted: Promise; - resolveSend: () => void; - rejectSend: (error: Error) => void; -} { +function assistantMessage(content: string): AssistantMessageEvent { + return { + type: "assistant.message", + id: "assistant-1", + parentId: null, + timestamp: new Date().toISOString(), + data: { messageId: "message-1", content }, + }; +} + +function controlledSession( + history: SessionEvent[] = [], + onSend?: (session: CopilotSession) => void +) { let resolveSendRequest: ((value: unknown) => void) | undefined; let rejectSendRequest: ((error: Error) => void) | undefined; let markSendStarted: () => void; const sendStarted = new Promise((resolve) => { markSendStarted = resolve; }); - const connection = { - sendRequest: () => - new Promise((resolve, reject) => { - resolveSendRequest = resolve; - rejectSendRequest = reject; - markSendStarted(); - }), - } as unknown as MessageConnection; + const sendRequest = vi.fn((method: string) => { + if (method === "session.getMessages") { + return Promise.resolve({ events: history }); + } + if (method !== "session.send") { + throw new Error(`Unexpected RPC: ${method}`); + } + return new Promise((resolve, reject) => { + resolveSendRequest = resolve; + rejectSendRequest = reject; + markSendStarted(); + onSend?.(session); + }); + }); + const connection = { sendRequest } as unknown as MessageConnection; + const session = new CopilotSession("session-1", connection); return { - session: new CopilotSession("session-1", connection), + session, + sendRequest, sendStarted, resolveSend: () => resolveSendRequest?.({ messageId: "msg-1" }), - rejectSend: (error) => rejectSendRequest?.(error), + rejectSend: (error: Error) => rejectSendRequest?.(error), }; } @@ -156,3 +174,133 @@ describe("sendAndWait", () => { await expect(errorFirstPending).rejects.toThrow("first error"); }); }); + +describe("completion subscriptions", () => { + it.each(["sendAndWait", "withFinalAssistantMessage"] as const)( + "%s captures final output and ephemeral idle before the send reply", + async (waiter) => { + const finalMessage = assistantMessage("done"); + const history: SessionEvent[] = []; + const { session, sendRequest, sendStarted, resolveSend } = controlledSession( + history, + (session) => { + const events = [ + assistantMessage("working"), + finalMessage, + sessionEvent("session.idle"), + ]; + for (const event of events) { + if (!event.ephemeral) { + history.push(event); + } + session._dispatchEvent(event); + } + } + ); + + let completed = false; + const pending = + waiter === "sendAndWait" + ? session.sendAndWait({ prompt: "hi" }) + : withFinalAssistantMessage(session, () => session.send({ prompt: "hi" })); + const observed = pending.then((message) => { + completed = true; + return message; + }); + await sendStarted; + expect(completed).toBe(false); + expect(sendRequest).toHaveBeenCalledTimes(1); + + // Only durable messages can be retrieved, even though idle was delivered. + const stored = await session.getEvents(); + expect(stored).toEqual(history); + expect(stored.map((event) => event.type)).toEqual([ + "assistant.message", + "assistant.message", + ]); + + resolveSend(); + await expect(observed).resolves.toBe(finalMessage); + } + ); +}); + +describe("withFinalAssistantMessage", () => { + it("does not accept a prior turn's message when the new turn has no assistant output", async () => { + const { session, resolveSend } = controlledSession( + [assistantMessage("old turn")], + (session) => session._dispatchEvent(sessionEvent("session.idle")) + ); + const pending = withFinalAssistantMessage(session, () => session.send({ prompt: "hi" })); + const outcome = expect(pending).rejects.toThrow( + "Received session.idle without a preceding assistant.message" + ); + + resolveSend(); + await outcome; + }); + + it("does not complete on an assistant message or an autopilot continuation", async () => { + const { session, resolveSend } = controlledSession(); + const pending = withFinalAssistantMessage(session, () => session.send({ prompt: "hi" })); + resolveSend(); + + session._dispatchEvent(assistantMessage("continuing")); + session._dispatchEvent(sessionEvent("session.idle", { mode: "autopilot" })); + + const finalMessage = assistantMessage("done"); + session._dispatchEvent(finalMessage); + session._dispatchEvent(sessionEvent("session.idle", { mode: "interactive" })); + await expect(pending).resolves.toBe(finalMessage); + }); + + it.each(["idle", "session.error", "send rejection", "trigger throw"] as const)( + "removes the completion subscription after %s", + async (outcome) => { + const { session, resolveSend, rejectSend } = controlledSession(); + const originalOn = session.on.bind(session); + const unsubscribe = vi.fn<() => void>(); + vi.spyOn(session, "on").mockImplementation((handler) => { + unsubscribe.mockImplementation(originalOn(handler)); + return unsubscribe; + }); + + const pending = withFinalAssistantMessage(session, () => { + if (outcome === "trigger throw") { + throw new Error("trigger failed"); + } + return session.send({ prompt: "hi" }); + }); + const finalMessage = assistantMessage("done"); + const errorMessage = + outcome === "session.error" + ? "session failed" + : outcome === "send rejection" + ? "send failed" + : "trigger failed"; + const expected = + outcome === "idle" + ? expect(pending).resolves.toBe(finalMessage) + : expect(pending).rejects.toThrow(errorMessage); + + if (outcome === "idle") { + session._dispatchEvent(finalMessage); + session._dispatchEvent(sessionEvent("session.idle")); + // Later events must not replace an already observed terminal outcome. + session._dispatchEvent(assistantMessage("next turn")); + session._dispatchEvent(errorEvent("later error")); + resolveSend(); + } else if (outcome === "session.error") { + session._dispatchEvent(errorEvent("session failed")); + session._dispatchEvent(sessionEvent("session.idle")); + resolveSend(); + } else if (outcome === "send rejection") { + session._dispatchEvent(errorEvent("session failed")); + rejectSend(new Error("send failed")); + } + + await expected; + expect(unsubscribe).toHaveBeenCalled(); + } + ); +}); From 231f587a40be9f073590541f56ed076e559b2822 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 09:30:07 -0400 Subject: [PATCH 04/14] Pre-arm Go test completion and recovery subscriptions Separate live event waiters from reads of already-completed history. Install listeners synchronously before sends, handler release, and abort/recovery operations; preserve caller contexts and required assistant output. Add deterministic fake-RPC regressions with ephemeral idle omitted from history. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- go/internal/e2e/mcp_and_agents_e2e_test.go | 8 +- go/internal/e2e/permissions_e2e_test.go | 14 +- go/internal/e2e/resume_mcp_oauth_e2e_test.go | 4 +- go/internal/e2e/session_e2e_test.go | 84 ++--- go/internal/e2e/telemetry_e2e_test.go | 4 +- go/internal/e2e/testharness/helper.go | 143 ++++---- go/internal/e2e/testharness/helper_test.go | 352 +++++++++++++++++++ go/internal/e2e/tool_results_e2e_test.go | 16 +- go/internal/e2e/tools_e2e_test.go | 36 +- 9 files changed, 509 insertions(+), 152 deletions(-) create mode 100644 go/internal/e2e/testharness/helper_test.go diff --git a/go/internal/e2e/mcp_and_agents_e2e_test.go b/go/internal/e2e/mcp_and_agents_e2e_test.go index 0cc57810ed..086f4b083f 100644 --- a/go/internal/e2e/mcp_and_agents_e2e_test.go +++ b/go/internal/e2e/mcp_and_agents_e2e_test.go @@ -34,6 +34,8 @@ func TestMCPServersE2E(t *testing.T) { waitForMCPServerStatus(t, session, "test-server", rpc.MCPServerStatusConnected) // Simple interaction to verify session works + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "What is 2+2?", }) @@ -41,7 +43,7 @@ func TestMCPServersE2E(t *testing.T) { t.Fatalf("Failed to send message: %v", err) } - message, err := testharness.GetFinalAssistantMessage(t.Context(), session) + message, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get final message: %v", err) } @@ -205,6 +207,8 @@ func TestCustomAgentsE2E(t *testing.T) { } // Simple interaction to verify session works + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "What is 5+5?", }) @@ -212,7 +216,7 @@ func TestCustomAgentsE2E(t *testing.T) { t.Fatalf("Failed to send message: %v", err) } - message, err := testharness.GetFinalAssistantMessage(t.Context(), session) + message, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get final message: %v", err) } diff --git a/go/internal/e2e/permissions_e2e_test.go b/go/internal/e2e/permissions_e2e_test.go index 89681470e8..1b3cf7aeb1 100644 --- a/go/internal/e2e/permissions_e2e_test.go +++ b/go/internal/e2e/permissions_e2e_test.go @@ -157,6 +157,8 @@ func TestPermissionsE2E(t *testing.T) { t.Fatalf("Failed to write test file: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "Edit protected.txt and replace 'protected' with 'hacked'.", }) @@ -164,7 +166,7 @@ func TestPermissionsE2E(t *testing.T) { t.Fatalf("Failed to send message: %v", err) } - _, err = testharness.GetFinalAssistantMessage(t.Context(), session) + _, err = finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get final message: %v", err) } @@ -285,12 +287,14 @@ func TestPermissionsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 2+2?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - message, err := testharness.GetFinalAssistantMessage(t.Context(), session) + message, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get final message: %v", err) } @@ -487,6 +491,8 @@ func TestPermissionsE2E(t *testing.T) { } }) + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() go func() { _, _ = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "Run 'echo slow_handler_test'", @@ -515,9 +521,9 @@ func TestPermissionsE2E(t *testing.T) { close(releaseHandler) - message, err := testharness.GetFinalAssistantMessage(t.Context(), session) + message, err := finalMessage.Wait(t.Context()) if err != nil { - t.Fatalf("GetFinalAssistantMessage failed: %v", err) + t.Fatalf("Waiting for final assistant message failed: %v", err) } lifecycleMu.Lock() diff --git a/go/internal/e2e/resume_mcp_oauth_e2e_test.go b/go/internal/e2e/resume_mcp_oauth_e2e_test.go index db61f483a9..0c8dbd6cc1 100644 --- a/go/internal/e2e/resume_mcp_oauth_e2e_test.go +++ b/go/internal/e2e/resume_mcp_oauth_e2e_test.go @@ -29,12 +29,14 @@ func TestResumeMCPOAuthE2E(t *testing.T) { } sessionID := session1.SessionID + finalMessage := testharness.SubscribeToFinalAssistantMessage(session1) + defer finalMessage.Close() _, err = session1.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 1+1?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session1) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } diff --git a/go/internal/e2e/session_e2e_test.go b/go/internal/e2e/session_e2e_test.go index 12550e6e2d..941015ed37 100644 --- a/go/internal/e2e/session_e2e_test.go +++ b/go/internal/e2e/session_e2e_test.go @@ -1,6 +1,7 @@ package e2e import ( + "context" "encoding/base64" "os" "path/filepath" @@ -156,12 +157,14 @@ func TestSessionE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is your full name?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session) + assistantMessage, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -368,12 +371,14 @@ func TestSessionE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is the secret number for key ALPHA?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session) + assistantMessage, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -402,12 +407,14 @@ func TestSessionE2E(t *testing.T) { } sessionID := session1.SessionID + finalMessage := testharness.SubscribeToFinalAssistantMessage(session1) + defer finalMessage.Close() _, err = session1.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 1+1?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session1) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -428,7 +435,7 @@ func TestSessionE2E(t *testing.T) { t.Errorf("Expected resumed session ID to match, got %q vs %q", session2.SessionID, sessionID) } - answer2, err := testharness.GetFinalAssistantMessage(t.Context(), session2, true) + answer2, err := testharness.GetFinalAssistantMessageFromHistory(t.Context(), session2) if err != nil { t.Fatalf("Failed to get assistant message from resumed session: %v", err) } @@ -463,12 +470,14 @@ func TestSessionE2E(t *testing.T) { } sessionID := session1.SessionID + finalMessage := testharness.SubscribeToFinalAssistantMessage(session1) + defer finalMessage.Close() _, err = session1.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 1+1?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session1) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -632,28 +641,13 @@ func TestSessionE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } - // Set up event listeners BEFORE sending to avoid race conditions - toolStartCh := make(chan *copilot.SessionEvent, 1) - toolStartErrCh := make(chan error, 1) - go func() { - evt, err := testharness.GetNextEventOfType(session, copilot.SessionEventTypeToolExecutionStart, 60*time.Second) - if err != nil { - toolStartErrCh <- err - } else { - toolStartCh <- evt - } - }() - - sessionIdleCh := make(chan *copilot.SessionEvent, 1) - sessionIdleErrCh := make(chan error, 1) - go func() { - evt, err := testharness.GetNextEventOfType(session, copilot.SessionEventTypeSessionIdle, 60*time.Second) - if err != nil { - sessionIdleErrCh <- err - } else { - sessionIdleCh <- evt - } - }() + // Install subscriptions synchronously; starting a goroutine is not a fence. + abortCtx, cancelAbort := context.WithTimeout(t.Context(), 60*time.Second) + defer cancelAbort() + toolStart := testharness.SubscribeToEvent(session, copilot.SessionEventTypeToolExecutionStart) + defer toolStart.Close() + sessionIdle := testharness.SubscribeToEvent(session, copilot.SessionEventTypeSessionIdle) + defer sessionIdle.Close() // Send a message that triggers a long-running shell command _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "run the shell command 'sleep 100' (note this works on both bash and PowerShell)"}) @@ -662,10 +656,7 @@ func TestSessionE2E(t *testing.T) { } // Wait for tool.execution_start - select { - case <-toolStartCh: - // Tool execution has started - case err := <-toolStartErrCh: + if _, err := toolStart.Wait(abortCtx); err != nil { t.Fatalf("Failed waiting for tool.execution_start: %v", err) } @@ -676,10 +667,7 @@ func TestSessionE2E(t *testing.T) { } // Wait for session.idle after abort - select { - case <-sessionIdleCh: - // Session is idle - case err := <-sessionIdleErrCh: + if _, err := sessionIdle.Wait(abortCtx); err != nil { t.Fatalf("Failed waiting for session.idle after abort: %v", err) } @@ -705,26 +693,18 @@ func TestSessionE2E(t *testing.T) { } // We should be able to send another message - answerCh := make(chan *copilot.SessionEvent, 1) - answerErrCh := make(chan error, 1) - go func() { - evt, err := testharness.GetNextEventOfType(session, copilot.SessionEventTypeAssistantMessage, 60*time.Second) - if err != nil { - answerErrCh <- err - } else { - answerCh <- evt - } - }() + answerCtx, cancelAnswer := context.WithTimeout(t.Context(), 60*time.Second) + defer cancelAnswer() + answerWaiter := testharness.SubscribeToEvent(session, copilot.SessionEventTypeAssistantMessage) + defer answerWaiter.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 2+2?"}) if err != nil { t.Fatalf("Failed to send message after abort: %v", err) } - var answer *copilot.SessionEvent - select { - case answer = <-answerCh: - case err := <-answerErrCh: + answer, err := answerWaiter.Wait(answerCtx) + if err != nil { t.Fatalf("Failed waiting for assistant message after abort: %v", err) } @@ -825,7 +805,7 @@ func TestSessionE2E(t *testing.T) { // Verify the assistant response contains the expected answer. // session.idle is ephemeral and not in GetEvents(), but we already // confirmed idle via the live event handler above. - assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session, true) + assistantMessage, err := testharness.GetFinalAssistantMessageFromHistory(t.Context(), session) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -852,12 +832,14 @@ func TestSessionE2E(t *testing.T) { } // Session should work normally with custom config dir + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What is 1+1?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - assistantMessage, err := testharness.GetFinalAssistantMessage(t.Context(), session) + assistantMessage, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } diff --git a/go/internal/e2e/telemetry_e2e_test.go b/go/internal/e2e/telemetry_e2e_test.go index 77f8bec8ea..6c03d83819 100644 --- a/go/internal/e2e/telemetry_e2e_test.go +++ b/go/internal/e2e/telemetry_e2e_test.go @@ -52,10 +52,12 @@ func TestTelemetryE2E(t *testing.T) { } sessionID := session.SessionID + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() if _, err := session.Send(t.Context(), copilot.MessageOptions{Prompt: prompt}); err != nil { t.Fatalf("Send failed: %v", err) } - final, err := testharness.GetFinalAssistantMessage(t.Context(), session) + final, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to wait for final assistant message: %v", err) } diff --git a/go/internal/e2e/testharness/helper.go b/go/internal/e2e/testharness/helper.go index af08b2dbcc..cc508074af 100644 --- a/go/internal/e2e/testharness/helper.go +++ b/go/internal/e2e/testharness/helper.go @@ -5,7 +5,7 @@ import ( "errors" "path/filepath" "runtime" - "time" + "sync" copilot "github.com/github/copilot-sdk/go" ) @@ -29,88 +29,87 @@ func RepoPath(elem ...string) string { return filepath.Join(append([]string{repoRoot}, elem...)...) } -// GetFinalAssistantMessage waits for and returns the final assistant message from a session turn. -// If alreadyIdle is true, skip waiting for session.idle (useful for resumed sessions where the -// idle event was ephemeral and not persisted in the event history). -func GetFinalAssistantMessage(ctx context.Context, session *copilot.Session, alreadyIdle ...bool) (*copilot.SessionEvent, error) { - result := make(chan *copilot.SessionEvent, 1) - errCh := make(chan error, 1) +type eventResult struct { + event *copilot.SessionEvent + err error +} + +// EventWaiter is a synchronously installed subscription. Call Close even if the +// operation that should produce the event fails before Wait is called. +type EventWaiter struct { + result chan eventResult + once sync.Once + unsubscribe func() +} + +func (w *EventWaiter) complete(event *copilot.SessionEvent, err error) { + w.once.Do(func() { w.result <- eventResult{event: event, err: err} }) +} + +// Wait waits using only the caller's context, without imposing a default timeout. +// Events received between subscription and Wait are retained. +func (w *EventWaiter) Wait(ctx context.Context) (*copilot.SessionEvent, error) { + defer w.Close() + select { + case result := <-w.result: + return result.event, result.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// Close removes the subscription and is safe to call more than once. +func (w *EventWaiter) Close() { + w.unsubscribe() +} - // Subscribe to future events +// SubscribeToFinalAssistantMessage subscribes before returning. Call it before +// Send, releasing a blocked handler, or any other operation that can finish a +// turn. Unlike durable assistant messages, session.idle is ephemeral: GetEvents +// cannot recover a missed completion. A successful wait always includes a message. +func SubscribeToFinalAssistantMessage(session *copilot.Session) *EventWaiter { + w := &EventWaiter{result: make(chan eventResult, 1)} var finalAssistantMessage *copilot.SessionEvent - unsubscribe := session.On(func(event copilot.SessionEvent) { + w.unsubscribe = session.On(func(event copilot.SessionEvent) { switch d := event.Data.(type) { case *copilot.AssistantMessageData: finalAssistantMessage = &event case *copilot.SessionIdleData: - if finalAssistantMessage != nil { - result <- finalAssistantMessage + if finalAssistantMessage == nil { + w.complete(nil, errors.New("session became idle without an assistant message")) + } else { + w.complete(finalAssistantMessage, nil) } case *copilot.SessionErrorData: - errCh <- errors.New(d.Message) + w.complete(nil, errors.New(d.Message)) } }) - defer unsubscribe() - - // Also check existing messages in case the response already arrived - isAlreadyIdle := len(alreadyIdle) > 0 && alreadyIdle[0] - go func() { - existing, err := getExistingFinalResponse(ctx, session, isAlreadyIdle) - if err != nil { - errCh <- err - return - } - if existing != nil { - result <- existing - } - }() - - select { - case msg := <-result: - return msg, nil - case err := <-errCh: - return nil, err - case <-ctx.Done(): - return nil, errors.New("timeout waiting for assistant message") - } + return w } -// GetNextEventOfType waits for and returns the next event of the specified type from a session. -func GetNextEventOfType(session *copilot.Session, eventType copilot.SessionEventType, timeout time.Duration) (*copilot.SessionEvent, error) { - result := make(chan *copilot.SessionEvent, 1) - errCh := make(chan error, 1) - - unsubscribe := session.On(func(event copilot.SessionEvent) { +// SubscribeToEvent subscribes before returning, so the triggering operation can +// run before Wait without losing events. Call Close if the operation fails. +func SubscribeToEvent(session *copilot.Session, eventType copilot.SessionEventType) *EventWaiter { + w := &EventWaiter{result: make(chan eventResult, 1)} + w.unsubscribe = session.On(func(event copilot.SessionEvent) { switch event.Type() { case eventType: - select { - case result <- &event: - default: - } + w.complete(&event, nil) case copilot.SessionEventTypeSessionError: msg := "session error" if d, ok := event.Data.(*copilot.SessionErrorData); ok { msg = d.Message } - select { - case errCh <- errors.New(msg): - default: - } + w.complete(nil, errors.New(msg)) } }) - defer unsubscribe() - - select { - case evt := <-result: - return evt, nil - case err := <-errCh: - return nil, err - case <-time.After(timeout): - return nil, errors.New("timeout waiting for event: " + string(eventType)) - } + return w } -func getExistingFinalResponse(ctx context.Context, session *copilot.Session, alreadyIdle bool) (*copilot.SessionEvent, error) { +// GetFinalAssistantMessageFromHistory reads a turn whose completion has already +// been observed independently, including after resuming an idle session. It does +// not wait for completion and must not be used as a substitute for a live waiter. +func GetFinalAssistantMessageFromHistory(ctx context.Context, session *copilot.Session) (*copilot.SessionEvent, error) { messages, err := session.GetEvents(ctx) if err != nil { return nil, err @@ -143,27 +142,11 @@ func getExistingFinalResponse(ctx context.Context, session *copilot.Session, alr } } - // Find session.idle and get last assistant message before it - sessionIdleIndex := -1 - if alreadyIdle { - sessionIdleIndex = len(currentTurnMessages) - } else { - for i, msg := range currentTurnMessages { - if msg.Type() == "session.idle" { - sessionIdleIndex = i - break - } - } - } - - if sessionIdleIndex != -1 { - // Find last assistant.message before session.idle - for i := sessionIdleIndex - 1; i >= 0; i-- { - if currentTurnMessages[i].Type() == "assistant.message" { - return ¤tTurnMessages[i], nil - } + for i := len(currentTurnMessages) - 1; i >= 0; i-- { + if currentTurnMessages[i].Type() == "assistant.message" { + return ¤tTurnMessages[i], nil } } - return nil, nil + return nil, errors.New("no assistant message in the completed turn") } diff --git a/go/internal/e2e/testharness/helper_test.go b/go/internal/e2e/testharness/helper_test.go new file mode 100644 index 0000000000..a7b378e25d --- /dev/null +++ b/go/internal/e2e/testharness/helper_test.go @@ -0,0 +1,352 @@ +package testharness + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "testing" + "time" + + copilot "github.com/github/copilot-sdk/go" + "github.com/github/copilot-sdk/go/internal/jsonrpc2" + "github.com/github/copilot-sdk/go/rpc" +) + +func TestFinalAssistantMessageWaiterBeforeCompletionRPC(t *testing.T) { + for _, method := range []string{ + "session.send", + "session.tools.handlePendingToolCall", + "session.permissions.handlePendingPermissionRequest", + } { + t.Run(method, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + assistant := copilot.SessionEvent{Data: &copilot.AssistantMessageData{MessageID: "answer", Content: "4"}} + f := newCompletionFixture(t, ctx, []copilot.SessionEvent{assistant}) + waiter := SubscribeToFinalAssistantMessage(f.session) + defer waiter.Close() + f.server.SetRequestHandler(method, f.beforeResponse(t, ctx, []copilot.SessionEvent{ + {Data: &copilot.AssistantMessageData{MessageID: "intermediate", Content: "thinking"}}, + assistant, + {Data: &copilot.SessionIdleData{}}, + {Data: &copilot.SessionIdleData{}}, + {Data: &copilot.SessionIdleData{}}, + })) + + switch method { + case "session.send": + messageID, err := f.session.Send(ctx, copilot.MessageOptions{Prompt: "What is 2+2?"}) + if err != nil || messageID != "sent" { + t.Fatalf("Send = %q, %v; want sent, nil", messageID, err) + } + case "session.tools.handlePendingToolCall": + result, err := f.session.RPC.Tools.HandlePendingToolCall(ctx, &rpc.HandlePendingToolCallRequest{ + RequestID: "tool-request", Result: rpc.ExternalToolStringResult("4"), + }) + if err != nil || !result.Success { + t.Fatalf("HandlePendingToolCall = %+v, %v", result, err) + } + case "session.permissions.handlePendingPermissionRequest": + result, err := f.session.RPC.Permissions.HandlePendingPermissionRequest(ctx, &rpc.PermissionDecisionRequest{ + RequestID: "permission-request", Result: &rpc.PermissionDecisionApproveOnce{}, + }) + if err != nil || !result.Success { + t.Fatalf("HandlePendingPermissionRequest = %+v, %v", result, err) + } + } + + // The RPC response is withheld until all notifications have been + // processed. No goroutine has called Wait, and history contains no idle. + history, err := f.session.GetEvents(ctx) + if err != nil || len(history) != 1 || history[0].Type() != copilot.SessionEventTypeAssistantMessage { + t.Fatalf("Expected durable assistant-only history, got %+v, %v", history, err) + } + answer, err := waiter.Wait(ctx) + requireCompletionAnswer(t, answer, err, "4") + + existing, err := GetFinalAssistantMessageFromHistory(ctx, f.session) + requireCompletionAnswer(t, existing, err, "4") + + // A new waiter must not mistake a previous turn's durable answer for + // completion of a turn it never observed. + next := SubscribeToFinalAssistantMessage(f.session) + cancelled, cancelNext := context.WithCancel(ctx) + cancelNext() + if answer, err := next.Wait(cancelled); answer != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("Late waiter = %+v, %v; want cancellation", answer, err) + } + }) + } +} + +func TestFinalAssistantMessageWaiterRequiresMessageAndPropagatesErrors(t *testing.T) { + for _, tc := range []struct { + name string + events []copilot.SessionEvent + wantErr string + }{ + { + name: "idle without assistant", + events: []copilot.SessionEvent{{Data: &copilot.SessionIdleData{}}}, + wantErr: "session became idle without an assistant message", + }, + { + name: "session error before idle", + events: []copilot.SessionEvent{ + {Data: &copilot.SessionErrorData{Message: "model failed"}}, + {Data: &copilot.SessionErrorData{Message: "another error"}}, + {Data: &copilot.SessionIdleData{}}, + }, + wantErr: "model failed", + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + // Prior-turn messages and errors must not satisfy or fail this wait. + f := newCompletionFixture(t, ctx, []copilot.SessionEvent{ + {Data: &copilot.UserMessageData{Content: "previous prompt"}}, + {Data: &copilot.AssistantMessageData{MessageID: "previous", Content: "previous answer"}}, + {Data: &copilot.SessionErrorData{Message: "previous error"}}, + }) + waiter := SubscribeToFinalAssistantMessage(f.session) + defer waiter.Close() + f.server.SetRequestHandler("session.send", f.beforeResponse(t, ctx, tc.events)) + if _, err := f.session.Send(ctx, copilot.MessageOptions{Prompt: "hello"}); err != nil { + t.Fatal(err) + } + answer, err := waiter.Wait(ctx) + if answer != nil || err == nil || err.Error() != tc.wantErr { + t.Fatalf("Wait = %+v, %v; want %q", answer, err, tc.wantErr) + } + }) + } +} + +func TestFinalAssistantMessageWaiterBeforeHandlerRelease(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + f := newCompletionFixture(t, ctx, nil) + waiter := SubscribeToFinalAssistantMessage(f.session) + defer waiter.Close() + complete := f.beforeResponse(t, ctx, []copilot.SessionEvent{ + {Data: &copilot.AssistantMessageData{Content: "released"}}, + {Data: &copilot.SessionIdleData{}}, + }) + entered := make(chan struct{}) + release := make(chan struct{}) + f.server.SetRequestHandler("session.send", func(params json.RawMessage) (json.RawMessage, *jsonrpc2.Error) { + close(entered) + select { + case <-release: + return complete(params) + case <-ctx.Done(): + return nil, &jsonrpc2.Error{Code: -32000, Message: ctx.Err().Error()} + } + }) + sent := make(chan error, 1) + go func() { + _, err := f.session.Send(ctx, copilot.MessageOptions{Prompt: "use the blocked handler"}) + sent <- err + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + close(release) + select { + case err := <-sent: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + answer, err := waiter.Wait(ctx) + requireCompletionAnswer(t, answer, err, "released") +} + +func TestFinalAssistantMessageWaiterRequiresLiveIdle(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + assistant := copilot.SessionEvent{Data: &copilot.AssistantMessageData{Content: "not finished"}} + f := newCompletionFixture(t, ctx, []copilot.SessionEvent{assistant}) + waiter := SubscribeToFinalAssistantMessage(f.session) + defer waiter.Close() + f.server.SetRequestHandler("session.send", f.beforeResponse(t, ctx, []copilot.SessionEvent{assistant})) + if _, err := f.session.Send(ctx, copilot.MessageOptions{Prompt: "hello"}); err != nil { + t.Fatal(err) + } + cancelled, cancelWait := context.WithCancel(ctx) + cancelWait() + if answer, err := waiter.Wait(cancelled); answer != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("Wait without idle = %+v, %v; want cancellation", answer, err) + } +} + +func TestEventWaitersBeforeAbortAndRecovery(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + f := newCompletionFixture(t, ctx, nil) + toolStart := SubscribeToEvent(f.session, copilot.SessionEventTypeToolExecutionStart) + defer toolStart.Close() + idle := SubscribeToEvent(f.session, copilot.SessionEventTypeSessionIdle) + defer idle.Close() + f.server.SetRequestHandler("session.send", f.beforeResponse(t, ctx, []copilot.SessionEvent{ + {Data: &copilot.ToolExecutionStartData{ToolCallID: "tool", ToolName: "shell"}}, + })) + if _, err := f.session.Send(ctx, copilot.MessageOptions{Prompt: "start tool"}); err != nil { + t.Fatal(err) + } + if _, err := toolStart.Wait(ctx); err != nil { + t.Fatal(err) + } + + f.server.SetRequestHandler("session.abort", f.beforeResponse(t, ctx, []copilot.SessionEvent{ + {Data: &copilot.SessionIdleData{}}, + })) + if err := f.session.Abort(ctx); err != nil { + t.Fatal(err) + } + if _, err := idle.Wait(ctx); err != nil { + t.Fatal(err) + } + + answerWaiter := SubscribeToEvent(f.session, copilot.SessionEventTypeAssistantMessage) + defer answerWaiter.Close() + f.server.SetRequestHandler("session.send", f.beforeResponse(t, ctx, []copilot.SessionEvent{ + {Data: &copilot.AssistantMessageData{Content: "recovered"}}, + {Data: &copilot.SessionIdleData{}}, + })) + if _, err := f.session.Send(ctx, copilot.MessageOptions{Prompt: "recover"}); err != nil { + t.Fatal(err) + } + answer, err := answerWaiter.Wait(ctx) + requireCompletionAnswer(t, answer, err, "recovered") +} + +func requireCompletionAnswer(t *testing.T, event *copilot.SessionEvent, err error, content string) { + t.Helper() + if err != nil || event == nil { + t.Fatalf("Expected assistant message, got %+v, %v", event, err) + } + if data, ok := event.Data.(*copilot.AssistantMessageData); !ok || data.Content != content { + t.Fatalf("Expected assistant content %q, got %+v", content, event.Data) + } +} + +type completionFixture struct { + session *copilot.Session + server *jsonrpc2.Client + conn net.Conn + nextFence int +} + +// Uses the same minimal JSON-RPC server approach as the client unit tests, with +// a public TCP client so these tests exercise real session event dispatch. +func newCompletionFixture(t *testing.T, ctx context.Context, history []copilot.SessionEvent) *completionFixture { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + historyResult, err := json.Marshal(map[string]any{"events": history}) + if err != nil { + t.Fatal(err) + } + ready := make(chan *completionFixture, 1) + go func() { + conn, err := listener.Accept() + if err != nil { + return + } + server := jsonrpc2.NewClient(conn, conn) + t.Cleanup(server.Stop) + for method, result := range map[string]string{ + "connect": `{"ok":true,"protocolVersion":3,"version":"test"}`, + "plugins.builtin.set": `{}`, + "session.create": `{"sessionId":"completion-session"}`, + "session.options.update": `{"success":true}`, + "session.detach": `{"success":true}`, + } { + server.SetRequestHandler(method, func(json.RawMessage) (json.RawMessage, *jsonrpc2.Error) { + return []byte(result), nil + }) + } + server.SetRequestHandler("session.getMessages", func(json.RawMessage) (json.RawMessage, *jsonrpc2.Error) { + return historyResult, nil + }) + server.Start() + ready <- &completionFixture{server: server, conn: conn} + }() + client := copilot.NewClient(&copilot.ClientOptions{ + Connection: copilot.URIConnection{URL: listener.Addr().String()}, + }) + t.Cleanup(func() { client.ForceStop() }) + session, err := client.CreateSession(ctx, &copilot.SessionConfig{ + SessionID: "completion-session", OnPermissionRequest: copilot.PermissionHandler.ApproveAll, + }) + if err != nil { + t.Fatal(err) + } + select { + case fixture := <-ready: + fixture.session = session + return fixture + case <-ctx.Done(): + t.Fatal(ctx.Err()) + return nil + } +} + +// Called after the waiters are armed. A trailing notification acknowledges that +// every preceding event has run through the session's consumer, not just reached +// its queue. All RPCs in the fixture are sequential; notifications are written +// only inside the current request handler, before its response is written. +func (f *completionFixture) beforeResponse(t *testing.T, ctx context.Context, events []copilot.SessionEvent) jsonrpc2.RequestHandler { + t.Helper() + f.nextFence++ + fenceID := fmt.Sprintf("fence-%d", f.nextFence) + delivered := make(chan struct{}, 1) + unsubscribe := f.session.On(func(event copilot.SessionEvent) { + if event.ID == fenceID { + select { + case delivered <- struct{}{}: + default: + } + } + }) + t.Cleanup(unsubscribe) + var frames bytes.Buffer + for _, event := range append(append([]copilot.SessionEvent(nil), events...), copilot.SessionEvent{ + ID: fenceID, Data: &copilot.SessionInfoData{Message: "delivery fence"}, + }) { + if event.Type() == copilot.SessionEventTypeSessionIdle { + event.Ephemeral = copilot.Bool(true) + } + data, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", "method": "session.event", + "params": map[string]any{"sessionId": f.session.SessionID, "event": event}, + }) + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(&frames, "Content-Length: %d\r\n\r\n%s", len(data), data) + } + return func(json.RawMessage) (json.RawMessage, *jsonrpc2.Error) { + if _, err := f.conn.Write(frames.Bytes()); err != nil { + return nil, &jsonrpc2.Error{Code: -32000, Message: err.Error()} + } + select { + case <-delivered: + return []byte(`{"messageId":"sent","success":true}`), nil + case <-ctx.Done(): + return nil, &jsonrpc2.Error{Code: -32000, Message: ctx.Err().Error()} + } + } +} diff --git a/go/internal/e2e/tool_results_e2e_test.go b/go/internal/e2e/tool_results_e2e_test.go index 8908ffcdaf..ecd00a078f 100644 --- a/go/internal/e2e/tool_results_e2e_test.go +++ b/go/internal/e2e/tool_results_e2e_test.go @@ -37,12 +37,14 @@ func TestToolResultsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What's the weather in Paris?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -83,6 +85,8 @@ func TestToolResultsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "Check the status of the service using check_status. If it fails, say 'service is down'.", }) @@ -90,7 +94,7 @@ func TestToolResultsE2E(t *testing.T) { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -135,12 +139,14 @@ func TestToolResultsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "Analyze the file main.ts for issues."}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -297,6 +303,8 @@ func TestToolResultsE2E(t *testing.T) { } }) + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "Use access_secret to get the API key. If access is denied, tell me it was 'access denied'.", }) @@ -326,7 +334,7 @@ func TestToolResultsE2E(t *testing.T) { t.Fatal("Timed out waiting for tool execution complete") } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get final assistant message: %v", err) } diff --git a/go/internal/e2e/tools_e2e_test.go b/go/internal/e2e/tools_e2e_test.go index 062d377917..75613bebdf 100644 --- a/go/internal/e2e/tools_e2e_test.go +++ b/go/internal/e2e/tools_e2e_test.go @@ -34,12 +34,14 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "What's the first line of README.md in this directory?"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -69,12 +71,14 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "Use encrypt_string to encrypt this string: Hello"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -126,6 +130,8 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "First, set the current phase to 'analyzing'. Then search for items with keyword 'copilot'. Report the phase and search results.", }) @@ -133,7 +139,7 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -188,6 +194,8 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "What is my location? If you can't find out, just say 'unknown'.", }) @@ -195,7 +203,7 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -306,6 +314,8 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{ Prompt: "Perform a DB query for the 'cities' table using IDs 12 and 19, sorting ascending. " + "Reply only with lines of the form: [cityname] [population]", @@ -314,7 +324,7 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -383,12 +393,14 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "Use safe_lookup to look up 'test123'"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -551,12 +563,14 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "Use grep to search for the word 'hello'"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -594,12 +608,14 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "Use encrypt_string to encrypt this string: Hello"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - answer, err := testharness.GetFinalAssistantMessage(t.Context(), session) + answer, err := finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } @@ -650,12 +666,14 @@ func TestToolsE2E(t *testing.T) { t.Fatalf("Failed to create session: %v", err) } + finalMessage := testharness.SubscribeToFinalAssistantMessage(session) + defer finalMessage.Close() _, err = session.Send(t.Context(), copilot.MessageOptions{Prompt: "Use encrypt_string to encrypt this string: Hello"}) if err != nil { t.Fatalf("Failed to send message: %v", err) } - _, err = testharness.GetFinalAssistantMessage(t.Context(), session) + _, err = finalMessage.Wait(t.Context()) if err != nil { t.Fatalf("Failed to get assistant message: %v", err) } From dbcb8bb5f03a63c0642a2b047c6bbca06677be70 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 09:31:53 -0400 Subject: [PATCH 05/14] Close Python test completion and scheduled-listener races Use send_and_wait for ordinary turns and synchronously subscribe before sends, aborts, and pending-work operations. Remove ephemeral-idle backfill, retain per-caller timeouts and error policies, and cover early RPC completion plus cancellation cleanup without initializing E2E runtime from unit tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- python/_session_test_helpers.py | 64 +++++++ python/e2e/test_mode_handlers_e2e.py | 123 +++++------- python/e2e/test_multi_client_e2e.py | 69 +++---- python/e2e/test_pending_work_resume_e2e.py | 136 ++++++------- python/e2e/test_permissions_e2e.py | 11 +- python/e2e/test_rpc_event_side_effects_e2e.py | 16 -- python/e2e/test_session_e2e.py | 175 +++++++++-------- python/e2e/test_session_todos_changed_e2e.py | 12 +- python/e2e/test_telemetry_e2e.py | 6 +- python/e2e/test_tool_results_e2e.py | 14 +- python/e2e/test_tools_e2e.py | 61 +++--- python/e2e/testharness/__init__.py | 4 +- python/e2e/testharness/helper.py | 138 +------------ python/test_session.py | 181 ++++++++++++++++++ 14 files changed, 554 insertions(+), 456 deletions(-) create mode 100644 python/_session_test_helpers.py diff --git a/python/_session_test_helpers.py b/python/_session_test_helpers.py new file mode 100644 index 0000000000..77eec022a7 --- /dev/null +++ b/python/_session_test_helpers.py @@ -0,0 +1,64 @@ +"""Live-event test helpers without E2E runtime or proxy initialization.""" + +import asyncio +from collections.abc import Callable + +from copilot.session import CopilotSession +from copilot.session_events import SessionErrorData, SessionEvent + + +def wait_for_event( + session: CopilotSession, + predicate: Callable[[SessionEvent], bool], + timeout: float = 30.0, + *, + fail_on_session_error: bool = False, +) -> asyncio.Task[SessionEvent]: + """Subscribe synchronously and return a task for the next matching live event. + + Call before the operation that emits the event, then await or cancel the task. + Merely scheduling an async subscriber would leave a race before it starts. + In particular, session.idle is ephemeral and cannot be recovered from history. + + Only predicate matches complete the wait by default. Set fail_on_session_error + to also fail on unmatched session errors when the caller requires that policy. + """ + loop = asyncio.get_running_loop() + result_future: asyncio.Future[SessionEvent] = loop.create_future() + + def on_event(event: SessionEvent) -> None: + if result_future.done(): + return + + if predicate(event): + result_future.set_result(event) + elif fail_on_session_error and isinstance(event.data, SessionErrorData): + result_future.set_exception(RuntimeError(event.data.message or "session error")) + + unsubscribe = session.on(on_event) + + async def wait() -> SessionEvent: + return await asyncio.wait_for(result_future, timeout=timeout) + + def cleanup(_task: asyncio.Task[SessionEvent]) -> None: + unsubscribe() + result_future.cancel() + if not result_future.cancelled(): + result_future.exception() + + task = loop.create_task(wait()) + # A task cancelled before its first step never executes a coroutine's finally. + task.add_done_callback(cleanup) + return task + + +def get_next_event_of_type( + session: CopilotSession, event_type: str, timeout: float = 30.0 +) -> asyncio.Task[SessionEvent]: + """Subscribe before an operation; fail on session errors unless waiting for that event.""" + return wait_for_event( + session, + lambda event: event.type.value == event_type, + timeout, + fail_on_session_error=True, + ) diff --git a/python/e2e/test_mode_handlers_e2e.py b/python/e2e/test_mode_handlers_e2e.py index d5182c453d..d681b403c9 100644 --- a/python/e2e/test_mode_handlers_e2e.py +++ b/python/e2e/test_mode_handlers_e2e.py @@ -20,7 +20,7 @@ SessionModelChangeData, ) -from .testharness import E2ETestContext +from .testharness import E2ETestContext, wait_for_event pytestmark = pytest.mark.asyncio(loop_scope="module") @@ -55,22 +55,6 @@ async def mode_ctx(ctx: E2ETestContext): return ctx -async def _wait_for_event(session, predicate, timeout: float = 30.0): - """Wait for the first session event matching predicate.""" - loop = asyncio.get_event_loop() - fut: asyncio.Future = loop.create_future() - - def on_event(event): - if not fut.done() and predicate(event): - fut.set_result(event) - - unsubscribe = session.on(on_event) - try: - return await asyncio.wait_for(fut, timeout=timeout) - finally: - unsubscribe() - - class TestModeHandlers: async def test_should_invoke_exit_plan_mode_handler_when_model_uses_tool( self, mode_ctx: E2ETestContext @@ -92,27 +76,23 @@ async def on_exit_plan_mode_request(request, invocation): on_exit_plan_mode_request=on_exit_plan_mode_request, ) - try: - requested_event = asyncio.create_task( - _wait_for_event( - session, - lambda event: ( - isinstance(event.data, ExitPlanModeRequestedData) - and event.data.summary == PLAN_SUMMARY - ), - ) - ) - completed_event = asyncio.create_task( - _wait_for_event( - session, - lambda event: ( - isinstance(event.data, ExitPlanModeCompletedData) - and event.data.approved is True - and event.data.selected_action == ExitPlanModeAction.INTERACTIVE - ), - ) - ) + requested_event = wait_for_event( + session, + lambda event: ( + isinstance(event.data, ExitPlanModeRequestedData) + and event.data.summary == PLAN_SUMMARY + ), + ) + completed_event = wait_for_event( + session, + lambda event: ( + isinstance(event.data, ExitPlanModeCompletedData) + and event.data.approved is True + and event.data.selected_action == ExitPlanModeAction.INTERACTIVE + ), + ) + try: await session.rpc.mode.set(ModeSetRequest(mode=SessionMode.PLAN)) response = await session.send_and_wait(PLAN_PROMPT) @@ -132,6 +112,9 @@ async def on_exit_plan_mode_request(request, invocation): assert completed.data.feedback == "Approved by the Python E2E test" assert response is not None finally: + requested_event.cancel() + completed_event.cancel() + await asyncio.gather(requested_event, completed_event, return_exceptions=True) await session.disconnect() async def test_should_invoke_auto_mode_switch_handler_when_rate_limited( @@ -150,42 +133,34 @@ async def on_auto_mode_switch_request(request, invocation): on_auto_mode_switch_request=on_auto_mode_switch_request, ) - try: - requested_event = asyncio.create_task( - _wait_for_event( - session, - lambda event: ( - isinstance(event.data, AutoModeSwitchRequestedData) - and event.data.error_code == "user_weekly_rate_limited" - and event.data.retry_after_seconds == 1 - ), - ) - ) - completed_event = asyncio.create_task( - _wait_for_event( - session, - lambda event: ( - isinstance(event.data, AutoModeSwitchCompletedData) - and event.data.response == AutoModeSwitchResponse.YES - ), - ) - ) - model_change_event = asyncio.create_task( - _wait_for_event( - session, - lambda event: ( - isinstance(event.data, SessionModelChangeData) - and event.data.cause == "rate_limit_auto_switch" - ), - ) - ) - idle_event = asyncio.create_task( - _wait_for_event( - session, - lambda event: isinstance(event.data, SessionIdleData), - ) - ) + requested_event = wait_for_event( + session, + lambda event: ( + isinstance(event.data, AutoModeSwitchRequestedData) + and event.data.error_code == "user_weekly_rate_limited" + and event.data.retry_after_seconds == 1 + ), + ) + completed_event = wait_for_event( + session, + lambda event: ( + isinstance(event.data, AutoModeSwitchCompletedData) + and event.data.response == AutoModeSwitchResponse.YES + ), + ) + model_change_event = wait_for_event( + session, + lambda event: ( + isinstance(event.data, SessionModelChangeData) + and event.data.cause == "rate_limit_auto_switch" + ), + ) + idle_event = wait_for_event( + session, + lambda event: isinstance(event.data, SessionIdleData), + ) + try: message_id = await session.send(AUTO_MODE_PROMPT) assert message_id @@ -206,4 +181,8 @@ async def on_auto_mode_switch_request(request, invocation): assert request["errorCode"] == "user_weekly_rate_limited" assert request["retryAfterSeconds"] == 1 finally: + waiters = [requested_event, completed_event, model_change_event, idle_event] + for waiter in waiters: + waiter.cancel() + await asyncio.gather(*waiters, return_exceptions=True) await session.disconnect() diff --git a/python/e2e/test_multi_client_e2e.py b/python/e2e/test_multi_client_e2e.py index 1938ddfe89..4840155cc6 100644 --- a/python/e2e/test_multi_client_e2e.py +++ b/python/e2e/test_multi_client_e2e.py @@ -22,7 +22,7 @@ from copilot.session import PermissionHandler, PermissionNoResult from copilot.tools import ToolInvocation -from .testharness import get_final_assistant_message +from .testharness import wait_for_event from .testharness.proxy import CapiProxy pytestmark = pytest.mark.asyncio(loop_scope="module") @@ -187,25 +187,6 @@ async def configure_multi_test(request, mctx): yield -def wait_for_event(session, predicate, timeout: float = 30.0): - loop = asyncio.get_running_loop() - future = loop.create_future() - - def on_event(event): - if not future.done() and predicate(event): - future.set_result(event) - - unsubscribe = session.on(on_event) - - async def wait(): - try: - return await asyncio.wait_for(future, timeout=timeout) - finally: - unsubscribe() - - return loop.create_task(wait()) - - class TestMultiClientBroadcast: async def test_both_clients_see_tool_request_and_completion_events( self, mctx: MultiClientContext @@ -245,11 +226,12 @@ def magic_number(params: SeedParams, invocation: ToolInvocation) -> str: waiters = [client1_requested, client2_requested, client1_completed, client2_completed] # Send a prompt that triggers the custom tool - await session1.send( - "Use the magic_number tool with seed 'hello' and tell me the result" - ) # Use a longer timeout: first multi-client TCP test on Windows CI needs extra time - response = await get_final_assistant_message(session1, timeout=30.0) + response = await session1.send_and_wait( + "Use the magic_number tool with seed 'hello' and tell me the result", + timeout=30.0, + ) + assert response is not None assert "MAGIC_hello_42" in (response.data.content or "") # Both clients should have seen the external_tool.requested and completed events @@ -296,8 +278,10 @@ async def test_one_client_approves_permission_and_both_see_the_result( waiters = [client1_requested, client2_requested, client1_completed, client2_completed] # Send a prompt that triggers a write operation (requires permission) - await session1.send("Create a file called hello.txt containing the text 'hello world'") - response = await get_final_assistant_message(session1) + response = await session1.send_and_wait( + "Create a file called hello.txt containing the text 'hello world'", timeout=10.0 + ) + assert response is not None assert response.data.content # Client 1 should have handled permission requests @@ -415,16 +399,18 @@ def currency_lookup(params: CountryCodeParams, invocation: ToolInvocation) -> st ) # Send prompts sequentially to avoid nondeterministic tool_call ordering - await session1.send( - "Use the city_lookup tool with countryCode 'US' and tell me the result." + response1 = await session1.send_and_wait( + "Use the city_lookup tool with countryCode 'US' and tell me the result.", + timeout=10.0, ) - response1 = await get_final_assistant_message(session1) + assert response1 is not None assert "CITY_FOR_US" in (response1.data.content or "") - await session1.send( - "Now use the currency_lookup tool with countryCode 'US' and tell me the result." + response2 = await session1.send_and_wait( + "Now use the currency_lookup tool with countryCode 'US' and tell me the result.", + timeout=10.0, ) - response2 = await get_final_assistant_message(session1) + assert response2 is not None assert "CURRENCY_FOR_US" in (response2.data.content or "") await session2.disconnect() @@ -464,12 +450,16 @@ def ephemeral_tool(params: InputParams, invocation: ToolInvocation) -> str: # Verify both tools work before disconnect. # Sequential prompts avoid nondeterministic tool_call ordering. - await session1.send("Use the stable_tool with input 'test1' and tell me the result.") - stable_response = await get_final_assistant_message(session1) + stable_response = await session1.send_and_wait( + "Use the stable_tool with input 'test1' and tell me the result.", timeout=10.0 + ) + assert stable_response is not None assert "STABLE_test1" in (stable_response.data.content or "") - await session1.send("Use the ephemeral_tool with input 'test2' and tell me the result.") - ephemeral_response = await get_final_assistant_message(session1) + ephemeral_response = await session1.send_and_wait( + "Use the ephemeral_tool with input 'test2' and tell me the result.", timeout=10.0 + ) + assert ephemeral_response is not None assert "EPHEMERAL_test2" in (ephemeral_response.data.content or "") # Force disconnect client 2 without destroying the shared session @@ -487,12 +477,13 @@ def ephemeral_tool(params: InputParams, invocation: ToolInvocation) -> str: ) # Now only stable_tool should be available - await session1.send( + after_response = await session1.send_and_wait( "Use the stable_tool with input 'still_here'." " Also try using ephemeral_tool" - " if it is available." + " if it is available.", + timeout=10.0, ) - after_response = await get_final_assistant_message(session1) + assert after_response is not None assert "STABLE_still_here" in (after_response.data.content or "") # ephemeral_tool should NOT have produced a result assert "EPHEMERAL_" not in (after_response.data.content or "") diff --git a/python/e2e/test_pending_work_resume_e2e.py b/python/e2e/test_pending_work_resume_e2e.py index 5b6d978f31..0a09c37c59 100644 --- a/python/e2e/test_pending_work_resume_e2e.py +++ b/python/e2e/test_pending_work_resume_e2e.py @@ -11,7 +11,6 @@ from __future__ import annotations import asyncio -from typing import Any import pytest @@ -23,9 +22,16 @@ SessionsCheckInUseRequest, ) from copilot.session import PermissionHandler +from copilot.session_events import ExternalToolRequestedData, PermissionRequestedData from copilot.tools import Tool, ToolInvocation, ToolResult -from .testharness import DEFAULT_GITHUB_TOKEN, E2ETestContext, wait_for_condition +from .testharness import ( + DEFAULT_GITHUB_TOKEN, + E2ETestContext, + get_next_event_of_type, + wait_for_condition, + wait_for_event, +) pytestmark = pytest.mark.asyncio(loop_scope="module") @@ -74,53 +80,6 @@ async def wrapped(invocation: ToolInvocation) -> ToolResult: ) -async def _wait_for_external_tool_requests( - session, tool_names: list[str], timeout: float = PENDING_WORK_TIMEOUT -) -> dict[str, Any]: - """Wait for ExternalToolRequested events for the named tools.""" - expected = set(tool_names) - seen: dict[str, Any] = {} - completed: asyncio.Future = asyncio.get_event_loop().create_future() - - def on_event(event): - if completed.done(): - return - if event.type.value == "external_tool.requested": - tool_name = event.data.tool_name - if tool_name in expected and tool_name not in seen: - seen[tool_name] = event - if len(seen) == len(expected): - completed.set_result(dict(seen)) - elif event.type.value == "session.error": - msg = event.data.message or "session error" - completed.set_exception(RuntimeError(msg)) - - unsubscribe = session.on(on_event) - try: - return await asyncio.wait_for(completed, timeout=timeout) - finally: - unsubscribe() - - -async def _wait_for_permission_request(session, timeout: float = PENDING_WORK_TIMEOUT) -> Any: - completed: asyncio.Future = asyncio.get_event_loop().create_future() - - def on_event(event): - if completed.done(): - return - if event.type.value == "permission.requested": - completed.set_result(event) - elif event.type.value == "session.error": - msg = event.data.message or "session error" - completed.set_exception(RuntimeError(msg)) - - unsubscribe = session.on(on_event) - try: - return await asyncio.wait_for(completed, timeout=timeout) - finally: - unsubscribe() - - async def _safe_force_stop(client: CopilotClient) -> None: try: await client.stop() @@ -160,13 +119,16 @@ def original_tool_handler(args): ) session_id = session1.session_id + permission_event_task = get_next_event_of_type( + session1, "permission.requested", timeout=PENDING_WORK_TIMEOUT + ) try: - permission_event_task = asyncio.create_task(_wait_for_permission_request(session1)) await session1.send( "Use resume_permission_tool with value 'alpha', then reply with the result." ) _ = await captured_request permission_event = await permission_event_task + assert isinstance(permission_event.data, PermissionRequestedData) # Force-stop the suspended client without releasing the in-flight # permission so the request remains pending in the runtime. @@ -204,6 +166,8 @@ def resumed_tool_handler(args): finally: await _safe_force_stop(resumed_client) finally: + permission_event_task.cancel() + await asyncio.gather(permission_event_task, return_exceptions=True) if not release_original.done(): release_original.set_result(PermissionDecisionUserNotAvailable()) finally: @@ -237,14 +201,21 @@ async def blocking_external_tool(args): ) session_id = session1.session_id + tool_request_task = wait_for_event( + session1, + lambda event: ( + isinstance(event.data, ExternalToolRequestedData) + and event.data.tool_name == "resume_external_tool" + ), + timeout=PENDING_WORK_TIMEOUT, + fail_on_session_error=True, + ) try: - tool_request_task = asyncio.create_task( - _wait_for_external_tool_requests(session1, ["resume_external_tool"]) - ) await session1.send( "Use resume_external_tool with value 'beta', then reply with the result." ) - tool_events = await tool_request_task + tool_event = await tool_request_task + assert isinstance(tool_event.data, ExternalToolRequestedData) assert (await asyncio.wait_for(tool_started, PENDING_WORK_TIMEOUT)) == "beta" await suspended_client.force_stop() @@ -263,7 +234,7 @@ async def blocking_external_tool(args): tool_result = await session2.rpc.tools.handle_pending_tool_call( HandlePendingToolCallRequest( - request_id=tool_events["resume_external_tool"].data.request_id, + request_id=tool_event.data.request_id, result="EXTERNAL_RESUMED_BETA", ) ) @@ -273,6 +244,8 @@ async def blocking_external_tool(args): finally: await _safe_force_stop(resumed_client) finally: + tool_request_task.cancel() + await asyncio.gather(tool_request_task, return_exceptions=True) if not release_original.done(): release_original.set_result("ORIGINAL_SHOULD_NOT_WIN") finally: @@ -315,17 +288,32 @@ async def tool_b(args): ) session_id = session1.session_id + tool_a_request = wait_for_event( + session1, + lambda event: ( + isinstance(event.data, ExternalToolRequestedData) + and event.data.tool_name == "pending_lookup_a" + ), + timeout=PENDING_WORK_TIMEOUT, + fail_on_session_error=True, + ) + tool_b_request = wait_for_event( + session1, + lambda event: ( + isinstance(event.data, ExternalToolRequestedData) + and event.data.tool_name == "pending_lookup_b" + ), + timeout=PENDING_WORK_TIMEOUT, + fail_on_session_error=True, + ) try: - tool_requests_task = asyncio.create_task( - _wait_for_external_tool_requests( - session1, ["pending_lookup_a", "pending_lookup_b"] - ) - ) await session1.send( "Call pending_lookup_a with value 'alpha' and " "pending_lookup_b with value 'beta', then reply with both results." ) - tool_events = await tool_requests_task + tool_a_event, tool_b_event = await asyncio.gather(tool_a_request, tool_b_request) + assert isinstance(tool_a_event.data, ExternalToolRequestedData) + assert isinstance(tool_b_event.data, ExternalToolRequestedData) await asyncio.wait_for( asyncio.gather(tool_a_started, tool_b_started), PENDING_WORK_TIMEOUT ) @@ -348,14 +336,14 @@ async def tool_b(args): result_b = await session2.rpc.tools.handle_pending_tool_call( HandlePendingToolCallRequest( - request_id=tool_events["pending_lookup_b"].data.request_id, + request_id=tool_b_event.data.request_id, result="PARALLEL_B_BETA", ) ) assert result_b.success result_a = await session2.rpc.tools.handle_pending_tool_call( HandlePendingToolCallRequest( - request_id=tool_events["pending_lookup_a"].data.request_id, + request_id=tool_a_event.data.request_id, result="PARALLEL_A_ALPHA", ) ) @@ -365,6 +353,9 @@ async def tool_b(args): finally: await _safe_force_stop(resumed_client) finally: + tool_a_request.cancel() + tool_b_request.cancel() + await asyncio.gather(tool_a_request, tool_b_request, return_exceptions=True) if not release_a.done(): release_a.set_result("ORIGINAL_A_SHOULD_NOT_WIN") if not release_b.done(): @@ -478,14 +469,21 @@ async def blocking_external_tool(args): ) session_id = session1.session_id + tool_request_task = wait_for_event( + session1, + lambda event: ( + isinstance(event.data, ExternalToolRequestedData) + and event.data.tool_name == "resume_external_tool" + ), + timeout=PENDING_WORK_TIMEOUT, + fail_on_session_error=True, + ) try: - tool_request_task = asyncio.create_task( - _wait_for_external_tool_requests(session1, ["resume_external_tool"]) - ) await session1.send( "Use resume_external_tool with value 'beta', then reply with the result." ) - tool_events = await tool_request_task + tool_event = await tool_request_task + assert isinstance(tool_event.data, ExternalToolRequestedData) assert (await asyncio.wait_for(tool_started, PENDING_WORK_TIMEOUT)) == "beta" if disconnect_original_client: @@ -564,7 +562,7 @@ async def resumed_external_tool(args): # session should still be healthy for new turns. tool_result = await session2.rpc.tools.handle_pending_tool_call( HandlePendingToolCallRequest( - request_id=tool_events["resume_external_tool"].data.request_id, + request_id=tool_event.data.request_id, result="EXTERNAL_RESUMED_BETA", ) ) @@ -582,6 +580,8 @@ async def resumed_external_tool(args): finally: await _safe_force_stop(resumed_client) finally: + tool_request_task.cancel() + await asyncio.gather(tool_request_task, return_exceptions=True) if not release_original.done(): release_original.set_result("ORIGINAL_SHOULD_NOT_WIN") await _safe_force_stop(suspended_client) diff --git a/python/e2e/test_permissions_e2e.py b/python/e2e/test_permissions_e2e.py index c6c644c934..472b958451 100644 --- a/python/e2e/test_permissions_e2e.py +++ b/python/e2e/test_permissions_e2e.py @@ -321,9 +321,10 @@ def on_event(event): add_event("tool-complete", event.data.tool_call_id) unsubscribe = session.on(on_event) + response_task = asyncio.create_task( + session.send_and_wait("Run 'echo slow_handler_test'", timeout=60.0) + ) try: - asyncio.ensure_future(session.send("Run 'echo slow_handler_test'")) - await asyncio.wait_for(handler_entered, timeout=30.0) target_id = await asyncio.wait_for(target_tool_call_id, timeout=30.0) @@ -334,9 +335,7 @@ def on_event(event): release_handler.set_result(True) - from .testharness.helper import get_final_assistant_message - - message = await get_final_assistant_message(session, timeout=60.0) + message = await response_task perm_start = next( ( @@ -386,6 +385,8 @@ def on_event(event): finally: if not release_handler.done(): release_handler.set_result(True) + response_task.cancel() + await asyncio.gather(response_task, return_exceptions=True) unsubscribe() await session.disconnect() diff --git a/python/e2e/test_rpc_event_side_effects_e2e.py b/python/e2e/test_rpc_event_side_effects_e2e.py index ce3951aacd..c759be2657 100644 --- a/python/e2e/test_rpc_event_side_effects_e2e.py +++ b/python/e2e/test_rpc_event_side_effects_e2e.py @@ -35,22 +35,6 @@ pytestmark = pytest.mark.asyncio(loop_scope="module") -async def _wait_for_event(session, predicate, timeout: float = 15.0): - """Wait for the first session event matching predicate.""" - loop = asyncio.get_event_loop() - fut: asyncio.Future = loop.create_future() - - def on_event(event): - if not fut.done() and predicate(event): - fut.set_result(event) - - unsub = session.on(on_event) - try: - return await asyncio.wait_for(fut, timeout=timeout) - finally: - unsub() - - class TestRpcEventSideEffects: async def test_should_emit_mode_changed_event_when_mode_set(self, ctx: E2ETestContext): session = await ctx.client.create_session( diff --git a/python/e2e/test_session_e2e.py b/python/e2e/test_session_e2e.py index f57b9f5736..160d627a0d 100644 --- a/python/e2e/test_session_e2e.py +++ b/python/e2e/test_session_e2e.py @@ -15,7 +15,6 @@ from .testharness import ( DEFAULT_GITHUB_TOKEN, E2ETestContext, - get_final_assistant_message, get_next_event_of_type, wait_for_condition, ) @@ -63,8 +62,8 @@ async def test_should_create_a_session_with_appended_systemMessage_config( system_message={"mode": "append", "content": system_message_suffix}, ) - await session.send("What is your full name?") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait("What is your full name?", timeout=10.0) + assert assistant_message is not None assert "GitHub" in assistant_message.data.content assert "Have a nice day!" in assistant_message.data.content @@ -83,8 +82,8 @@ async def test_should_create_a_session_with_replaced_systemMessage_config( system_message={"mode": "replace", "content": test_system_message}, ) - await session.send("What is your full name?") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait("What is your full name?", timeout=10.0) + assert assistant_message is not None assert "GitHub" not in assistant_message.data.content assert "Testy" in assistant_message.data.content @@ -235,8 +234,12 @@ async def test_should_resume_a_session_using_the_same_client(self, ctx: E2ETestC session_id, on_permission_request=PermissionHandler.approve_all ) assert session2.session_id == session_id - answer2 = await get_final_assistant_message(session2, already_idle=True) - assert "2" in answer2.data.content + # The completed turn's assistant message is durable; session.idle is not. + messages = await session2.get_events() + assert not any(message.type.value == "session.error" for message in messages) + answers = [message for message in messages if message.type.value == "assistant.message"] + assert answers + assert "2" in answers[-1].data.content # Can continue the conversation statefully answer3 = await session2.send_and_wait("Now if you double that, what do you get?") @@ -582,26 +585,27 @@ async def test_should_abort_a_session(self, ctx: E2ETestContext): ) # Set up event listeners BEFORE sending to avoid race conditions - wait_for_tool_start = asyncio.create_task( - get_next_event_of_type(session, "tool.execution_start", timeout=60.0) - ) - wait_for_session_idle = asyncio.create_task( - get_next_event_of_type(session, "session.idle", timeout=30.0) - ) + wait_for_tool_start = get_next_event_of_type(session, "tool.execution_start", timeout=60.0) + wait_for_session_idle = get_next_event_of_type(session, "session.idle", timeout=30.0) - # Send a message that will trigger a long-running shell command - await session.send( - "run the shell command 'sleep 100' (note this works on both bash and PowerShell)" - ) + try: + # Send a message that will trigger a long-running shell command + await session.send( + "run the shell command 'sleep 100' (note this works on both bash and PowerShell)" + ) - # Wait for the tool to start executing - _ = await wait_for_tool_start + # Wait for the tool to start executing + _ = await wait_for_tool_start - # Abort the session while the tool is running - await session.abort() + # Abort the session while the tool is running + await session.abort() - # Wait for session to become idle after abort - _ = await wait_for_session_idle + # Wait for session to become idle after abort + _ = await wait_for_session_idle + finally: + wait_for_tool_start.cancel() + wait_for_session_idle.cancel() + await asyncio.gather(wait_for_tool_start, wait_for_session_idle, return_exceptions=True) # The session should still be alive and usable after abort messages = await session.get_events() @@ -612,11 +616,8 @@ async def test_should_abort_a_session(self, ctx: E2ETestContext): assert len(abort_events) > 0, "Expected an abort event in messages" # We should be able to send another message - wait_for_answer = asyncio.create_task( - get_next_event_of_type(session, "assistant.message", timeout=60.0) - ) - await session.send("What is 2+2?") - answer = await wait_for_answer + answer = await session.send_and_wait("What is 2+2?", timeout=60.0) + assert answer is not None assert "4" in answer.data.content async def test_should_receive_session_events(self, ctx: E2ETestContext): @@ -671,11 +672,13 @@ def on_event(event): assert "assistant.message" in event_types assert "session.idle" in event_types - # Verify the assistant response contains the expected answer. - # session.idle is ephemeral and not in get_events(), but we already - # confirmed idle via the live event handler above. - assistant_message = await get_final_assistant_message(session, already_idle=True) - assert "300" in assistant_message.data.content + # Idle was observed live, so inspect the messages captured for this turn. + assert "session.error" not in event_types + assistant_messages = [ + event for event in received_events if event.type.value == "assistant.message" + ] + assert assistant_messages + assert "300" in assistant_messages[-1].data.content async def test_should_create_session_with_custom_config_dir(self, ctx: E2ETestContext): import os @@ -688,8 +691,8 @@ async def test_should_create_session_with_custom_config_dir(self, ctx: E2ETestCo assert session.session_id # Session should work normally with custom config dir - await session.send("What is 1+1?") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait("What is 1+1?", timeout=10.0) + assert assistant_message is not None assert "2" in assistant_message.data.content async def test_session_log_emits_events_at_all_levels(self, ctx: E2ETestContext): @@ -993,28 +996,35 @@ async def test_send_returns_immediately_while_events_stream_in_background( self, ctx: E2ETestContext ): """`send` returns before the session goes idle; events are streamed.""" + import asyncio + session = await ctx.client.create_session( on_permission_request=PermissionHandler.approve_all, ) - events: list[str] = [] + events = [] def on_event(event): - events.append(event.type.value) + events.append(event) - session.on(on_event) - - # Use a slow command so we can verify send() returns before completion - await session.send("Run 'sleep 2 && echo done'") - - # send() should return before turn completes (no session.idle yet) - assert "session.idle" not in events + unsubscribe = session.on(on_event) + idle_task = get_next_event_of_type(session, "session.idle", timeout=10.0) + try: + # Use a slow command so we can verify send() returns before completion + await session.send("Run 'sleep 2 && echo done'") - message = await get_final_assistant_message(session) - assert "done" in message.data.content - assert "session.idle" in events - assert "assistant.message" in events + # send() should return before turn completes (no session.idle yet) + assert not any(event.type.value == "session.idle" for event in events) - await session.disconnect() + await idle_task + messages = [event for event in events if event.type.value == "assistant.message"] + assert messages + assert "done" in messages[-1].data.content + assert any(event.type.value == "session.idle" for event in events) + finally: + idle_task.cancel() + await asyncio.gather(idle_task, return_exceptions=True) + unsubscribe() + await session.disconnect() async def test_sendandwait_blocks_until_session_idle_and_returns_final_assistant_message( self, ctx: E2ETestContext @@ -1043,24 +1053,24 @@ async def test_sendandwait_throws_on_timeout(self, ctx: E2ETestContext): on_permission_request=PermissionHandler.approve_all, ) - # Start a background wait for session.idle so we can drain after we abort. - idle_task = asyncio.create_task( - get_next_event_of_type(session, "session.idle", timeout=30.0) - ) - - with pytest.raises(TimeoutError) as exc_info: - await session.send_and_wait( - "Run 'sleep 2 && echo done'", - timeout=0.1, - ) - assert "Timeout" in str(exc_info.value) or "timed out" in str(exc_info.value).lower() - - # The timeout only cancels the client-side wait; abort the agent and wait for idle - # so leftover requests don't leak into subsequent tests. - await session.abort() - await idle_task + # Subscribe before sending so even an idle emitted before the abort reply is captured. + idle_task = get_next_event_of_type(session, "session.idle", timeout=30.0) + try: + with pytest.raises(TimeoutError) as exc_info: + await session.send_and_wait( + "Run 'sleep 2 && echo done'", + timeout=0.1, + ) + assert "Timeout" in str(exc_info.value) or "timed out" in str(exc_info.value).lower() - await session.disconnect() + # The timeout only cancels the client-side wait; abort the agent and wait for idle + # so leftover requests don't leak into subsequent tests. + await session.abort() + await idle_task + finally: + idle_task.cancel() + await asyncio.gather(idle_task, return_exceptions=True) + await session.disconnect() async def test_sendandwait_throws_operationcanceledexception_when_token_cancelled( self, ctx: E2ETestContext @@ -1072,12 +1082,8 @@ async def test_sendandwait_throws_operationcanceledexception_when_token_cancelle on_permission_request=PermissionHandler.approve_all, ) - tool_start_task = asyncio.create_task( - get_next_event_of_type(session, "tool.execution_start", timeout=60.0) - ) - idle_task = asyncio.create_task( - get_next_event_of_type(session, "session.idle", timeout=30.0) - ) + tool_start_task = get_next_event_of_type(session, "tool.execution_start", timeout=60.0) + idle_task = get_next_event_of_type(session, "session.idle", timeout=30.0) send_task = asyncio.create_task( session.send_and_wait( @@ -1086,18 +1092,23 @@ async def test_sendandwait_throws_operationcanceledexception_when_token_cancelle ) ) - # Wait for the tool to begin executing before cancelling. - await tool_start_task - - send_task.cancel() - with pytest.raises((asyncio.CancelledError, BaseException)): - await send_task + try: + # Wait for the tool to begin executing before cancelling. + await tool_start_task - # Cancelling only cancels the client-side wait; abort and wait for idle. - await session.abort() - await idle_task + send_task.cancel() + with pytest.raises((asyncio.CancelledError, BaseException)): + await send_task - await session.disconnect() + # Cancelling only cancels the client-side wait; abort and wait for idle. + await session.abort() + await idle_task + finally: + tool_start_task.cancel() + idle_task.cancel() + send_task.cancel() + await asyncio.gather(tool_start_task, idle_task, send_task, return_exceptions=True) + await session.disconnect() async def test_should_set_model_on_existing_session(self, ctx: E2ETestContext): """`set_model` emits a session.model_change event with the new model.""" diff --git a/python/e2e/test_session_todos_changed_e2e.py b/python/e2e/test_session_todos_changed_e2e.py index 8911ffb117..fd6d92f6c8 100644 --- a/python/e2e/test_session_todos_changed_e2e.py +++ b/python/e2e/test_session_todos_changed_e2e.py @@ -31,11 +31,13 @@ async def test_fires_session_todos_changed_and_exposes_rows_and_dependencies( async with await ctx.client.create_session( on_permission_request=PermissionHandler.approve_all, ) as session: - todos_changed = asyncio.create_task( - get_next_event_of_type(session, "session.todos_changed", timeout=120.0) - ) - await session.send_and_wait(PROMPT, timeout=120.0) - await todos_changed + todos_changed = get_next_event_of_type(session, "session.todos_changed", timeout=120.0) + try: + await session.send_and_wait(PROMPT, timeout=120.0) + await todos_changed + finally: + todos_changed.cancel() + await asyncio.gather(todos_changed, return_exceptions=True) result = await session.rpc.plan.read_sql_todos_with_dependencies() ids = sorted(row.id for row in result.rows if row.id) diff --git a/python/e2e/test_telemetry_e2e.py b/python/e2e/test_telemetry_e2e.py index 8b9c82abef..56031a14ec 100644 --- a/python/e2e/test_telemetry_e2e.py +++ b/python/e2e/test_telemetry_e2e.py @@ -26,7 +26,7 @@ from copilot.session import PermissionHandler from copilot.tools import Tool, ToolInvocation, ToolResult -from .testharness import E2ETestContext, get_final_assistant_message +from .testharness import E2ETestContext pytestmark = pytest.mark.asyncio(loop_scope="module") @@ -102,8 +102,8 @@ def echo(invocation: ToolInvocation) -> ToolResult: ) session_id = session.session_id - await session.send(prompt) - answer = await get_final_assistant_message(session, timeout=60.0) + answer = await session.send_and_wait(prompt, timeout=60.0) + assert answer is not None assert "TELEMETRY_E2E_DONE" in (answer.data.content or "") await session.disconnect() diff --git a/python/e2e/test_tool_results_e2e.py b/python/e2e/test_tool_results_e2e.py index 41d1967bc9..f2f646f648 100644 --- a/python/e2e/test_tool_results_e2e.py +++ b/python/e2e/test_tool_results_e2e.py @@ -9,7 +9,7 @@ from copilot.session import PermissionHandler from copilot.tools import ToolInvocation, ToolResult -from .testharness import E2ETestContext, get_final_assistant_message +from .testharness import E2ETestContext pytestmark = pytest.mark.asyncio(loop_scope="module") @@ -33,8 +33,10 @@ def get_weather(params: WeatherParams, invocation: ToolInvocation) -> ToolResult ) try: - await session.send("What's the weather in Paris?") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "What's the weather in Paris?", timeout=10.0 + ) + assert assistant_message is not None assert ( "sunny" in assistant_message.data.content.lower() or "72" in assistant_message.data.content @@ -87,8 +89,10 @@ def analyze_code(params: AnalyzeParams, invocation: ToolInvocation) -> ToolResul ) try: - await session.send("Analyze the file main.ts for issues.") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "Analyze the file main.ts for issues.", timeout=10.0 + ) + assert assistant_message is not None assert "no issues" in assistant_message.data.content.lower() # Verify the LLM received just textResultForLlm, not stringified JSON diff --git a/python/e2e/test_tools_e2e.py b/python/e2e/test_tools_e2e.py index 1421dbaf40..303ddc3ac9 100644 --- a/python/e2e/test_tools_e2e.py +++ b/python/e2e/test_tools_e2e.py @@ -13,7 +13,7 @@ from copilot.session import PermissionHandler, PermissionNoResult from copilot.tools import Tool, ToolInvocation, ToolResult -from .testharness import E2ETestContext, get_final_assistant_message +from .testharness import E2ETestContext pytestmark = pytest.mark.asyncio(loop_scope="module") @@ -28,8 +28,10 @@ async def test_invokes_built_in_tools(self, ctx: E2ETestContext): on_permission_request=PermissionHandler.approve_all ) - await session.send("What's the first line of README.md in this directory?") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "What's the first line of README.md in this directory?", timeout=10.0 + ) + assert assistant_message is not None assert "ELIZA" in assistant_message.data.content async def test_invokes_custom_tool(self, ctx: E2ETestContext): @@ -44,8 +46,10 @@ def encrypt_string(params: EncryptParams, invocation: ToolInvocation) -> str: on_permission_request=PermissionHandler.approve_all, tools=[encrypt_string] ) - await session.send("Use encrypt_string to encrypt this string: Hello") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "Use encrypt_string to encrypt this string: Hello", timeout=10.0 + ) + assert assistant_message is not None assert "HELLO" in assistant_message.data.content async def test_low_level_tool_definition(self, ctx: E2ETestContext): @@ -83,8 +87,8 @@ def search_items(params: SearchArgs, invocation: ToolInvocation) -> str: "First, set the current phase to 'analyzing'. Then search for items with " "keyword 'copilot'. Report the phase and search results." ) - await session.send(prompt) - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait(prompt, timeout=10.0) + assert assistant_message is not None content = assistant_message.data.content or "" assert content != "" assert "analyzing" in content.lower() @@ -100,8 +104,10 @@ def get_user_location() -> str: on_permission_request=PermissionHandler.approve_all, tools=[get_user_location] ) - await session.send("What is my location? If you can't find out, just say 'unknown'.") - answer = await get_final_assistant_message(session) + answer = await session.send_and_wait( + "What is my location? If you can't find out, just say 'unknown'.", timeout=10.0 + ) + assert answer is not None # Check the underlying traffic traffic = await ctx.get_exchanges() @@ -164,12 +170,13 @@ def db_query(params: DbQueryParams, invocation: ToolInvocation) -> list[City]: ) expected_session_id = session.session_id - await session.send( + assistant_message = await session.send_and_wait( "Perform a DB query for the 'cities' table using IDs 12 and 19, " - "sorting ascending. Reply only with lines of the form: [cityname] [population]" + "sorting ascending. Reply only with lines of the form: [cityname] [population]", + timeout=10.0, ) - assistant_message = await get_final_assistant_message(session) + assert assistant_message is not None response_content = assistant_message.data.content or "" assert response_content != "" @@ -201,8 +208,10 @@ def tracking_handler(request, invocation): on_permission_request=tracking_handler, tools=[safe_lookup] ) - await session.send("Use safe_lookup to look up 'test123'") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "Use safe_lookup to look up 'test123'", timeout=10.0 + ) + assert assistant_message is not None assert "RESULT: test123" in assistant_message.data.content assert not did_run_permission_request @@ -222,8 +231,10 @@ def custom_grep(params: GrepParams, invocation: ToolInvocation) -> str: on_permission_request=PermissionHandler.approve_all, tools=[custom_grep] ) - await session.send("Use grep to search for the word 'hello'") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "Use grep to search for the word 'hello'", timeout=10.0 + ) + assert assistant_message is not None assert "CUSTOM_GREP_RESULT" in assistant_message.data.content async def test_invokes_custom_tool_with_permission_handler(self, ctx: E2ETestContext): @@ -244,8 +255,10 @@ def on_permission_request(request, invocation): on_permission_request=on_permission_request, tools=[encrypt_string] ) - await session.send("Use encrypt_string to encrypt this string: Hello") - assistant_message = await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "Use encrypt_string to encrypt this string: Hello", timeout=10.0 + ) + assert assistant_message is not None assert "HELLO" in assistant_message.data.content # Should have received a custom-tool permission request @@ -272,8 +285,10 @@ def on_permission_request(request, invocation): on_permission_request=on_permission_request, tools=[encrypt_string] ) - await session.send("Use encrypt_string to encrypt this string: Hello") - await get_final_assistant_message(session) + assistant_message = await session.send_and_wait( + "Use encrypt_string to encrypt this string: Hello", timeout=10.0 + ) + assert assistant_message is not None # The tool handler should NOT have been called since permission was denied assert not tool_handler_called @@ -328,9 +343,10 @@ def lookup_country(invocation: ToolInvocation) -> ToolResult: ) try: - await session.send( + assistant_message = await session.send_and_wait( "Use lookup_city with 'Paris' and lookup_country with 'France' at the same time," - " then combine both results in your reply." + " then combine both results in your reply.", + timeout=60.0, ) city_result = await asyncio.wait_for(city_called, timeout=60.0) @@ -338,7 +354,6 @@ def lookup_country(invocation: ToolInvocation) -> ToolResult: assert city_result == "Paris" assert country_result == "France" - assistant_message = await get_final_assistant_message(session, timeout=60.0) assert assistant_message is not None content = assistant_message.data.content or "" assert "CITY_PARIS" in content diff --git a/python/e2e/testharness/__init__.py b/python/e2e/testharness/__init__.py index 75ce76d9c5..cfc33f29f7 100644 --- a/python/e2e/testharness/__init__.py +++ b/python/e2e/testharness/__init__.py @@ -1,7 +1,7 @@ """Test harness for E2E tests.""" from .context import CLI_PATH, DEFAULT_GITHUB_TOKEN, E2ETestContext, is_inprocess_transport -from .helper import get_final_assistant_message, get_next_event_of_type, wait_for_condition +from .helper import get_next_event_of_type, wait_for_condition, wait_for_event from .proxy import CapiProxy __all__ = [ @@ -9,8 +9,8 @@ "DEFAULT_GITHUB_TOKEN", "E2ETestContext", "CapiProxy", - "get_final_assistant_message", "get_next_event_of_type", "wait_for_condition", + "wait_for_event", "is_inprocess_transport", ] diff --git a/python/e2e/testharness/helper.py b/python/e2e/testharness/helper.py index 7933dd9ec8..ed50d17efb 100644 --- a/python/e2e/testharness/helper.py +++ b/python/e2e/testharness/helper.py @@ -8,104 +8,8 @@ import time from collections.abc import Awaitable, Callable -from copilot import CopilotSession -from copilot.session_events import ( - AssistantMessageData, - SessionErrorData, - SessionIdleData, -) - - -async def get_final_assistant_message( - session: CopilotSession, timeout: float = 10.0, already_idle: bool = False -): - """ - Wait for and return the final assistant message from a session turn. - - Args: - session: The session to wait on - timeout: Maximum time to wait in seconds - - Returns: - The final assistant message event - - Raises: - TimeoutError: If no message arrives within timeout - RuntimeError: If a session error occurs - """ - result_future: asyncio.Future = asyncio.get_event_loop().create_future() - - final_assistant_message = None - - def on_event(event): - nonlocal final_assistant_message - if result_future.done(): - return - - match event.data: - case AssistantMessageData(): - final_assistant_message = event - case SessionIdleData(): - if final_assistant_message is not None: - result_future.set_result(final_assistant_message) - case SessionErrorData() as data: - msg = data.message if data.message else "session error" - result_future.set_exception(RuntimeError(msg)) - - # Subscribe to future events - unsubscribe = session.on(on_event) - - try: - # Also check existing messages in case the response already arrived - existing = await _get_existing_final_response(session, already_idle) - if existing is not None: - return existing - - return await asyncio.wait_for(result_future, timeout=timeout) - finally: - unsubscribe() - - -async def _get_existing_final_response(session: CopilotSession, already_idle: bool = False): - """Check existing messages for a final response.""" - messages = await session.get_events() - - # Find last user message - final_user_message_index = -1 - for i in range(len(messages) - 1, -1, -1): - if messages[i].type.value == "user.message": - final_user_message_index = i - break - - if final_user_message_index < 0: - current_turn_messages = messages - else: - current_turn_messages = messages[final_user_message_index:] - - # Check for errors - for msg in current_turn_messages: - match msg.data: - case SessionErrorData() as data: - err_msg = data.message if data.message else "session error" - raise RuntimeError(err_msg) - - # Find session.idle and get last assistant message before it - if already_idle: - session_idle_index = len(current_turn_messages) - else: - session_idle_index = -1 - for i, msg in enumerate(current_turn_messages): - if msg.type.value == "session.idle": - session_idle_index = i - break - - if session_idle_index != -1: - # Find last assistant.message before session.idle - for i in range(session_idle_index - 1, -1, -1): - if current_turn_messages[i].type.value == "assistant.message": - return current_turn_messages[i] - - return None +from _session_test_helpers import get_next_event_of_type as get_next_event_of_type +from _session_test_helpers import wait_for_event as wait_for_event def write_file(work_dir: str, filename: str, content: str) -> str: @@ -165,41 +69,3 @@ async def wait_for_condition( if result: return raise TimeoutError(timeout_message) - - -async def get_next_event_of_type(session: CopilotSession, event_type: str, timeout: float = 30.0): - """ - Wait for and return the next event of a specific type from a session. - - Args: - session: The session to wait on - event_type: The event type to wait for (e.g., "tool.execution_start", "session.idle") - timeout: Maximum time to wait in seconds - - Returns: - The matching event - - Raises: - TimeoutError: If no matching event arrives within timeout - RuntimeError: If a session error occurs - """ - result_future: asyncio.Future = asyncio.get_event_loop().create_future() - - def on_event(event): - if result_future.done(): - return - - if event.type.value == event_type: - result_future.set_result(event) - else: - match event.data: - case SessionErrorData() as data: - msg = data.message if data.message else "session error" - result_future.set_exception(RuntimeError(msg)) - - unsubscribe = session.on(on_event) - - try: - return await asyncio.wait_for(result_future, timeout=timeout) - finally: - unsubscribe() diff --git a/python/test_session.py b/python/test_session.py index 8fcecba439..5750364bb5 100644 --- a/python/test_session.py +++ b/python/test_session.py @@ -8,6 +8,7 @@ import pytest +from _session_test_helpers import get_next_event_of_type, wait_for_event from copilot import AgentMessageSource, MessageSource from copilot.session import Attachment, CopilotSession from copilot.session_events import ( @@ -57,6 +58,186 @@ def _event(data, event_type: SessionEventType) -> SessionEvent: ) +@pytest.mark.parametrize("use_send_and_wait", [False, True]) +@pytest.mark.asyncio +async def test_completion_captures_live_idle_before_send_reply_without_idle_in_history( + use_send_and_wait, +): + client = Mock() + session = CopilotSession("session-1", client) + intermediate = _event( + AssistantMessageData(content="working", message_id="assistant-1"), + SessionEventType.ASSISTANT_MESSAGE, + ) + assistant = _event( + AssistantMessageData(content="done", message_id="assistant-2"), + SessionEventType.ASSISTANT_MESSAGE, + ) + idle = _event(SessionIdleData(), SessionEventType.SESSION_IDLE) + idle.ephemeral = True + history = [intermediate, assistant] + received = [] + unsubscribe = session.on(received.append) + + async def respond(method, params): + assert params["sessionId"] == session.session_id + if method == "session.getMessages": + return {"events": [event.to_dict() for event in history]} + assert method == "session.send" + # No suspension: even a create_task(async_waiter()) cannot subscribe in time. + session._dispatch_event(intermediate) + session._dispatch_event(assistant) + session._dispatch_event(idle) + return {"messageId": "message-1"} + + client.request = AsyncMock(side_effect=respond) + try: + if use_send_and_wait: + message = await session.send_and_wait("hello", timeout=1) + assert message is not None + assert message is assistant + else: + idle_task = get_next_event_of_type(session, "session.idle", timeout=1) + try: + assert await session.send("hello") == "message-1" + assert received == [intermediate, assistant, idle] + assert await idle_task is idle + messages = [ + event for event in received if event.type == SessionEventType.ASSISTANT_MESSAGE + ] + assert messages[-1] is assistant + finally: + idle_task.cancel() + await asyncio.gather(idle_task, return_exceptions=True) + + client.request.assert_awaited_once() + persisted = await session.get_events() + assert [event.type for event in persisted] == [ + SessionEventType.ASSISTANT_MESSAGE, + SessionEventType.ASSISTANT_MESSAGE, + ] + assert persisted[-1].data.content == "done" + finally: + unsubscribe() + + +@pytest.mark.asyncio +async def test_event_waiters_capture_abort_and_recovery_before_rpc_replies(): + client = Mock() + session = CopilotSession("session-1", client) + idle = _event(SessionIdleData(), SessionEventType.SESSION_IDLE) + assistant = _event( + AssistantMessageData(content="recovered", message_id="assistant-1"), + SessionEventType.ASSISTANT_MESSAGE, + ) + + async def respond(method, params): + if method == "session.abort": + session._dispatch_event(idle) + return {} + assert method == "session.send" + session._dispatch_event(assistant) + session._dispatch_event(idle) + return {"messageId": "message-1"} + + client.request = AsyncMock(side_effect=respond) + aborted = get_next_event_of_type(session, "session.idle", timeout=1) + try: + await session.abort() + assert await aborted is idle + finally: + aborted.cancel() + await asyncio.gather(aborted, return_exceptions=True) + + recovery = wait_for_event( + session, + lambda event: ( + isinstance(event.data, AssistantMessageData) and event.data.content == "recovered" + ), + timeout=1, + ) + recovered_idle = get_next_event_of_type(session, "session.idle", timeout=1) + try: + await session.send("recover") + assert await recovery is assistant + assert await recovered_idle is idle + finally: + recovery.cancel() + recovered_idle.cancel() + await asyncio.gather(recovery, recovered_idle, return_exceptions=True) + + +@pytest.mark.parametrize( + "outcome", ["success", "error", "timeout", "cancel-before-start", "cancel-after-start"] +) +@pytest.mark.asyncio +async def test_event_waiter_unsubscribes_for_every_outcome(outcome): + session = Mock(spec=CopilotSession) + unsubscribe = session.on.return_value + waiter = get_next_event_of_type( + session, "session.idle", timeout=0 if outcome == "timeout" else 1 + ) + session.on.assert_called_once() + on_event = session.on.call_args.args[0] + + if outcome == "success": + idle = _event(SessionIdleData(), SessionEventType.SESSION_IDLE) + on_event(idle) + on_event(idle) + assert await waiter is idle + elif outcome == "error": + on_event( + _event( + SessionErrorData(error_type="notification", message="turn failed"), + SessionEventType.SESSION_ERROR, + ) + ) + with pytest.raises(RuntimeError, match="turn failed"): + await waiter + elif outcome == "timeout": + with pytest.raises(TimeoutError): + await waiter + else: + if outcome == "cancel-after-start": + loop = asyncio.get_running_loop() + started = loop.create_future() + loop.call_soon(started.set_result, None) + await started + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + + unsubscribe.assert_called_once() + + +@pytest.mark.parametrize("fail_on_session_error", [None, False, True]) +@pytest.mark.asyncio +async def test_event_waiter_preserves_caller_error_policy(fail_on_session_error): + session = Mock(spec=CopilotSession) + options = ( + {} if fail_on_session_error is None else {"fail_on_session_error": fail_on_session_error} + ) + waiter = wait_for_event( + session, lambda event: isinstance(event.data, SessionIdleData), timeout=1, **options + ) + on_event = session.on.call_args.args[0] + on_event( + _event( + SessionErrorData(error_type="rate_limit", message="rate limited"), + SessionEventType.SESSION_ERROR, + ) + ) + idle = _event(SessionIdleData(), SessionEventType.SESSION_IDLE) + on_event(idle) + + if fail_on_session_error: + with pytest.raises(RuntimeError, match="rate limited"): + await waiter + else: + assert await waiter is idle + session.on.return_value.assert_called_once() + + @pytest.mark.asyncio async def test_send_omits_source_for_plain_human_prompt(monkeypatch): monkeypatch.setattr("copilot.session.get_trace_context", lambda: {}) From df3457bdc6e983e60635efb5c614a6d9273d7bd6 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 09:38:17 -0400 Subject: [PATCH 06/14] Wait for controlled Node send RPC before completing it CI exposed five regression fixtures completing a fake send before trace-context setup reached the RPC handler. Await the existing sendStarted fence so the RPC resolver is installed before delivering its response, without sleeps or timeout changes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- nodejs/test/session-send-and-wait.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nodejs/test/session-send-and-wait.test.ts b/nodejs/test/session-send-and-wait.test.ts index 07d67fe9fe..d88b80c896 100644 --- a/nodejs/test/session-send-and-wait.test.ts +++ b/nodejs/test/session-send-and-wait.test.ts @@ -227,7 +227,7 @@ describe("completion subscriptions", () => { describe("withFinalAssistantMessage", () => { it("does not accept a prior turn's message when the new turn has no assistant output", async () => { - const { session, resolveSend } = controlledSession( + const { session, sendStarted, resolveSend } = controlledSession( [assistantMessage("old turn")], (session) => session._dispatchEvent(sessionEvent("session.idle")) ); @@ -236,13 +236,15 @@ describe("withFinalAssistantMessage", () => { "Received session.idle without a preceding assistant.message" ); + await sendStarted; resolveSend(); await outcome; }); it("does not complete on an assistant message or an autopilot continuation", async () => { - const { session, resolveSend } = controlledSession(); + const { session, sendStarted, resolveSend } = controlledSession(); const pending = withFinalAssistantMessage(session, () => session.send({ prompt: "hi" })); + await sendStarted; resolveSend(); session._dispatchEvent(assistantMessage("continuing")); @@ -257,7 +259,7 @@ describe("withFinalAssistantMessage", () => { it.each(["idle", "session.error", "send rejection", "trigger throw"] as const)( "removes the completion subscription after %s", async (outcome) => { - const { session, resolveSend, rejectSend } = controlledSession(); + const { session, sendStarted, resolveSend, rejectSend } = controlledSession(); const originalOn = session.on.bind(session); const unsubscribe = vi.fn<() => void>(); vi.spyOn(session, "on").mockImplementation((handler) => { @@ -283,6 +285,10 @@ describe("withFinalAssistantMessage", () => { ? expect(pending).resolves.toBe(finalMessage) : expect(pending).rejects.toThrow(errorMessage); + if (outcome !== "trigger throw") { + await sendStarted; + } + if (outcome === "idle") { session._dispatchEvent(finalMessage); session._dispatchEvent(sessionEvent("session.idle")); From 9751b28cc5ecc99ee9083322c257971b1a559249 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 10:45:41 -0400 Subject: [PATCH 07/14] Preserve Python timeout diagnostics before async cleanup Capture suspended await chains, pending RPC metadata, session and transport state, and Python thread stacks at the original pytest-timeout signal. Sample native threads for macOS in-process failures and preserve evidence in xdist reports and CI artifacts without changing timeout or failure semantics. Add deterministic diagnostic regressions and subprocess coverage for xdist reporting, including real POSIX signal timeouts during test calls and fixture teardown. Generated by Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/python-sdk-tests.yml | 9 + python/.gitignore | 1 + python/README.md | 8 + python/e2e/conftest.py | 3 + python/e2e/timeout_diagnostics.py | 211 ++++++++++++++++ python/test_timeout_diagnostics.py | 317 +++++++++++++++++++++++++ 6 files changed, 549 insertions(+) create mode 100644 python/e2e/timeout_diagnostics.py create mode 100644 python/test_timeout_diagnostics.py diff --git a/.github/workflows/python-sdk-tests.yml b/.github/workflows/python-sdk-tests.yml index e51ee3f3c8..4023a8becf 100644 --- a/.github/workflows/python-sdk-tests.yml +++ b/.github/workflows/python-sdk-tests.yml @@ -99,6 +99,15 @@ jobs: # running independent modules concurrently in isolated workers. run: uv run pytest -v -s -n 2 --dist=loadfile + - name: Upload Python timeout diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: python-timeout-${{ matrix.os }}-${{ matrix.transport }} + path: python/.pytest-diagnostics/ + include-hidden-files: true + if-no-files-found: ignore + # JavaScript actions use a glibc-linked Node runtime, so Alpine runs through Docker. test-musl-arm64: name: "Python SDK Tests (Alpine ARM64, ${{ matrix.transport }})" diff --git a/python/.gitignore b/python/.gitignore index 671fe9a8bb..122f34c543 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -49,6 +49,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.pytest-diagnostics/ cover/ # Translations diff --git a/python/README.md b/python/README.md index 2d1d8ca153..7a8726608e 100644 --- a/python/README.md +++ b/python/README.md @@ -1233,3 +1233,11 @@ cd python uv sync uv run pytest ``` + +Signal-based E2E failures from `pytest-timeout` include an **Async timeout diagnostics** report +section with suspended coroutine await chains, pending JSON-RPC request IDs and +methods, session/transport state, and Python thread stacks. The same report is +saved under `python/.pytest-diagnostics/`. macOS in-process timeouts also capture a +one-second native thread sample there. CI uploads these files as +`python-timeout--` artifacts. RPC payloads and arbitrary frame locals +are not included. The existing test timeout and failure behavior are unchanged. diff --git a/python/e2e/conftest.py b/python/e2e/conftest.py index d61fe0d875..6e11d5b2fe 100644 --- a/python/e2e/conftest.py +++ b/python/e2e/conftest.py @@ -10,6 +10,8 @@ import copilot._cli_download as cli_download from .testharness import E2ETestContext, is_inprocess_transport +from .timeout_diagnostics import add_timeout_diagnostics +from .timeout_diagnostics import pytest_timeout_set_timer as pytest_timeout_set_timer # Host-side auth resolution ranks HMAC above the GitHub token, so an ambient # COPILOT_HMAC_KEY (CI sets one as a job-level credential) would be picked over @@ -34,6 +36,7 @@ def pytest_runtest_makereport(item, call): """Track test failures to avoid writing corrupted snapshots.""" outcome = yield rep = outcome.get_result() + add_timeout_diagnostics(item, call, rep) if rep.when == "call" and rep.failed: # Store on the item's stash so the fixture can access it item.session.stash.setdefault("any_test_failed", False) diff --git a/python/e2e/timeout_diagnostics.py b/python/e2e/timeout_diagnostics.py new file mode 100644 index 0000000000..12de7fa2c0 --- /dev/null +++ b/python/e2e/timeout_diagnostics.py @@ -0,0 +1,211 @@ +"""Failure-only diagnostics for async E2E timeouts, including xdist workers.""" + +import asyncio +import gc +import inspect +import io +import os +import signal +import subprocess +import sys +import threading +import time +import traceback +import uuid +from pathlib import Path + +import pytest + +from copilot._jsonrpc import JsonRpcClient + +_TIMEOUT_DIAGNOSTICS = pytest.StashKey[tuple[str, Path | None]]() + + +@pytest.hookimpl(hookwrapper=True, optionalhook=True) +def pytest_timeout_set_timer(item, settings): + yield + if settings.method != "signal" or threading.current_thread() is not threading.main_thread(): + return + original_handler = signal.getsignal(signal.SIGALRM) + if not callable(original_handler): + return + + def capture_timeout(signum, frame): + try: + original_handler(signum, frame) + except pytest.fail.Exception as exc: + if "from pytest-timeout" in str(exc): + # Capture before fixture finalizers/Runner.close cancel the + # suspended tasks. In particular, makereport is too late for teardown. + item.stash[_TIMEOUT_DIAGNOSTICS] = _collect_timeout_diagnostics(item) + raise + + signal.signal(signal.SIGALRM, capture_timeout) + + +def _dump_awaitable(awaitable, output, seen=None): + if seen is None: + seen = set() + while awaitable is not None and id(awaitable) not in seen: + seen.add(id(awaitable)) + frame = None + next_awaitable = None + for frame_attr, await_attr in ( + ("cr_frame", "cr_await"), + ("ag_frame", "ag_await"), + ("gi_frame", "gi_yieldfrom"), + ): + frame = getattr(awaitable, frame_attr, None) + if frame is not None: + next_awaitable = getattr(awaitable, await_attr, None) + break + if frame is not None: + code = frame.f_code + print(f" {code.co_filename}:{frame.f_lineno} in {code.co_qualname}", file=output) + # Do not dump arbitrary locals, RPC payloads, prompts, tokens, or results. + if code is JsonRpcClient.request.__code__: + values = frame.f_locals + params = values.get("params") or {} + print( + f" outbound method={values.get('method')}" + f" request_id={values.get('request_id')}" + f" session_id={params.get('sessionId')}" + f" elapsed={time.perf_counter() - values['request_start']:.3f}s", + file=output, + ) + elif code is JsonRpcClient._dispatch_request.__code__: + message = frame.f_locals["message"] + print( + f" inbound method={message.get('method')} request_id={message.get('id')}", + file=output, + ) + else: + print(f" awaiting {type(awaitable).__name__}", file=output) + # Async fixture finalizers await an asend object, which hides ag_await. + if type(awaitable).__name__ in ("async_generator_asend", "async_generator_athrow"): + for referent in gc.get_referents(awaitable): + if inspect.isasyncgen(referent): + _dump_awaitable(referent, output, seen) + awaitable = next_awaitable + + +def _dump_client(client, output): + rpc = client._client + if rpc is not None: + reader = rpc._read_thread + print( + f"JSON-RPC running={rpc._running}" + f" reader_alive={reader is not None and reader.is_alive()}" + f" write_locked={rpc._write_lock.locked()}" + f" pending_locked={rpc._pending_lock.locked()}", + file=output, + ) + # A timeout may have interrupted a lock owner: snapshots must not acquire locks. + for request_id, future in rpc.pending_requests.copy().items(): + print( + f" pending request_id={request_id} done={future.done()}" + f" cancelled={future.cancelled()}", + file=output, + ) + for session_id, session in client._sessions.copy().items(): + print( + f"Session {session_id} destroyed={session._destroyed}" + f" disconnect_locked={session._disconnect_lock.locked()}", + file=output, + ) + host = client._ffi_host + if host is not None: + print( + f"FFI server_id={host._server_id} connection_id={host._connection_id}" + f" disposed={host._disposed} starting={host._starting}" + f" operation_locked={host._operation_lock.locked()}" + f" dispose_locked={host._dispose_lock.locked()}" + f" receive_closed={host._receive_buffer._closed}" + f" receive_bytes={len(host._receive_buffer._buffer)}", + file=output, + ) + + +def _sample_native_threads(path: Path, output): + try: + result = subprocess.run( + ["sample", str(os.getpid()), "1", "-file", str(path)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=10, + check=False, + ) + print(f"Native sample exit={result.returncode} file={path}", file=output) + if result.returncode: + print(result.stdout, file=output) + except (OSError, subprocess.TimeoutExpired) as exc: + print(f"Native sample unavailable: {type(exc).__name__}", file=output) + + +def _collect_timeout_diagnostics(item): + output = io.StringIO() + print(f"Test: {item.nodeid}\nPID: {os.getpid()}", file=output) + client = None + try: + context = item.funcargs.get("ctx") + client = getattr(context, "_client", None) + loops = set() + if client is not None: + _dump_client(client, output) + if client._client is not None and client._client._loop is not None: + loops.add(client._client._loop) + for fixture in item.funcargs.values(): + if isinstance(fixture, asyncio.Runner): + loops.add(fixture.get_loop()) + for loop in loops: + print(f"Event loop running={loop.is_running()} closed={loop.is_closed()}", file=output) + for task in sorted(asyncio.all_tasks(loop), key=lambda task: task.get_name()): + print( + f"Task {task.get_name()} done={task.done()} cancelling={task.cancelling()}", + file=output, + ) + _dump_awaitable(task.get_coro(), output) + names = {thread.ident: thread.name for thread in threading.enumerate()} + for ident, frame in sys._current_frames().items(): + print(f"Thread {ident} ({names.get(ident, 'native')})", file=output) + traceback.print_stack(frame, file=output) + except Exception as exc: + print(f"Diagnostic collection failed: {type(exc).__name__}", file=output) + + path = None + try: + directory = item.config.rootpath / ".pytest-diagnostics" + directory.mkdir(exist_ok=True) + stem = f"{os.getpid()}-{uuid.uuid4().hex}" + path = directory / f"{stem}.txt" + path.write_text(output.getvalue(), encoding="utf-8") + if sys.platform == "darwin" and getattr(client, "_ffi_host", None) is not None: + _sample_native_threads(directory / f"{stem}.sample.txt", output) + print(f"Diagnostics saved to {path}", file=output) + path.write_text(output.getvalue(), encoding="utf-8") + except Exception as exc: + print(f"Diagnostic artifact unavailable: {type(exc).__name__}", file=output) + return output.getvalue(), path + + +def add_timeout_diagnostics(item, call, report): + """Attach the timeout snapshot to a report so it survives xdist's suppressed stdout.""" + if call.excinfo is None: + return + error = str(call.excinfo.value) + if not report.failed or "Timeout (" not in error or "from pytest-timeout" not in error: + return + snapshot = item.stash.get(_TIMEOUT_DIAGNOSTICS, None) + if snapshot is None: + snapshot = _collect_timeout_diagnostics(item) + else: + del item.stash[_TIMEOUT_DIAGNOSTICS] + text, path = snapshot + text = f"Phase: {report.when}\n{text}" + if path is not None: + try: + path.write_text(text, encoding="utf-8") + except OSError as exc: + text += f"Diagnostic artifact update failed: {type(exc).__name__}\n" + report.sections.append(("Async timeout diagnostics", text)) diff --git a/python/test_timeout_diagnostics.py b/python/test_timeout_diagnostics.py new file mode 100644 index 0000000000..25a57494c7 --- /dev/null +++ b/python/test_timeout_diagnostics.py @@ -0,0 +1,317 @@ +"""Regression tests for diagnostic evidence retained after an async timeout.""" + +import asyncio +import io +import os +import signal +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from copilot._jsonrpc import JsonRpcClient +from copilot.session import CopilotSession +from e2e import timeout_diagnostics +from e2e.timeout_diagnostics import ( + _dump_awaitable, + _sample_native_threads, + add_timeout_diagnostics, +) + + +@pytest.mark.parametrize("blocked_write", [False, True]) +async def test_timeout_report_identifies_pending_rpc_without_payloads( + tmp_path, monkeypatch, blocked_write +): + rpc = JsonRpcClient(None) + rpc._loop = asyncio.get_running_loop() + sent = asyncio.Event() + release_write = asyncio.Event() + + async def send_message(message): + sent.set() + if blocked_write: + await release_write.wait() + + monkeypatch.setattr(rpc, "_send_message", send_message) + task = asyncio.create_task( + rpc.request( + "session.resume", + {"sessionId": "diagnostic-session", "githubToken": "secret-not-in-diagnostics"}, + ), + name="pending-resume", + ) + try: + await sent.wait() + client = SimpleNamespace(_client=rpc, _sessions={}, _ffi_host=None) + item = SimpleNamespace( + nodeid="test_session_config_e2e.py::test_resume", + config=SimpleNamespace(rootpath=tmp_path), + funcargs={"ctx": SimpleNamespace(_client=client)}, + stash=pytest.Stash(), + ) + call = SimpleNamespace( + excinfo=SimpleNamespace(value=Exception("Timeout (>300.0s) from pytest-timeout.")) + ) + report = SimpleNamespace(failed=True, when="call", sections=[]) + # Snapshotting must not try to acquire a potentially orphaned SDK lock. + with rpc._pending_lock: + add_timeout_diagnostics(item, call, report) + + title, text = report.sections[0] + assert title == "Async timeout diagnostics" + assert "Phase: call" in text + assert "Task pending-resume" in text + assert "JsonRpcClient.request" in text + assert "outbound method=session.resume" in text + assert "session_id=diagnostic-session" in text + assert f"pending request_id={next(iter(rpc.pending_requests))} done=False" in text + assert "pending_locked=True" in text + if blocked_write: + assert ".send_message" in text + else: + assert "awaiting FutureIter" in text + assert "secret-not-in-diagnostics" not in text + assert "githubToken" not in text + (artifact,) = (tmp_path / ".pytest-diagnostics").glob("*.txt") + assert artifact.read_text(encoding="utf-8") == text + assert report.failed + assert not task.done() + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_teardown_dump_follows_async_generator_and_disconnect_lock(monkeypatch): + rpc = JsonRpcClient(None) + rpc._loop = asyncio.get_running_loop() + sent = asyncio.Event() + + async def send_message(message): + sent.set() + + monkeypatch.setattr(rpc, "_send_message", send_message) + session = CopilotSession("diagnostic-session", rpc) + disconnect = asyncio.create_task(session.disconnect()) + teardown_started = asyncio.Event() + + async def fixture(): + yield + teardown_started.set() + await session.disconnect() + + generator = fixture() + await anext(generator) + + async def finalize(): + await anext(generator) + + finalizer = None + try: + await sent.wait() + finalizer = asyncio.create_task(finalize()) + await teardown_started.wait() + output = io.StringIO() + _dump_awaitable(finalizer.get_coro(), output) + text = output.getvalue() + assert "async_generator_asend" in text + assert ".fixture" in text + assert "CopilotSession.disconnect" in text + assert "Lock.acquire" in text + assert not finalizer.done() + assert not disconnect.done() + finally: + tasks = [disconnect] + ([finalizer] if finalizer is not None else []) + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + await generator.aclose() + + +@pytest.mark.parametrize("error", [None, AssertionError("ordinary failure")]) +def test_non_timeout_does_not_collect_diagnostics(error): + item = SimpleNamespace() + call = SimpleNamespace(excinfo=None if error is None else SimpleNamespace(value=error)) + report = SimpleNamespace(failed=error is not None, sections=[]) + add_timeout_diagnostics(item, call, report) + assert report.sections == [] + + +def test_native_sample_is_bounded_and_targets_this_worker(tmp_path, monkeypatch): + calls = [] + + def run(args, **kwargs): + calls.append((args, kwargs)) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(subprocess, "run", run) + path = tmp_path / "native.sample.txt" + output = io.StringIO() + _sample_native_threads(path, output) + args, options = calls[0] + assert args == ["sample", str(os.getpid()), "1", "-file", str(path)] + assert options["timeout"] == 10 + assert "Native sample exit=0" in output.getvalue() + + +@pytest.mark.parametrize( + "error", [FileNotFoundError(), subprocess.TimeoutExpired("sample", timeout=10)] +) +def test_native_sample_failure_preserves_diagnostics(tmp_path, monkeypatch, error): + def run(*args, **kwargs): + raise error + + monkeypatch.setattr(subprocess, "run", run) + output = io.StringIO() + _sample_native_threads(tmp_path / "native.sample.txt", output) + assert f"Native sample unavailable: {type(error).__name__}" in output.getvalue() + + +async def test_signal_snapshot_precedes_task_cleanup_and_keeps_original_failure( + tmp_path, monkeypatch +): + installed = [] + failure = pytest.fail.Exception("Timeout (>300.0s) from pytest-timeout.") + + def original_handler(signum, frame): + raise failure + + monkeypatch.setattr(signal, "SIGALRM", 12345, raising=False) + monkeypatch.setattr(signal, "getsignal", lambda signum: original_handler) + monkeypatch.setattr(signal, "signal", lambda signum, handler: installed.append(handler)) + rpc = JsonRpcClient(None) + rpc._loop = asyncio.get_running_loop() + client = SimpleNamespace(_client=rpc, _sessions={}, _ffi_host=None) + item = SimpleNamespace( + nodeid="test_teardown", + config=SimpleNamespace(rootpath=tmp_path), + funcargs={"ctx": SimpleNamespace(_client=client)}, + stash=pytest.Stash(), + ) + hook = timeout_diagnostics.pytest_timeout_set_timer(item, SimpleNamespace(method="signal")) + next(hook) + with pytest.raises(StopIteration): + next(hook) + started = asyncio.Event() + + async def teardown_waiting_for_disconnect(): + started.set() + await asyncio.Future() + + task = asyncio.create_task(teardown_waiting_for_disconnect()) + try: + await started.wait() + with pytest.raises(pytest.fail.Exception) as caught: + installed[0](signal.SIGALRM, None) + assert caught.value is failure + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + report = SimpleNamespace(failed=True, when="teardown", sections=[]) + add_timeout_diagnostics(item, SimpleNamespace(excinfo=SimpleNamespace(value=failure)), report) + text = report.sections[0][1] + assert "Phase: teardown" in text + assert ( + "in test_signal_snapshot_precedes_task_cleanup_and_keeps_original_failure.." in text + ) + assert "teardown_waiting_for_disconnect" in text + assert not item.stash + + +@pytest.mark.parametrize("phase", ["call", "teardown"] if hasattr(signal, "SIGALRM") else ["call"]) +def test_timeout_report_survives_xdist_without_stdout_capture(tmp_path, phase): + # Exercise the real signal timeout on POSIX. On Windows emulate its exception + # at the event-loop boundary, since the thread timeout terminates the worker. + (tmp_path / "conftest.py").write_text( + """ +import pytest +from e2e.timeout_diagnostics import add_timeout_diagnostics +from e2e.timeout_diagnostics import pytest_timeout_set_timer as pytest_timeout_set_timer + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + add_timeout_diagnostics(item, call, outcome.get_result()) +""", + encoding="utf-8", + ) + (tmp_path / "test_stalled.py").write_text( + """ +import asyncio +import signal +import pytest + +@pytest.fixture +def runner(): + with asyncio.Runner() as runner: + yield runner + +def stall(runner): + loop = runner.get_loop() + if not hasattr(signal, "SIGALRM"): + run_once = loop._run_once + def interrupt_loop(): + run_once() + loop._run_once = run_once + pytest.fail("Timeout (>1.0s) from pytest-timeout.") + loop._run_once = interrupt_loop + + async def stalled_rpc(): + await asyncio.Future() + + runner.run(stalled_rpc()) + +@pytest.fixture +def cleanup(runner): + yield + if PHASE == "teardown": + stall(runner) + +@pytest.mark.timeout( + 1 if hasattr(signal, "SIGALRM") else 10, + method="signal" if hasattr(signal, "SIGALRM") else "thread", +) +def test_stalled(runner, cleanup): + if PHASE == "call": + stall(runner) +""", + encoding="utf-8", + ) + with (tmp_path / "test_stalled.py").open("a", encoding="utf-8") as source: + source.write(f"\nPHASE = {phase!r}\n") + env = dict(os.environ) + env["PYTHONPATH"] = str(Path(__file__).parent.resolve()) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-v", + "-s", + "-n", + "1", + "--dist=loadfile", + "--basetemp", + str(tmp_path / "workers"), + "--rootdir", + str(tmp_path), + str(tmp_path / "test_stalled.py"), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 1, result.stdout + result.stderr + assert ("1 failed" if phase == "call" else "1 error") in result.stdout + assert "Async timeout diagnostics" in result.stdout + assert f"Phase: {phase}" in result.stdout + assert "stalled_rpc" in result.stdout + assert "awaiting FutureIter" in result.stdout + (artifact,) = (tmp_path / ".pytest-diagnostics").glob("*.txt") + assert "stalled_rpc" in artifact.read_text(encoding="utf-8") From d6762319d0bb9960be8c588fd951d317d9c6ba3b Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 11:00:06 -0400 Subject: [PATCH 08/14] Add opt-in fail-first Python timeout reproduction dispatch Keep ordinary PR and reusable checks unchanged. The manual reproduce_timeout input selects a single macOS/inprocess job, runs up to five complete pytest/xdist suites, and exits on the first failure with its original status and existing diagnostic artifacts. Preserve the 20-minute job budget. Validated extracted shell syntax and injected failures on invocations 1, 3, and 5, plus the five-success path. Generated by Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/python-sdk-tests.yml | 32 ++++++++++++++++++++++---- python/README.md | 6 +++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-sdk-tests.yml b/.github/workflows/python-sdk-tests.yml index 4023a8becf..1e16d0672b 100644 --- a/.github/workflows/python-sdk-tests.yml +++ b/.github/workflows/python-sdk-tests.yml @@ -5,6 +5,11 @@ env: on: workflow_dispatch: + inputs: + reproduce_timeout: + description: "Reproduce macOS/inprocess timeouts: up to five full suites, stop on first failure" + type: boolean + default: false workflow_call: permissions: @@ -13,7 +18,7 @@ permissions: jobs: validate: name: "Python SDK Format and Typecheck" - if: github.event.repository.fork == false + if: github.event.repository.fork == false && !inputs.reproduce_timeout runs-on: ubuntu-latest timeout-minutes: 20 defaults: @@ -46,10 +51,10 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: ${{ fromJSON(inputs.reproduce_timeout && '["macos-latest"]' || '["ubuntu-latest", "macos-latest", "windows-latest"]') }} # Test the oldest supported Python version to make sure compatibility is maintained. python-version: ["3.11"] - transport: ["default", "inprocess"] + transport: ${{ fromJSON(inputs.reproduce_timeout && '["inprocess"]' || '["default", "inprocess"]') }} runs-on: ${{ matrix.os }} timeout-minutes: 20 defaults: @@ -93,12 +98,31 @@ jobs: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" - name: Run Python SDK tests + if: ${{ !inputs.reproduce_timeout }} env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} # Keep each module's shared E2E client and proxy on one process while # running independent modules concurrently in isolated workers. run: uv run pytest -v -s -n 2 --dist=loadfile + - name: Reproduce macOS inprocess timeout + if: inputs.reproduce_timeout + env: + COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} + run: | + for attempt in 1 2 3 4 5; do + echo "::group::Full-suite reproduction invocation $attempt/5" + if uv run pytest -v -s -n 2 --dist=loadfile; then + echo "::endgroup::" + else + status=$? + echo "::endgroup::" + echo "::error::Invocation $attempt failed (exit $status); stopping reproduction." + exit "$status" + fi + done + echo "::notice::No failure reproduced in five full-suite invocations." + - name: Upload Python timeout diagnostics if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -111,7 +135,7 @@ jobs: # JavaScript actions use a glibc-linked Node runtime, so Alpine runs through Docker. test-musl-arm64: name: "Python SDK Tests (Alpine ARM64, ${{ matrix.transport }})" - if: github.event.repository.fork == false + if: github.event.repository.fork == false && !inputs.reproduce_timeout strategy: fail-fast: false matrix: diff --git a/python/README.md b/python/README.md index 7a8726608e..0ba6a1f837 100644 --- a/python/README.md +++ b/python/README.md @@ -1241,3 +1241,9 @@ saved under `python/.pytest-diagnostics/`. macOS in-process timeouts also captur one-second native thread sample there. CI uploads these files as `python-timeout--` artifacts. RPC payloads and arbitrary frame locals are not included. The existing test timeout and failure behavior are unchanged. + +To investigate an intermittent timeout, manually dispatch the **Python SDK Tests** +workflow with `reproduce_timeout=true`. This selects only macOS/inprocess and runs +up to five full suites with the usual xdist ordering, stopping with a failed job +on the first nonzero exit and uploading its diagnostics. The job retains its +20-minute budget. Ordinary manual dispatches and reusable PR checks are unchanged. From c7e6a5a505f392220fa34d330d629a74f51b66e8 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 11:10:27 -0400 Subject: [PATCH 09/14] Cancel abandoned Python test task after recording its timeout pytest-timeout's signal interrupts run_until_complete without cancelling the test coroutine. Its held session disconnect lock can then block module cleanup. Identify exactly that task from the interrupted runner's traceback and schedule cancellation after preserving diagnostics; do not cancel unrelated tasks, force-stop the runtime, or hide the first failure. Add actual-plugin/module-fixture regressions for lock-only cleanup recovery and a still-unresponsive runtime. Both retain the original failed test; the latter still reports teardown failure. POSIX uses the real signal timer and Windows invokes the same plugin handler at the event-loop boundary. Generated by Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- python/README.md | 5 +- python/e2e/conftest.py | 3 +- python/e2e/timeout_diagnostics.py | 21 ++++ python/test_timeout_cleanup.py | 163 ++++++++++++++++++++++++++++++ 4 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 python/test_timeout_cleanup.py diff --git a/python/README.md b/python/README.md index 0ba6a1f837..38f5570262 100644 --- a/python/README.md +++ b/python/README.md @@ -1240,7 +1240,10 @@ methods, session/transport state, and Python thread stacks. The same report is saved under `python/.pytest-diagnostics/`. macOS in-process timeouts also capture a one-second native thread sample there. CI uploads these files as `python-timeout--` artifacts. RPC payloads and arbitrary frame locals -are not included. The existing test timeout and failure behavior are unchanged. +are not included. After recording the timeout, the harness cancels only the +abandoned test coroutine so it does not retain locks needed by later fixture +cleanup. The original timeout failure is retained; this does not abort native +runtime work or repair a missing RPC response. To investigate an intermittent timeout, manually dispatch the **Python SDK Tests** workflow with `reproduce_timeout=true`. This selects only macOS/inprocess and runs diff --git a/python/e2e/conftest.py b/python/e2e/conftest.py index 6e11d5b2fe..309c090c3a 100644 --- a/python/e2e/conftest.py +++ b/python/e2e/conftest.py @@ -10,7 +10,7 @@ import copilot._cli_download as cli_download from .testharness import E2ETestContext, is_inprocess_transport -from .timeout_diagnostics import add_timeout_diagnostics +from .timeout_diagnostics import add_timeout_diagnostics, cancel_timed_out_test from .timeout_diagnostics import pytest_timeout_set_timer as pytest_timeout_set_timer # Host-side auth resolution ranks HMAC above the GitHub token, so an ambient @@ -38,6 +38,7 @@ def pytest_runtest_makereport(item, call): rep = outcome.get_result() add_timeout_diagnostics(item, call, rep) if rep.when == "call" and rep.failed: + cancel_timed_out_test(call) # Store on the item's stash so the fixture can access it item.session.stash.setdefault("any_test_failed", False) item.session.stash["any_test_failed"] = True diff --git a/python/e2e/timeout_diagnostics.py b/python/e2e/timeout_diagnostics.py index 12de7fa2c0..c02f069246 100644 --- a/python/e2e/timeout_diagnostics.py +++ b/python/e2e/timeout_diagnostics.py @@ -209,3 +209,24 @@ def add_timeout_diagnostics(item, call, report): except OSError as exc: text += f"Diagnostic artifact update failed: {type(exc).__name__}\n" report.sections.append(("Async timeout diagnostics", text)) + + +def cancel_timed_out_test(call): + """Cancel only the test task abandoned by a timeout outside its coroutine.""" + if call.when != "call" or call.excinfo is None: + return False + error = call.excinfo.value + if "Timeout (" not in str(error) or "from pytest-timeout" not in str(error): + return False + traceback_entry = error.__traceback__ + while traceback_entry is not None: + frame = traceback_entry.tb_frame + if frame.f_code is asyncio.BaseEventLoop.run_until_complete.__code__: + task = frame.f_locals.get("future") + if isinstance(task, asyncio.Task) and not task.done(): + # The signal interrupts the runner, not its task. Let cancellation + # release that task's locks when the fixture's loop next resumes. + return task.cancel() + return False + traceback_entry = traceback_entry.tb_next + return False diff --git a/python/test_timeout_cleanup.py b/python/test_timeout_cleanup.py new file mode 100644 index 0000000000..4e9e9fad46 --- /dev/null +++ b/python/test_timeout_cleanup.py @@ -0,0 +1,163 @@ +"""Regression tests for timed-out tasks retaining module-fixture session locks.""" + +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from e2e.timeout_diagnostics import cancel_timed_out_test + + +@pytest.mark.parametrize( + "cancel_test,withhold_all,teardown_error", + [(False, False, True), (True, False, False), (True, True, True)], +) +def test_module_cleanup_after_timeout(tmp_path, cancel_test, withhold_all, teardown_error): + (tmp_path / "conftest.py").write_text( + """ +import asyncio +import os +import signal +from types import SimpleNamespace + +import pytest +import pytest_asyncio +import pytest_timeout + +from copilot import CopilotClient, RuntimeConnection +from copilot._jsonrpc import JsonRpcClient +from copilot.session import CopilotSession +from e2e.timeout_diagnostics import add_timeout_diagnostics, cancel_timed_out_test +from e2e.timeout_diagnostics import pytest_timeout_set_timer as pytest_timeout_set_timer + +active_item = None + +def pytest_runtest_setup(item): + global active_item + active_item = item + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + add_timeout_diagnostics(item, call, report) + if os.environ["CANCEL_TEST"] == "1" and report.when == "call" and report.failed: + assert cancel_timed_out_test(call) + +@pytest_asyncio.fixture(scope="module", loop_scope="module") +async def ctx(): + loop = asyncio.get_running_loop() + rpc = JsonRpcClient(None) + rpc._loop = loop + client = CopilotClient(connection=RuntimeConnection.for_uri("localhost:1234")) + client._client = rpc + client._state = "connected" + session = CopilotSession("resumed-session", rpc) + client._sessions[session.session_id] = session + state = {"armed": False, "calls": 0} + background = asyncio.create_task(asyncio.Event().wait()) + + async def send(message): + assert message["method"] == "session.detach" + state["calls"] += 1 + if state["calls"] > 1 and os.environ["WITHHOLD_ALL"] == "0": + rpc._handle_message({"id": message["id"], "result": {"success": True}}) + else: + state["armed"] = True + + rpc._send_message = send + if not hasattr(signal, "SIGALRM"): + # Exercise the actual plugin's exception at the runner boundary on + # Windows, where the real thread timeout would terminate the process. + run_once = loop._run_once + def interrupt_blocked_loop(): + if state["armed"] and not loop._ready: + state["armed"] = False + active_item.config.hook.pytest_timeout_cancel_timer(item=active_item) + pytest_timeout.timeout_sigalrm( + active_item, pytest_timeout._get_item_settings(active_item) + ) + run_once() + loop._run_once = interrupt_blocked_loop + + yield SimpleNamespace(_client=client, session=session, state=state, background=background) + try: + state["armed"] = True + await client.stop() + assert state["calls"] == 2 + assert session._destroyed + assert not session._disconnect_lock.locked() + assert not rpc.pending_requests + finally: + background.cancel() + await asyncio.gather(background, return_exceptions=True) +""", + encoding="utf-8", + ) + (tmp_path / "test_stalled.py").write_text( + """ +import signal +import pytest + +pytestmark = [ + pytest.mark.asyncio(loop_scope="module"), + pytest.mark.timeout( + 1 if hasattr(signal, "SIGALRM") else 20, + method="signal" if hasattr(signal, "SIGALRM") else "thread", + ), +] + +async def test_first_detach_times_out(ctx): + await ctx.session.disconnect() + +async def test_later_test_passes(ctx): + assert ctx.state["calls"] == 1 + assert not ctx.background.done() +""", + encoding="utf-8", + ) + env = dict(os.environ) + env["CANCEL_TEST"] = str(int(cancel_test)) + env["WITHHOLD_ALL"] = str(int(withhold_all)) + env["PYTHONPATH"] = str(Path(__file__).parent.resolve()) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-v", + "-s", + "-n", + "0", + "--rootdir", + str(tmp_path), + "--basetemp", + str(tmp_path / "workers"), + str(tmp_path / "test_stalled.py"), + ], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + output = result.stdout + result.stderr + assert result.returncode == 1, output + assert "1 failed, 1 passed" in output + assert ("1 error" in output) == teardown_error + assert "disconnect_locked=True" in output + assert "outbound method=session.detach" in output + assert "from pytest-timeout" in output + + +@pytest.mark.parametrize("when", ["setup", "call", "teardown"]) +@pytest.mark.parametrize("error", [None, AssertionError("ordinary failure")]) +def test_cleanup_ignores_other_failures(when, error): + call = SimpleNamespace( + when=when, excinfo=None if error is None else SimpleNamespace(value=error) + ) + assert not cancel_timed_out_test(call) From efa64cbc3587ba9054c78b7ad4abb54777900038 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 11:33:51 -0400 Subject: [PATCH 10/14] Bound Python timeout reproduction scopes within the job budget Limit the full-suite diagnostic scope to two invocations and add an allowlisted session-config scope with ten invocations. Preserve pytest/xdist options and stop at the first nonzero status. Record started/completed invocations, only mark complete after all pass, and attempt diagnostic artifact upload even after cancellation. Ordinary PR/reusable tests and the 20-minute budgets are unchanged. Validated extracted shell syntax, both count limits, first/middle/last failures, exit 130, process interruption, and rejection of unsupported targets. Generated by Copilot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/python-sdk-tests.yml | 41 ++++++++++++++++++++++---- python/README.md | 14 +++++++-- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-sdk-tests.yml b/.github/workflows/python-sdk-tests.yml index 1e16d0672b..26fc4f837a 100644 --- a/.github/workflows/python-sdk-tests.yml +++ b/.github/workflows/python-sdk-tests.yml @@ -7,9 +7,16 @@ on: workflow_dispatch: inputs: reproduce_timeout: - description: "Reproduce macOS/inprocess timeouts: up to five full suites, stop on first failure" + description: "Reproduce macOS/inprocess timeouts, stopping on the first failure" type: boolean default: false + reproduction_scope: + description: "Diagnostic scope (used only when reproduce_timeout is enabled)" + type: choice + default: full + options: + - full + - session-config workflow_call: permissions: @@ -109,22 +116,44 @@ jobs: if: inputs.reproduce_timeout env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} + REPRODUCTION_SCOPE: ${{ inputs.reproduction_scope || 'full' }} run: | - for attempt in 1 2 3 4 5; do - echo "::group::Full-suite reproduction invocation $attempt/5" - if uv run pytest -v -s -n 2 --dist=loadfile; then + case "$REPRODUCTION_SCOPE" in + full) + attempts=2 + targets=() + ;; + session-config) + attempts=10 + targets=(e2e/test_session_config_e2e.py) + ;; + *) + echo "::error::Unsupported reproduction scope." + exit 64 + ;; + esac + mkdir -p .pytest-diagnostics + summary=.pytest-diagnostics/reproduction-summary.txt + printf 'scope=%s\nplanned_invocations=%s\n' "$REPRODUCTION_SCOPE" "$attempts" > "$summary" + for ((attempt=1; attempt<=attempts; attempt++)); do + printf 'started_invocation=%s\n' "$attempt" >> "$summary" + echo "::group::Reproduction ($REPRODUCTION_SCOPE) invocation $attempt/$attempts" + if uv run pytest -v -s -n 2 --dist=loadfile "${targets[@]}"; then + printf 'completed_invocation=%s\n' "$attempt" >> "$summary" echo "::endgroup::" else status=$? + printf 'failed_invocation=%s\nexit_code=%s\n' "$attempt" "$status" >> "$summary" echo "::endgroup::" echo "::error::Invocation $attempt failed (exit $status); stopping reproduction." exit "$status" fi done - echo "::notice::No failure reproduced in five full-suite invocations." + echo 'reproduction_complete=true' >> "$summary" + echo "::notice::No failure reproduced in $attempts completed $REPRODUCTION_SCOPE invocations." - name: Upload Python timeout diagnostics - if: failure() + if: ${{ always() && (inputs.reproduce_timeout || failure()) }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: python-timeout-${{ matrix.os }}-${{ matrix.transport }} diff --git a/python/README.md b/python/README.md index 38f5570262..1dc1a11627 100644 --- a/python/README.md +++ b/python/README.md @@ -1247,6 +1247,14 @@ runtime work or repair a missing RPC response. To investigate an intermittent timeout, manually dispatch the **Python SDK Tests** workflow with `reproduce_timeout=true`. This selects only macOS/inprocess and runs -up to five full suites with the usual xdist ordering, stopping with a failed job -on the first nonzero exit and uploading its diagnostics. The job retains its -20-minute budget. Ordinary manual dispatches and reusable PR checks are unchanged. +up to two full suites with the usual xdist ordering. Set +`reproduction_scope=session-config` to instead run the original session-config +module up to ten times with the same xdist options. Both modes stop with a failed +job on the first nonzero exit and retain the 20-minute budget. The conservative +counts leave time for a failing test and its cleanup. + +Diagnostic artifacts include `reproduction-summary.txt`, recording started and +completed invocations; `reproduction_complete=true` appears only after every +planned invocation passes. Artifacts are uploaded after diagnostic cancellation +when the runner can still execute cleanup. An interrupted run is not successful +reproduction. Ordinary manual dispatches and reusable PR checks are unchanged. From 733232b44bcd6f7957399fce7955401df9dfed92 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 12:20:15 -0400 Subject: [PATCH 11/14] Preserve bounded macOS .NET hang diagnostics Keep the existing dotnet test command and its selection unchanged. Record allowlisted build/runtime/test/shutdown progress and owned process metadata, then collect bounded native stack samples and terminate only the owned process group before the job deadline. Retain artifacts on failure, cancellation, and successful diagnostic controls. Add focused watchdog regressions including macOS sampling and POSIX pipe-retention cleanup. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/dotnet-sdk-tests.yml | 23 +- dotnet/ci/test-watchdog.mjs | 312 ++++++++++++++++++++++++ dotnet/ci/test-watchdog.test.mjs | 316 +++++++++++++++++++++++++ 3 files changed, 649 insertions(+), 2 deletions(-) create mode 100644 dotnet/ci/test-watchdog.mjs create mode 100644 dotnet/ci/test-watchdog.test.mjs diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index f12f53bd96..a7d293ca7c 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -193,6 +193,12 @@ jobs: working-directory: ./dotnet steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # Reserve time for bounded sampling, process cleanup and artifact upload + # before the job deadline, including time spent in setup below. + - name: Reserve .NET diagnostic budget + if: runner.os == 'macOS' && matrix.shard == '1' + run: echo "DOTNET_TEST_DEADLINE=$(( ($(date +%s) + 16 * 60) * 1000 ))" >> "$GITHUB_ENV" + - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 with: dotnet-version: "10.0.x" @@ -223,7 +229,13 @@ jobs: if: matrix.transport == 'inprocess' run: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" + - name: Validate .NET watchdog + if: runner.os == 'macOS' && matrix.shard == '1' + timeout-minutes: 1 + run: node --test --test-timeout=30000 ci/test-watchdog.test.mjs + - name: Run .NET SDK tests + timeout-minutes: ${{ runner.os == 'macOS' && matrix.shard == '1' && 16 || 20 }} env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} DOTNET_TEST_SHARD: ${{ matrix.shard }} @@ -339,10 +351,17 @@ jobs: if [[ -n "$filter" ]]; then args+=(--filter "$filter") fi - dotnet test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" + if [[ "$RUNNER_OS" == "macOS" && "$DOTNET_TEST_SHARD" == "1" ]]; then + # Preserve the exact command. The watchdog observes known progress + # markers and samples only its process group; it never saves raw logs. + node ci/test-watchdog.mjs test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" + else + dotnet test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" + fi - name: Upload .NET test diagnostics - if: failure() + if: failure() || cancelled() || (runner.os == 'macOS' && matrix.shard == '1') + timeout-minutes: 2 uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: dotnet-test-diagnostics-${{ matrix.os }}-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.shard }}-${{ github.run_attempt }} diff --git a/dotnet/ci/test-watchdog.mjs b/dotnet/ci/test-watchdog.mjs new file mode 100644 index 0000000000..cf23bfbdfa --- /dev/null +++ b/dotnet/ci/test-watchdog.mjs @@ -0,0 +1,312 @@ +import { execFile, spawn } from "node:child_process"; +import { + appendFileSync, + mkdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { basename, join, resolve } from "node:path"; +import { constants } from "node:os"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); + +// Persist only recognized markers, never arbitrary console output (Actions' +// secret masking does not apply to artifact files). +export function progressMarker(line) { + if (/\b_DownloadCopilotCli:/.test(line)) + return { phase: "runtime-provisioning" }; + if (/\bCoreCompile:/.test(line)) return { phase: "compilation" }; + if (/\b_CopyCopilotCliToOutput:/.test(line)) return { phase: "runtime-copy" }; + if (/^Test run for /.test(line)) return { phase: "testhost-startup" }; + if (/^Starting test execution,/.test(line)) + return { phase: "test-discovery" }; + if (/^\[xUnit\.net [\d:.]+\]\s+Starting:/.test(line)) { + return { phase: "tests-and-fixture-cleanup" }; + } + if (/^\[xUnit\.net [\d:.]+\]\s+Finished:/.test(line)) { + return { phase: "testhost-shutdown" }; + } + if (/^Test Run (Successful|Failed|Aborted)\./.test(line)) { + return { phase: "test-command-shutdown" }; + } + const test = + /^\s*(Passed|Failed|Skipped) (GitHub\.Copilot\.Test\.[A-Za-z0-9_.]+)(?=[(\s]|$)/.exec( + line, + ); + if (test) return { outcome: test[1], test: test[2] }; + return null; +} + +export function ownedProcesses(output, group) { + return output.split("\n").flatMap((line) => { + const match = + /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+([\d.]+)\s+(\d+)\s+([\d:-]+)\s+(.+?)\s*$/.exec( + line, + ); + if (!match || Number(match[3]) !== group) return []; + const name = basename(match[8]); + return [ + { + pid: Number(match[1]), + ppid: Number(match[2]), + group, + state: match[4], + cpu: Number(match[5]), + rssKiB: Number(match[6]), + elapsed: match[7], + role: /^(dotnet|testhost|copilot|copilot-runtime|node|tar)$/.test(name) + ? name + : "other", + }, + ]; + }); +} + +export function sampleStacks(output) { + // sample reports native stacks, not memory or local variables. Omit its + // process/path headers and binary-image paths as well. + const graph = + /Call graph:\r?\n([\s\S]*?)(?:\r?\nTotal number in stack|\r?\nBinary Images:|$)/.exec( + output, + ); + let stacks = graph?.[1] ?? "No call graph available"; + for (const name of [ + "COPILOT_HMAC_KEY", + "GH_TOKEN", + "GITHUB_TOKEN", + "COPILOT_GITHUB_TOKEN", + ]) { + if (process.env[name]) + stacks = stacks.replaceAll(process.env[name], "[REDACTED]"); + } + return stacks; +} + +export async function collectProcesses(group) { + const { stdout } = await exec( + "ps", + ["-axo", "pid=,ppid=,pgid=,stat=,%cpu=,rss=,etime=,comm="], + { + timeout: 3_000, + killSignal: "SIGKILL", + maxBuffer: 4 * 1024 * 1024, + env: { ...process.env, LC_ALL: "C" }, + }, + ); + return ownedProcesses(stdout, group); +} + +export async function collectSamples(processes, directory, record) { + if (process.platform !== "darwin") return; + // Bound diagnostics too: eight one-second samples, each capped at five seconds. + for (const { pid } of processes.slice(0, 8)) { + try { + // Explicit stdout avoids sample's default on-disk report. Only the + // filtered call graph below is written to the artifact directory. + const { stdout } = await exec( + "/usr/bin/sample", + [String(pid), "1", "1", "-file", "/dev/stdout"], + { + timeout: 5_000, + killSignal: "SIGKILL", + maxBuffer: 4 * 1024 * 1024, + }, + ); + writeFileSync(join(directory, `sample-${pid}.txt`), sampleStacks(stdout)); + record({ event: "sample", pid }); + } catch { + record({ event: "sample-unavailable", pid }); + } + } +} + +export async function runWithWatchdog({ + command = "dotnet", + args, + directory, + timeoutMs, + intervalMs = 60_000, + graceMs = 5_000, + inspect = collectProcesses, + sample = collectSamples, + forwardOutput = true, +}) { + mkdirSync(directory, { recursive: true }); + const started = performance.now(); + let phase = "build-startup"; + let finalized = false; + const record = (data) => + !finalized && + appendFileSync( + join(directory, "watchdog.jsonl"), + `${JSON.stringify({ at: new Date().toISOString(), elapsedMs: Math.round(performance.now() - started), phase, ...data })}\n`, + ); + record({ event: "start", timeoutMs }); + if (timeoutMs <= 0) { + record({ event: "deadline-expired-before-start" }); + return 124; + } + + // On macOS the child leads a process group. Kill only that owned group, even + // if dotnet exits while a descendant still holds its output pipe open. + const child = spawn(command, args, { + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"], + }); + let result; + let stopping = false; + let finish; + const completed = new Promise((resolve) => { + finish = resolve; + }); + const signal = (name) => { + try { + if (process.platform === "win32") child.kill(name); + else process.kill(-child.pid, name); + } catch (error) { + if (error.code !== "ESRCH") + record({ event: "signal-failed", signal: name }); + } + }; + const snapshot = async () => { + try { + const processes = await inspect(child.pid); + record({ event: "processes", processes }); + return processes; + } catch { + record({ event: "process-snapshot-unavailable" }); + return []; + } + }; + const stop = async (reason, code) => { + if (stopping) return; + stopping = true; + result = result || code; + record({ event: reason }); + if (forwardOutput) + console.error( + `[.NET watchdog] ${reason} during ${phase}; preserving diagnostics.`, + ); + clearInterval(heartbeat); + clearTimeout(deadline); + const processes = await snapshot(); + // Actions allows only a short signal grace period on cancellation. Keep + // the already-written timeline and snapshot; sample only our own deadline. + if (reason === "deadline-exceeded") { + try { + await sample(processes, directory, record); + } catch { + record({ event: "samples-unavailable" }); + } + } + signal("SIGTERM"); + await new Promise((resolve) => + setTimeout( + resolve, + reason === "deadline-exceeded" ? graceMs : Math.min(graceMs, 1_000), + ), + ); + signal("SIGKILL"); + child.stdout.destroy(); + child.stderr.destroy(); + child.unref(); + record({ event: "stopped", exitCode: result }); + finish(); + }; + const onInterrupt = () => { + void stop("interrupted", 130); + }; + const onTerminate = () => { + void stop("terminated", 143); + }; + process.on("SIGINT", onInterrupt); + process.on("SIGTERM", onTerminate); + + for (const [stream, destination] of [ + [child.stdout, process.stdout], + [child.stderr, process.stderr], + ]) { + if (forwardOutput) stream.pipe(destination); + let pending = ""; + stream.setEncoding("utf8"); + stream.on("data", (chunk) => { + pending += chunk; + let newline; + while ((newline = pending.indexOf("\n")) !== -1) { + const marker = progressMarker(pending.slice(0, newline)); + if (marker) { + if (marker.phase) phase = marker.phase; + record({ event: "progress", ...marker }); + } + pending = pending.slice(newline + 1); + } + // Compiler invocations can be very long; none are diagnostic markers. + if (pending.length > 16_384) pending = ""; + }); + } + child.on("spawn", () => record({ event: "spawn", pid: child.pid })); + child.on("error", () => { + result ??= 127; + record({ event: "spawn-error" }); + }); + child.on("exit", (code, exitSignal) => { + result ??= code ?? (exitSignal ? 128 + constants.signals[exitSignal] : 1); + record({ event: "command-exit", exitCode: code, signal: exitSignal }); + phase = "output-drain"; + }); + child.on("close", () => { + record({ event: "output-closed" }); + if (!stopping) finish(); + }); + const heartbeat = setInterval(() => { + void snapshot(); + }, intervalMs); + const deadline = setTimeout(() => { + void stop("deadline-exceeded", 124); + }, timeoutMs); + await completed; + clearInterval(heartbeat); + clearTimeout(deadline); + process.off("SIGINT", onInterrupt); + process.off("SIGTERM", onTerminate); + record({ event: "finish", exitCode: result }); + finalized = true; + return result ?? 1; +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + const directory = resolve("TestResults"); + mkdirSync(directory, { recursive: true }); + const { copilotCliVersion } = JSON.parse( + readFileSync(new URL("../../nodejs/package.json", import.meta.url)), + ); + writeFileSync( + join(directory, "watchdog-runtime.json"), + JSON.stringify({ + copilotCliVersion, + platform: process.platform, + arch: process.arch, + node: process.version, + }), + ); + const deadline = Number(process.env.DOTNET_TEST_DEADLINE); + if ( + !Number.isFinite(deadline) || + deadline <= 0 || + process.platform !== "darwin" + ) { + throw new Error( + "The .NET CI watchdog requires macOS and DOTNET_TEST_DEADLINE", + ); + } + process.exitCode = await runWithWatchdog({ + args: process.argv.slice(2), + directory, + timeoutMs: Math.min(15 * 60_000, deadline - Date.now()), + }); +} diff --git a/dotnet/ci/test-watchdog.test.mjs b/dotnet/ci/test-watchdog.test.mjs new file mode 100644 index 0000000000..4624e6f089 --- /dev/null +++ b/dotnet/ci/test-watchdog.test.mjs @@ -0,0 +1,316 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; +import { + collectProcesses, + collectSamples, + ownedProcesses, + progressMarker, + runWithWatchdog, + sampleStacks, +} from "./test-watchdog.mjs"; + +function outputDirectory(t) { + const directory = join( + import.meta.dirname, + `.watchdog-test-${process.pid}-${crypto.randomUUID()}`, + ); + mkdirSync(directory, { recursive: true }); + t.after(() => rmSync(directory, { recursive: true, force: true })); + return directory; +} + +function events(directory) { + return readFileSync(join(directory, "watchdog.jsonl"), "utf8") + .trim() + .split("\n") + .map(JSON.parse); +} + +function run(t, source, options = {}) { + const directory = outputDirectory(t); + return { + directory, + result: runWithWatchdog({ + command: process.execPath, + args: ["-e", source, "--", ...(options.extraArgs ?? [])], + directory, + timeoutMs: 5_000, + graceMs: 25, + inspect: async () => [], + sample: async () => {}, + forwardOutput: false, + ...options, + }), + }; +} + +test("recognizes build, provisioning, test and shutdown without recording raw output", () => { + for (const [line, phase] of [ + [" _DownloadCopilotCli:", "runtime-provisioning"], + [" CoreCompile:", "compilation"], + [" _CopyCopilotCliToOutput:", "runtime-copy"], + ["Test run for /private/path.dll (net8.0)", "testhost-startup"], + ["Starting test execution, please wait...", "test-discovery"], + [ + "[xUnit.net 00:00:00.10] Starting: GitHub.Copilot.SDK.Test", + "tests-and-fixture-cleanup", + ], + [ + "[xUnit.net 00:01:50.44] Finished: GitHub.Copilot.SDK.Test", + "testhost-shutdown", + ], + ["Test Run Successful.", "test-command-shutdown"], + ["Test Run Aborted.", "test-command-shutdown"], + ]) + assert.deepEqual(progressMarker(line), { phase }); + assert.deepEqual( + progressMarker( + ' Passed GitHub.Copilot.Test.E2E.Example.Test(token: "secret") [1 s]', + ), + { + outcome: "Passed", + test: "GitHub.Copilot.Test.E2E.Example.Test", + }, + ); + assert.equal(progressMarker("secret output"), null); +}); + +test("process snapshots contain only owned numeric metadata and known executable roles", () => { + assert.deepEqual( + ownedProcesses( + ` + 123 1 123 S 0.1 4096 01:02 /private/dotnet + 124 123 123 R+ 10.0 2048 00:01 /private/unrecognized-secret + 125 1 125 S 0.0 1024 01:00 /private/node +`, + 123, + ), + [ + { + pid: 123, + ppid: 1, + group: 123, + state: "S", + cpu: 0.1, + rssKiB: 4096, + elapsed: "01:02", + role: "dotnet", + }, + { + pid: 124, + ppid: 123, + group: 123, + state: "R+", + cpu: 10, + rssKiB: 2048, + elapsed: "00:01", + role: "other", + }, + ], + ); +}); + +test("samples omit headers and image paths and redact credentials", (t) => { + const previous = process.env.COPILOT_HMAC_KEY; + process.env.COPILOT_HMAC_KEY = "watchdog-test-secret"; + t.after(() => { + if (previous === undefined) delete process.env.COPILOT_HMAC_KEY; + else process.env.COPILOT_HMAC_KEY = previous; + }); + assert.equal( + sampleStacks( + "Path: private\nCall graph:\n wait watchdog-test-secret\nBinary Images:\nprivate", + ), + " wait [REDACTED]", + ); +}); + +test("forwards arguments, preserves success and failure, and records split output markers", async (t) => { + for (const code of [0, 23]) { + const { directory, result } = run( + t, + ` + const assert = require("node:assert/strict"); + assert.deepEqual(process.argv.slice(1), ["--filter", "(A|B)&C", "--blame-hang"]); + process.stdout.write(" _DownloadCopilot"); + setTimeout(() => { + console.log("Cli:"); + console.log("secret output"); + console.log("Test Run Successful."); + process.exitCode = ${code}; + }, 20); + `, + { extraArgs: ["--filter", "(A|B)&C", "--blame-hang"] }, + ); + assert.equal(await result, code); + assert.ok( + events(directory).some((event) => event.phase === "runtime-provisioning"), + ); + assert.ok( + !readFileSync(join(directory, "watchdog.jsonl"), "utf8").includes( + "secret output", + ), + ); + } +}); + +test("expired job budget never starts a command", async (t) => { + const { directory, result } = run(t, "process.exit(99)", { timeoutMs: 0 }); + assert.equal(await result, 124); + assert.ok(!events(directory).some((event) => event.event === "spawn")); +}); + +test("a hung command is sampled before termination and fails within the inner budget", async (t) => { + let sampled = false; + const { directory, result } = run( + t, + ` + console.log("Test run for test.dll"); + setInterval(() => {}, 1000); + `, + { + timeoutMs: 1_000, + sample: async () => { + sampled = true; + throw new Error("unavailable"); + }, + inspect: async () => { + throw new Error("unavailable"); + }, + }, + ); + assert.equal(await result, 124); + assert.ok(sampled); + assert.equal( + events(directory).find((event) => event.event === "deadline-exceeded") + .phase, + "testhost-startup", + ); + assert.ok( + events(directory).some( + (event) => event.event === "process-snapshot-unavailable", + ), + ); + assert.ok( + events(directory).some((event) => event.event === "samples-unavailable"), + ); +}); + +test("missing executables preserve spawn failure", async (t) => { + const { result } = run(t, "", { + command: "nonexistent-dotnet-watchdog-test-command", + }); + assert.equal(await result, 127); +}); + +test( + "signal exits preserve the shell exit status", + { + skip: process.platform === "win32", + }, + async (t) => { + const { result } = run(t, "process.kill(process.pid, 'SIGTERM')"); + assert.equal(await result, 143); + }, +); + +test( + "owned descendants retaining pipes cannot hide the first command failure", + { + skip: process.platform === "win32", + }, + async (t) => { + for (const code of [0, 37]) { + let descendants = []; + const { directory, result } = run( + t, + ` + const { spawn } = require("node:child_process"); + spawn(process.execPath, ["-e", "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)"], + { stdio: ["ignore", 1, 2] }).unref(); + process.exit(${code}); + `, + { + timeoutMs: 1_000, + inspect: collectProcesses, + sample: async (processes) => { + descendants = processes; + }, + }, + ); + assert.equal(await result, code || 124); + assert.equal( + events(directory).find((event) => event.event === "deadline-exceeded") + .phase, + "output-drain", + ); + assert.ok(descendants.length > 0); + // kill(0) can still see a zombie briefly; ps must not see a live descendant. + const remaining = await collectProcesses(descendants[0].group); + assert.ok(remaining.every((process) => process.state.startsWith("Z"))); + } + }, +); + +test( + "termination requests retain diagnostics and a failing status", + { + skip: process.platform === "win32", + }, + async (t) => { + const directory = outputDirectory(t); + const script = ` + import { runWithWatchdog } from ${JSON.stringify(new URL("./test-watchdog.mjs", import.meta.url).href)}; + process.exitCode = await runWithWatchdog({ + command: process.execPath, + args: ["-e", "process.stdout.write('ready\\\\n'); setInterval(() => {}, 1000)"], + directory: ${JSON.stringify(directory)}, timeoutMs: 5000, graceMs: 25, + inspect: async () => [], sample: async () => {}, + }); + `; + const child = spawn( + process.execPath, + ["--input-type=module", "-e", script], + { stdio: ["ignore", "pipe", "inherit"] }, + ); + t.after(() => child.kill("SIGKILL")); + const exited = new Promise((resolve) => child.on("exit", resolve)); + await new Promise((resolve) => child.stdout.once("data", resolve)); + child.kill("SIGTERM"); + assert.equal(await exited, 143); + assert.ok(events(directory).some((event) => event.event === "terminated")); + }, +); + +test( + "macOS sample captures an owned process call graph without raw files", + { + skip: process.platform !== "darwin", + }, + async (t) => { + const directory = outputDirectory(t); + const child = spawn( + process.execPath, + ["-e", "process.stdout.write('ready'); setInterval(() => {}, 1000)"], + { + stdio: ["ignore", "pipe", "inherit"], + }, + ); + t.after(() => child.kill("SIGKILL")); + await new Promise((resolve) => child.stdout.once("data", resolve)); + const records = []; + await collectSamples([{ pid: child.pid }], directory, (record) => + records.push(record), + ); + assert.equal(records[0].event, "sample"); + const stacks = readFileSync( + join(directory, `sample-${child.pid}.txt`), + "utf8", + ); + assert.notEqual(stacks, "No call graph available"); + assert.ok(!stacks.includes("Binary Images:")); + }, +); From c9aee7d9477e7493b70996ab62f6167b8562ea62 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 18:09:55 -0400 Subject: [PATCH 12/14] Capture Go and Windows .NET hangs before CI deadlines Retain goroutine and bounded managed/native stack evidence before the outer job timeout. Reuse the watchdog with owned Windows Job Object cleanup and preserve test selection, assertions, timeouts, and original failures. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/dotnet-sdk-tests.yml | 24 +- .github/workflows/go-sdk-tests.yml | 25 +- dotnet/ci/README.md | 86 +++++ dotnet/ci/WindowsJob.cs | 126 +++++++ dotnet/ci/WindowsWatchdog.cs | 205 ++++++++++++ dotnet/ci/WindowsWatchdog.csproj | 6 + dotnet/ci/dotnet-tools.json | 11 + dotnet/ci/test-watchdog-windows.test.mjs | 312 ++++++++++++++++++ dotnet/ci/test-watchdog.mjs | 162 +++++++-- dotnet/ci/test-watchdog.test.mjs | 97 +++++- dotnet/ci/windows-watchdog.mjs | 203 ++++++++++++ go/.gitignore | 3 + go/README.md | 13 + go/ci/test-watchdog.mjs | 69 ++++ go/ci/test-watchdog.test.mjs | 87 +++++ go/internal/e2e/main_test.go | 12 + go/internal/testdiagnostics/diagnostics.go | 100 ++++++ .../testdiagnostics/diagnostics_test.go | 141 ++++++++ 18 files changed, 1646 insertions(+), 36 deletions(-) create mode 100644 dotnet/ci/README.md create mode 100644 dotnet/ci/WindowsJob.cs create mode 100644 dotnet/ci/WindowsWatchdog.cs create mode 100644 dotnet/ci/WindowsWatchdog.csproj create mode 100644 dotnet/ci/dotnet-tools.json create mode 100644 dotnet/ci/test-watchdog-windows.test.mjs create mode 100644 dotnet/ci/windows-watchdog.mjs create mode 100644 go/ci/test-watchdog.mjs create mode 100644 go/ci/test-watchdog.test.mjs create mode 100644 go/internal/e2e/main_test.go create mode 100644 go/internal/testdiagnostics/diagnostics.go create mode 100644 go/internal/testdiagnostics/diagnostics_test.go diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index a7d293ca7c..671118eeb7 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -196,7 +196,7 @@ jobs: # Reserve time for bounded sampling, process cleanup and artifact upload # before the job deadline, including time spent in setup below. - name: Reserve .NET diagnostic budget - if: runner.os == 'macOS' && matrix.shard == '1' + if: (runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1' run: echo "DOTNET_TEST_DEADLINE=$(( ($(date +%s) + 16 * 60) * 1000 ))" >> "$GITHUB_ENV" - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 @@ -225,17 +225,28 @@ jobs: if: runner.os == 'Windows' run: pwsh.exe -Command "Write-Host 'PowerShell ready'" + - name: Restore Windows managed-stack diagnostics + if: runner.os == 'Windows' && matrix.shard == '1' + run: | + dotnet tool restore --tool-manifest ci/dotnet-tools.json + dotnet build ci/WindowsWatchdog.csproj -c Release -p:UseSharedCompilation=false + - name: Select inprocess transport if: matrix.transport == 'inprocess' run: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" - name: Validate .NET watchdog - if: runner.os == 'macOS' && matrix.shard == '1' + if: (runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1' timeout-minutes: 1 run: node --test --test-timeout=30000 ci/test-watchdog.test.mjs + - name: Validate Windows .NET diagnostics + if: runner.os == 'Windows' && matrix.shard == '1' + timeout-minutes: 2 + run: node --test --test-timeout=60000 ci/test-watchdog-windows.test.mjs + - name: Run .NET SDK tests - timeout-minutes: ${{ runner.os == 'macOS' && matrix.shard == '1' && 16 || 20 }} + timeout-minutes: ${{ (runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1' && 16 || 20 }} env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} DOTNET_TEST_SHARD: ${{ matrix.shard }} @@ -351,16 +362,17 @@ jobs: if [[ -n "$filter" ]]; then args+=(--filter "$filter") fi - if [[ "$RUNNER_OS" == "macOS" && "$DOTNET_TEST_SHARD" == "1" ]]; then + if [[ ("$RUNNER_OS" == "macOS" || "$RUNNER_OS" == "Windows") && "$DOTNET_TEST_SHARD" == "1" ]]; then # Preserve the exact command. The watchdog observes known progress - # markers and samples only its process group; it never saves raw logs. + # markers and samples only its process group / Windows Job Object; + # it never saves raw logs or process memory. node ci/test-watchdog.mjs test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" else dotnet test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" fi - name: Upload .NET test diagnostics - if: failure() || cancelled() || (runner.os == 'macOS' && matrix.shard == '1') + if: failure() || cancelled() || ((runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1') timeout-minutes: 2 uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: diff --git a/.github/workflows/go-sdk-tests.yml b/.github/workflows/go-sdk-tests.yml index cde400615a..da999078a0 100644 --- a/.github/workflows/go-sdk-tests.yml +++ b/.github/workflows/go-sdk-tests.yml @@ -56,6 +56,9 @@ jobs: working-directory: ./go steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Reserve Go diagnostic budget + if: matrix.os == 'macos-latest' && matrix.transport == 'inprocess' + run: echo "GO_TEST_DEADLINE=$(($(date +%s) * 1000 + 16 * 60 * 1000))" >> "$GITHUB_ENV" - uses: ./.github/actions/setup-copilot id: setup-copilot - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 @@ -76,11 +79,31 @@ jobs: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" echo "GOFLAGS=-tags=copilot_inprocess" >> "$GITHUB_ENV" + - name: Validate Go watchdog + if: matrix.os == 'macos-latest' && matrix.transport == 'inprocess' + run: | + node --test ci/test-watchdog.test.mjs ../dotnet/ci/test-watchdog.test.mjs + go test ./internal/testdiagnostics -race -count=1 + - name: Run Go SDK tests env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} COPILOT_CLI_PATH: ${{ steps.setup-copilot.outputs.cli-path }} - run: /bin/bash test.sh + run: | + if [ "$RUNNER_OS" = "macOS" ] && [ "$COPILOT_SDK_DEFAULT_CONNECTION" = "inprocess" ]; then + node ci/test-watchdog.mjs + else + /bin/bash test.sh + fi + + - name: Upload Go test diagnostics + if: always() && matrix.os == 'macos-latest' && matrix.transport == 'inprocess' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: go-test-diagnostics-${{ matrix.os }}-${{ matrix.transport }}-${{ github.run_attempt }} + path: go/TestResults/ + if-no-files-found: warn + retention-days: 7 # JavaScript actions use a glibc-linked Node runtime, so Alpine runs through Docker. test-musl-arm64: diff --git a/dotnet/ci/README.md b/dotnet/ci/README.md new file mode 100644 index 0000000000..7e4cecb745 --- /dev/null +++ b/dotnet/ci/README.md @@ -0,0 +1,86 @@ +# .NET CI hang diagnostics + +The macOS and Windows default/CAPI shard 1 jobs run the **unchanged** `dotnet test` +command through `test-watchdog.mjs`. Frameworks, filters, environment, and the +10-minute per-test blame timeout are unchanged. The watchdog deadline is the +earlier of 15 minutes after command launch or 16 minutes after checkout. The +20-minute job limit is unchanged; the remaining time is for diagnostics, cleanup, +and the two-minute artifact upload. + +On Windows, the workflow builds `WindowsWatchdog.csproj` with the already +installed .NET 10 SDK and restores the pinned Microsoft `dotnet-stack` tool from +`dotnet-tools.json`. The helper joins a kill-on-close Windows Job Object **before** +starting the command. Nested jobs contain grandchildren even when ancestors exit +between snapshots. Cleanup never uses executable-name matching or an unrelated +process search. Held process handles prevent sampled PIDs from being reused. + +The root command exiting is not sufficient: the helper also waits for stdout and +stderr EOF. A descendant retaining either pipe therefore still reaches the +watchdog deadline, preserving an earlier command failure (otherwise exit 124). +After a normal root exit and EOF, any remaining background servers are cleaned +up without changing the command's result. Killing the supervisor also closes its +job and terminates its owned descendants. Supervisor/launch failures stay failures. + +## Reading the next CI artifact + +Download `dotnet-test-diagnostics-windows-latest-default-capi-1-`: + +- `watchdog.jsonl`: allowlisted build/provisioning/test/shutdown phase markers, + recognized target framework, completed test method names (no argument values), + exit-versus-output-drain timing, deadline, inspection errors, and final status. +- `windows-job.jsonl`: append-only snapshots on a five-second cadence, plus + lifecycle changes. Only owned PIDs, known executable roles, runtime kind, + CPU/RSS, process start times, and numeric thread states/wait reasons are stored. + Snapshots cover at most 128 processes and 64 threads per process; total + `processCount`/`threadCount` values expose truncation. +- `managed-stack-.txt`: at the watchdog deadline, readable .NET managed + thread stacks for up to four owned CoreCLR processes, prioritizing testhosts. + Each collector has a ten-second deadline (including startup), a one-second + forced-close bound, and its own Job Object. Collector stdout/stderr are captured + in memory only, capped at 4 MiB; artifacts retain only thread IDs, native boundaries, and + module/method names, capped at 4,096 lines / 64 KiB. Truncation and unavailable + captures are explicit. `stack-collector-/windows-job.jsonl` diagnoses the + collector itself. +- Existing TRX and blame sequence files: correlate the framework and last + completed test with the test host's existing failure/active-test evidence. +- `watchdog-runtime.json`: pinned CLI version, platform, architecture, and Node. + +`dotnet-stack` supports CoreCLR, not .NET Framework. Framework processes get an +explicit `managed-stack-unsupported` event, numeric thread information, and the +existing blame/TRX diagnostics. Native CLI stacks are not collected on Windows. +Thread stacks are not a dump of suspended async state machines; an off-thread +await may still need follow-up investigation. No heap/process dumps, environment, +command lines, arbitrary console output, or locals are added to diagnostic +artifacts. The existing TRX/blame artifact behavior is unchanged. + +Instrumentation does not establish the cause of the original Windows timeout. +Use the next failure's phase, framework, exit/EOF timing, test sequence and stacks +to distinguish provisioning, test execution, fixture disposal and pipe retention +before attributing it or changing SDK behavior. + +## Focused local validation (Windows) + +From `dotnet`: + +```powershell +dotnet tool restore --tool-manifest ci\dotnet-tools.json +dotnet build ci\WindowsWatchdog.csproj -c Release -p:UseSharedCompilation=false +dotnet format ci\WindowsWatchdog.csproj --no-restore --verify-no-changes +node --test --test-timeout=30000 ci\test-watchdog.test.mjs +node --test --test-timeout=60000 ci\test-watchdog-windows.test.mjs +``` + +The Windows-only controls cover orphaned pipe holders, original failure +preservation, unrelated-process survival, supervisor termination, normal EOF with +background servers, and real managed waiting-stack capture. `--stack-probe` on the +helper is their small managed fixture, not part of SDK test selection. The shared +suite retains its existing POSIX-only controls; run it on macOS to exercise native +sampling and process-group behavior. + +Other CI entry points may import `runWithWatchdog` and supply `command`, `args`, +an absolute artifact `directory`, and `timeoutMs`. Optional `marker` and `label` +parameters default to `progressMarker` and `".NET"`; a custom marker must return +only allowlisted metadata or `null`, never raw console output or argument values. +This lets the Go macOS entry point reuse process-group cleanup and native sampling +without duplicating the watchdog. Importing the module does not execute its .NET +CLI entry point. diff --git a/dotnet/ci/WindowsJob.cs b/dotnet/ci/WindowsJob.cs new file mode 100644 index 0000000000..c2ae086f84 --- /dev/null +++ b/dotnet/ci/WindowsJob.cs @@ -0,0 +1,126 @@ +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.InteropServices; + +namespace GitHub.Copilot.Ci; + +// The supervisor joins before spawning anything. All descendants inherit this +// nested job, including children whose parent exits before the next snapshot. +internal sealed class WindowsJob : IDisposable +{ + private readonly IntPtr handle; + + public WindowsJob() + { + handle = CreateJobObject(IntPtr.Zero, null); + Check(handle != IntPtr.Zero); + SetKillOnClose(true); + Check(AssignProcessToJobObject(handle, Process.GetCurrentProcess().Handle)); + } + + public int[] ProcessIds() + { + // A bounded allocation, with a visible error rather than a truncated tree. + const int capacity = 4096; + IntPtr buffer = Marshal.AllocHGlobal(8 + capacity * IntPtr.Size); + try + { + Check(QueryInformationJobObject(handle, 3, buffer, 8 + capacity * IntPtr.Size, IntPtr.Zero)); + int count = Marshal.ReadInt32(buffer, 4); + int[] ids = new int[count]; + for (int i = 0; i < count; i++) + ids[i] = checked((int)Marshal.ReadIntPtr(buffer, 8 + i * IntPtr.Size)); + return ids; + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + + public void Complete() + { + // Only the supervisor may remain when disabling kill-on-close. + int[] ids = ProcessIds(); + if (ids.Length != 1 || ids[0] != Environment.ProcessId) + throw new InvalidOperationException("The owned job is not empty."); + SetKillOnClose(false); + } + + public bool Owns(IntPtr process) + { + Check(IsProcessInJob(process, handle, out bool owned)); + return owned; + } + + public void Abort(int exitCode) => Check(TerminateJobObject(handle, unchecked((uint)exitCode))); + + private void SetKillOnClose(bool enabled) + { + var limits = new ExtendedLimits(); + limits.Basic.LimitFlags = enabled ? 0x2000u : 0; + Check(SetInformationJobObject(handle, 9, ref limits, Marshal.SizeOf())); + } + + private static void Check(bool succeeded) + { + if (!succeeded) + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + public void Dispose() => Check(CloseHandle(handle)); + + [StructLayout(LayoutKind.Sequential)] + private struct BasicLimits + { + public long PerProcessUserTime, PerJobUserTime; + public uint LimitFlags; + public UIntPtr MinimumWorkingSet, MaximumWorkingSet; + public uint ActiveProcessLimit; + public UIntPtr Affinity; + public uint PriorityClass, SchedulingClass; + } + + [StructLayout(LayoutKind.Sequential)] + private struct IoCounters + { + public ulong ReadOperations, WriteOperations, OtherOperations; + public ulong ReadBytes, WriteBytes, OtherBytes; + } + + [StructLayout(LayoutKind.Sequential)] + private struct ExtendedLimits + { + public BasicLimits Basic; + public IoCounters Io; + public UIntPtr ProcessMemory, JobMemory, PeakProcessMemory, PeakJobMemory; + } + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern IntPtr CreateJobObject(IntPtr attributes, string? name); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool SetInformationJobObject(IntPtr job, int infoClass, ref ExtendedLimits info, int length); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool IsProcessInJob(IntPtr process, IntPtr job, [MarshalAs(UnmanagedType.Bool)] out bool owned); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool TerminateJobObject(IntPtr job, uint exitCode); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool QueryInformationJobObject(IntPtr job, int infoClass, IntPtr info, int length, IntPtr returnedLength); + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool CloseHandle(IntPtr handle); +} diff --git a/dotnet/ci/WindowsWatchdog.cs b/dotnet/ci/WindowsWatchdog.cs new file mode 100644 index 0000000000..95a959e13f --- /dev/null +++ b/dotnet/ci/WindowsWatchdog.cs @@ -0,0 +1,205 @@ +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.Versioning; +using System.Text.Json; +using System.Text.Json.Serialization; + +[assembly: SupportedOSPlatform("windows")] + +namespace GitHub.Copilot.Ci; + +internal static class WindowsWatchdog +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private static int Main(string[] args) + { + if (args is ["--stack-probe"]) + { + Console.WriteLine("secret test payload"); + WaitForStackProbe(); + return 0; + } + + var state = new Status(); + var tracked = new Dictionary(); + WindowsJob? job = null; + bool completed = false; + // Append-only IPC avoids Windows rename/delete sharing races with readers. + using var output = new StreamWriter(new FileStream(args[0], FileMode.Append, FileAccess.Write, FileShare.Read)); + output.AutoFlush = true; + try + { + job = new WindowsJob(); + var request = JsonSerializer.Deserialize(Console.ReadLine()!, JsonOptions)!; + var start = new ProcessStartInfo(request.Command) + { + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + foreach (string argument in request.Args) + start.ArgumentList.Add(argument); + using var root = Process.Start(start)!; + root.StandardInput.Close(); + state.RootPid = root.Id; + Task drained = Task.WhenAll( + root.StandardOutput.BaseStream.CopyToAsync(Console.OpenStandardOutput()), + root.StandardError.BaseStream.CopyToAsync(Console.OpenStandardError())); + long nextSnapshot = 0; + do + { + int? previousExitCode = state.ExitCode; + if (root.HasExited) + state.ExitCode = root.ExitCode; + int[] ids = job.ProcessIds().Where(id => id != Environment.ProcessId).ToArray(); + bool closing = state.ExitCode.HasValue && drained.IsCompleted; + if (closing) + drained.GetAwaiter().GetResult(); + if (closing || state.ExitCode != previousExitCode || Environment.TickCount64 >= nextSnapshot) + { + state.ProcessCount = ids.Length; + state.Processes = ids.Take(128).Select(id => Snapshot(id, job, tracked)).ToArray(); + state.OutputClosed = closing; + output.WriteLine(JsonSerializer.Serialize(state, JsonOptions)); + nextSnapshot = Environment.TickCount64 + 5_000; + } + if (closing) + { + // Compiler servers may outlive a successful command without + // holding its pipes. Clean them up, but only after BOTH the + // actual root exit and output EOF, never merely root exit. + if (ids.Length != 0) + job.Abort(state.ExitCode!.Value); + break; + } + Thread.Sleep(250); + } while (true); + job.Complete(); + completed = true; + return state.ExitCode!.Value; + } + catch (Exception error) when (error is Win32Exception or InvalidOperationException or IOException or JsonException or ArgumentException) + { + state.Error = error.GetType().Name; + state.ErrorCode = error.HResult; + state.ExitCode = state.ExitCode is null or 0 ? 127 : state.ExitCode; + Console.Error.WriteLine($"[.NET watchdog] Windows supervisor failed: {state.Error}"); + output.WriteLine(JsonSerializer.Serialize(state, JsonOptions)); + return state.ExitCode.Value; + } + finally + { + foreach (Process process in tracked.Values) + process.Dispose(); + if (job is not null) + { + // Never turn an inspector/supervisor failure into success. Abort + // gives the supervisor AND its descendants a nonzero exit code. + if (!completed) + job.Abort(state.ExitCode is null or 0 ? 127 : state.ExitCode.Value); + job.Dispose(); + } + } + } + + private static object Snapshot(int pid, WindowsJob job, Dictionary tracked) + { + try + { + if (!tracked.TryGetValue(pid, out Process? process)) + { + process = Process.GetProcessById(pid); + try + { + if (!job.Owns(process.Handle)) + throw new InvalidOperationException(); + // Retain the handle while sampling so an exited PID cannot + // be recycled into an unrelated diagnostic target. + tracked.Add(pid, process); + } + catch + { + process.Dispose(); + throw; + } + } + process.Refresh(); + string role = process.ProcessName.ToLowerInvariant(); + if (role is not ("dotnet" or "testhost" or "testhost.x86" or "msbuild" or "copilot" or "copilot-runtime" or "node" or "tar" or "pwsh")) + role = "other"; + string runtime = "native"; + foreach (ProcessModule module in process.Modules) + { + if (module.ModuleName.Equals("coreclr.dll", StringComparison.OrdinalIgnoreCase)) + { + runtime = "core"; + break; + } + if (module.ModuleName.Equals("clr.dll", StringComparison.OrdinalIgnoreCase)) + { + runtime = "framework"; + break; + } + } + ProcessThreadCollection threads = process.Threads; + return new + { + pid, + role, + runtime, + cpuMs = Math.Round(process.TotalProcessorTime.TotalMilliseconds), + rssKiB = Math.Round(process.WorkingSet64 / 1024d), + started = process.StartTime.ToUniversalTime().ToString("O"), + threadCount = threads.Count, + threads = threads.Cast().Take(64).Select(ThreadSnapshot).ToArray(), + }; + } + catch (Exception error) when (error is ArgumentException or InvalidOperationException or Win32Exception) + { + return new { pid, unavailable = error.GetType().Name, code = error.HResult }; + } + } + + private static object ThreadSnapshot(ProcessThread thread) + { + try + { + System.Diagnostics.ThreadState state = thread.ThreadState; + return new + { + id = thread.Id, + state = (int)state, + wait = state == System.Diagnostics.ThreadState.Wait ? (int?)thread.WaitReason : null, + }; + } + catch (InvalidOperationException error) + { + return new { id = thread.Id, unavailable = error.GetType().Name }; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void WaitForStackProbe() => Thread.Sleep(Timeout.Infinite); + + private sealed record Request(string Command, string[] Args); + + private sealed class Status + { + public int? RootPid { get; set; } + public int? ExitCode { get; set; } + public int ProcessCount { get; set; } + public bool OutputClosed { get; set; } + public object[] Processes { get; set; } = []; + public string? Error { get; set; } + public int? ErrorCode { get; set; } + } +} diff --git a/dotnet/ci/WindowsWatchdog.csproj b/dotnet/ci/WindowsWatchdog.csproj new file mode 100644 index 0000000000..36a29620ed --- /dev/null +++ b/dotnet/ci/WindowsWatchdog.csproj @@ -0,0 +1,6 @@ + + + Exe + net10.0 + + diff --git a/dotnet/ci/dotnet-tools.json b/dotnet/ci/dotnet-tools.json new file mode 100644 index 0000000000..7aa5c5fc7c --- /dev/null +++ b/dotnet/ci/dotnet-tools.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "dotnet-stack": { + "version": "10.0.745401", + "commands": ["dotnet-stack"], + "rollForward": true + } + } +} diff --git a/dotnet/ci/test-watchdog-windows.test.mjs b/dotnet/ci/test-watchdog-windows.test.mjs new file mode 100644 index 0000000000..ef3d6b6068 --- /dev/null +++ b/dotnet/ci/test-watchdog-windows.test.mjs @@ -0,0 +1,312 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { test } from "node:test"; +import { runWithWatchdog } from "./test-watchdog.mjs"; +import { + collectWindowsStacks, + managedStacks, + startWindowsJob, + windowsSupervisorPath, +} from "./windows-watchdog.mjs"; + +assert.equal(process.platform, "win32", "Run these controls on Windows."); + +function outputDirectory(t) { + const directory = join( + import.meta.dirname, + `.watchdog-test-${process.pid}-${crypto.randomUUID()}`, + ); + mkdirSync(directory, { recursive: true }); + t.after(() => rmSync(directory, { recursive: true, force: true })); + return directory; +} + +function events(directory) { + return readFileSync(join(directory, "watchdog.jsonl"), "utf8") + .trim() + .split("\n") + .map(JSON.parse); +} + +function alive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error.code === "ESRCH") return false; + throw error; + } +} + +async function until(predicate, timeoutMs = 10_000) { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + assert.ok(Date.now() < deadline, "Condition did not become true in time"); + await delay(50); + } +} + +test("managed-stack artifacts allowlist frames and thread IDs, not paths or values", (t) => { + const previous = process.env.COPILOT_HMAC_KEY; + process.env.COPILOT_HMAC_KEY = "SecretType"; + t.after(() => { + if (previous === undefined) delete process.env.COPILOT_HMAC_KEY; + else process.env.COPILOT_HMAC_KEY = previous; + }); + assert.equal( + managedStacks(` +private-path secret output +Thread (0x1234): + [Native Frames] + System.Private.CoreLib!System.Threading.Thread.Sleep(int32) + Test!SecretType.Wait(class System.String[]) + Unrecognized secret content + C:\\private\\path!Method(value) +`), + "Thread (0x1234):\n[Native Frames]\n System.Private.CoreLib!System.Threading.Thread.Sleep\n Test![REDACTED].Wait", + ); + assert.equal(managedStacks("raw diagnostic error with secret"), ""); + assert.ok(managedStacks(" A!B()\n".repeat(50_000)).length <= 65_536); +}); + +test("orphaned descendants retaining pipes preserve the first failure and die with their owned job", async (t) => { + const unrelated = spawn( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { + stdio: "ignore", + }, + ); + t.after(() => unrelated.kill("SIGKILL")); + for (const code of [0, 37]) { + const directory = outputDirectory(t); + let descendants = []; + // Both ancestors exit before a heartbeat. PID enumeration by surviving + // parent IDs or taskkill /T on the exited root would miss the grandchild. + const grandchild = "setInterval(() => {}, 1000)"; + const intermediate = ` + const { spawn } = require("node:child_process"); + spawn(process.execPath, ["-e", ${JSON.stringify(grandchild)}], + { detached: true, stdio: ["ignore", 1, 2] }).unref(); + `; + const source = ` + const { spawn } = require("node:child_process"); + console.log("Test run for private.dll (.NETCoreApp,Version=v8.0)"); + spawn(process.execPath, ["-e", ${JSON.stringify(intermediate)}], + { detached: true, stdio: ["ignore", 1, 2] }).unref(); + process.exit(${code}); + `; + const started = performance.now(); + const result = await runWithWatchdog({ + command: process.execPath, + args: ["-e", source], + directory, + timeoutMs: 15_000, + intervalMs: 500, + graceMs: 50, + sample: async (processes) => { + descendants = processes; + }, + forwardOutput: false, + }); + assert.equal(result, code || 124, JSON.stringify(events(directory))); + assert.ok(performance.now() - started < 19_000); + const deadline = events(directory).find( + (event) => event.event === "deadline-exceeded", + ); + assert.equal(deadline.phase, "output-drain"); + assert.equal(deadline.framework, "net8.0"); + assert.ok(descendants.some(({ role }) => role === "node")); + assert.ok(descendants.every(({ pid }) => pid !== unrelated.pid)); + await until(() => descendants.every(({ pid }) => !alive(pid)), 3_000); + assert.ok( + alive(unrelated.pid), + "An unrelated process must not be terminated", + ); + const exits = events(directory).filter( + ({ event }) => event === "command-exit", + ); + assert.equal(exits.length, 1); + assert.equal(exits[0].exitCode, code); + } +}); + +test("terminating the supervisor closes its job and kills live descendants", async (t) => { + const directory = outputDirectory(t); + const { child, status } = startWindowsJob( + process.execPath, + [ + "-e", + ` + require("node:child_process").spawn(process.execPath, + ["-e", "setInterval(() => {}, 1000)"], { stdio: "inherit" }); + setInterval(() => {}, 1000); + `, + ], + directory, + ); + t.after(() => child.kill("SIGKILL")); + const closed = new Promise((resolve) => child.once("close", resolve)); + await until( + () => status().processes.filter(({ role }) => role === "node").length === 2, + ); + const owned = status().processes; + assert.ok(owned.every(({ threads }) => threads.length > 0)); + assert.ok(owned.every(({ runtime }) => runtime === "native")); + child.kill("SIGKILL"); + await closed; + await until(() => owned.every(({ pid }) => !alive(pid)), 3_000); +}); + +test("normal output EOF cleans up background servers without hiding the command result", async (t) => { + for (const code of [0, 43]) { + const directory = outputDirectory(t); + const result = await runWithWatchdog({ + command: process.execPath, + args: [ + "-e", + ` + const child = require("node:child_process").spawn(process.execPath, + ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + require("node:fs").writeFileSync(${JSON.stringify(join(directory, "background-pid"))}, String(child.pid)); + child.unref(); + process.exit(${code}); + `, + ], + directory, + timeoutMs: 15_000, + forwardOutput: false, + }); + assert.equal(result, code); + assert.ok( + !events(directory).some(({ event }) => event === "deadline-exceeded"), + ); + assert.ok( + events(directory).some( + ({ event }) => event === "owned-background-cleanup", + ), + ); + const pid = Number(readFileSync(join(directory, "background-pid"), "utf8")); + await until(() => !alive(pid), 3_000); + } +}); + +test("termination requests retain a failing status and clean the owned Windows job", async (t) => { + const directory = outputDirectory(t); + const result = runWithWatchdog({ + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"], + directory, + timeoutMs: 20_000, + intervalMs: 100, + graceMs: 50, + forwardOutput: false, + }); + let pid; + await until(() => { + pid = events(directory) + .flatMap((event) => event.processes ?? []) + .at(-1)?.pid; + return pid !== undefined; + }); + // Windows TerminateProcess cannot deliver POSIX signals. Exercise the SDK's + // termination handler separately from the real supervisor-kill control above. + process.emit("SIGTERM"); + assert.equal(await result, 143); + assert.ok(events(directory).some(({ event }) => event === "terminated")); + await until(() => !alive(pid), 3_000); +}); + +test("the command-line entry point accepts Windows and records its runtime", async (t) => { + const directory = outputDirectory(t); + const child = spawn( + process.execPath, + [join(import.meta.dirname, "test-watchdog.mjs"), "--version"], + { + cwd: directory, + env: { + ...process.env, + DOTNET_TEST_DEADLINE: String(Date.now() + 15_000), + }, + stdio: "ignore", + }, + ); + t.after(() => child.kill("SIGKILL")); + assert.equal(await new Promise((resolve) => child.once("exit", resolve)), 0); + const artifacts = join(directory, "TestResults"); + assert.equal(events(artifacts).at(-1).exitCode, 0); + assert.equal( + JSON.parse(readFileSync(join(artifacts, "watchdog-runtime.json"), "utf8")) + .platform, + "win32", + ); +}); + +test("Windows captures real managed waiting stacks without a memory dump or raw log", async (t) => { + const directory = outputDirectory(t); + const result = await runWithWatchdog({ + command: "dotnet", + args: [windowsSupervisorPath, "--stack-probe"], + directory, + timeoutMs: 10_000, + intervalMs: 500, + graceMs: 50, + forwardOutput: false, + }); + assert.equal(result, 124); + const captured = events(directory).filter( + ({ event }) => event === "managed-stack", + ); + assert.equal( + captured.length, + 1, + JSON.stringify( + events(directory).filter(({ event }) => event !== "processes"), + ), + ); + const stacks = readFileSync( + join(directory, `managed-stack-${captured[0].pid}.txt`), + "utf8", + ); + assert.match(stacks, /WindowsWatchdog\.WaitForStackProbe/); + assert.match(stacks, /^Thread \(0x[0-9a-f]+\):/im); + assert.ok(!stacks.includes("secret test payload")); + assert.ok( + !readFileSync(join(directory, "watchdog.jsonl"), "utf8").includes( + "secret test payload", + ), + ); + assert.ok( + !readdirSync(directory, { recursive: true }).some((file) => + /\.(dmp|nettrace|log)$/i.test(file), + ), + ); + await until(() => !alive(captured[0].pid), 3_000); +}); + +test("unsupported runtimes and failed stack collection are explicitly recorded", async (t) => { + const directory = outputDirectory(t); + const records = []; + await collectWindowsStacks( + [ + { pid: 2147483647, runtime: "core", role: "dotnet" }, + { pid: 2147483646, runtime: "framework", role: "testhost" }, + ], + directory, + (event) => records.push(event), + ); + assert.deepEqual(records[0], { + event: "managed-stack-unsupported", + pid: 2147483646, + runtime: "framework", + }); + assert.equal(records[1].event, "managed-stack-unavailable"); + assert.equal(records[1].pid, 2147483647); + assert.equal(records[1].code, 4294967295); + assert.ok( + !readdirSync(directory).some((file) => file.startsWith("managed-stack-")), + ); +}); diff --git a/dotnet/ci/test-watchdog.mjs b/dotnet/ci/test-watchdog.mjs index cf23bfbdfa..9355f5c29d 100644 --- a/dotnet/ci/test-watchdog.mjs +++ b/dotnet/ci/test-watchdog.mjs @@ -9,6 +9,7 @@ import { basename, join, resolve } from "node:path"; import { constants } from "node:os"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { collectWindowsStacks, startWindowsJob } from "./windows-watchdog.mjs"; const exec = promisify(execFile); @@ -19,7 +20,14 @@ export function progressMarker(line) { return { phase: "runtime-provisioning" }; if (/\bCoreCompile:/.test(line)) return { phase: "compilation" }; if (/\b_CopyCopilotCliToOutput:/.test(line)) return { phase: "runtime-copy" }; - if (/^Test run for /.test(line)) return { phase: "testhost-startup" }; + if (/^Test run for /.test(line)) { + const framework = /\.NETCoreApp,Version=v8\.0|net8\.0/.test(line) + ? "net8.0" + : /\.NETFramework,Version=v4\.7\.2|net472/.test(line) + ? "net472" + : undefined; + return { phase: "testhost-startup", ...(framework && { framework }) }; + } if (/^Starting test execution,/.test(line)) return { phase: "test-discovery" }; if (/^\[xUnit\.net [\d:.]+\]\s+Starting:/.test(line)) { @@ -56,7 +64,9 @@ export function ownedProcesses(output, group) { cpu: Number(match[5]), rssKiB: Number(match[6]), elapsed: match[7], - role: /^(dotnet|testhost|copilot|copilot-runtime|node|tar)$/.test(name) + role: /^(dotnet|testhost|copilot|copilot-runtime|node|tar|go|e2e\.test)$/.test( + name, + ) ? name : "other", }, @@ -129,19 +139,22 @@ export async function runWithWatchdog({ timeoutMs, intervalMs = 60_000, graceMs = 5_000, - inspect = collectProcesses, - sample = collectSamples, + inspect, + sample = process.platform === "win32" ? collectWindowsStacks : collectSamples, forwardOutput = true, + marker = progressMarker, + label = ".NET", }) { mkdirSync(directory, { recursive: true }); const started = performance.now(); let phase = "build-startup"; + let framework; let finalized = false; const record = (data) => !finalized && appendFileSync( join(directory, "watchdog.jsonl"), - `${JSON.stringify({ at: new Date().toISOString(), elapsedMs: Math.round(performance.now() - started), phase, ...data })}\n`, + `${JSON.stringify({ at: new Date().toISOString(), elapsedMs: Math.round(performance.now() - started), phase, framework, ...data })}\n`, ); record({ event: "start", timeoutMs }); if (timeoutMs <= 0) { @@ -149,14 +162,57 @@ export async function runWithWatchdog({ return 124; } - // On macOS the child leads a process group. Kill only that owned group, even - // if dotnet exits while a descendant still holds its output pipe open. - const child = spawn(command, args, { - detached: process.platform !== "win32", - stdio: ["ignore", "pipe", "pipe"], - }); + // POSIX uses an owned process group. Windows uses a supervisor in a nested + // kill-on-close Job Object; killing it also kills orphaned pipe holders. + const windows = + process.platform === "win32" + ? startWindowsJob(command, args, directory) + : undefined; + const child = + windows?.child ?? + spawn(command, args, { + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"], + }); let result; let stopping = false; + let commandExited = false; + let commandOutputClosed = false; + let backgroundCleaned = false; + let supervisorErrorRecorded = false; + const windowsStatus = () => { + const status = windows.status(); + commandOutputClosed ||= status.outputClosed === true; + if (status.exitCode !== undefined && !commandExited) { + commandExited = true; + result ??= status.exitCode; + record({ + event: "command-exit", + exitCode: status.exitCode, + pid: status.rootPid, + }); + phase = "output-drain"; + } + if (status.error && !supervisorErrorRecorded) { + supervisorErrorRecorded = true; + record({ + event: "windows-supervisor-error", + error: status.error, + code: status.errorCode, + }); + } + if (status.outputClosed && status.processCount > 0 && !backgroundCleaned) { + backgroundCleaned = true; + record({ + event: "owned-background-cleanup", + processCount: status.processCount, + }); + } + return status; + }; + inspect ??= windows + ? async () => windowsStatus().processes + : collectProcesses; let finish; const completed = new Promise((resolve) => { finish = resolve; @@ -175,19 +231,32 @@ export async function runWithWatchdog({ const processes = await inspect(child.pid); record({ event: "processes", processes }); return processes; - } catch { - record({ event: "process-snapshot-unavailable" }); + } catch (error) { + record({ + event: "process-snapshot-unavailable", + code: error.code ?? error.name, + }); return []; } }; const stop = async (reason, code) => { if (stopping) return; stopping = true; + if (windows) { + try { + windowsStatus(); + } catch (error) { + record({ + event: "windows-status-unavailable", + code: error.code ?? error.name, + }); + } + } result = result || code; record({ event: reason }); if (forwardOutput) console.error( - `[.NET watchdog] ${reason} during ${phase}; preserving diagnostics.`, + `[${label} watchdog] ${reason} during ${phase}; preserving diagnostics.`, ); clearInterval(heartbeat); clearTimeout(deadline); @@ -197,8 +266,11 @@ export async function runWithWatchdog({ if (reason === "deadline-exceeded") { try { await sample(processes, directory, record); - } catch { - record({ event: "samples-unavailable" }); + } catch (error) { + record({ + event: "samples-unavailable", + code: error.code ?? error.name, + }); } } signal("SIGTERM"); @@ -235,10 +307,11 @@ export async function runWithWatchdog({ pending += chunk; let newline; while ((newline = pending.indexOf("\n")) !== -1) { - const marker = progressMarker(pending.slice(0, newline)); - if (marker) { - if (marker.phase) phase = marker.phase; - record({ event: "progress", ...marker }); + const progress = marker(pending.slice(0, newline)); + if (progress) { + if (progress.phase) phase = progress.phase; + if (progress.framework) framework = progress.framework; + record({ event: "progress", ...progress }); } pending = pending.slice(newline + 1); } @@ -252,8 +325,27 @@ export async function runWithWatchdog({ record({ event: "spawn-error" }); }); child.on("exit", (code, exitSignal) => { - result ??= code ?? (exitSignal ? 128 + constants.signals[exitSignal] : 1); - record({ event: "command-exit", exitCode: code, signal: exitSignal }); + if (windows) { + try { + windowsStatus(); + } catch (error) { + record({ + event: "windows-status-unavailable", + code: error.code ?? error.name, + }); + } + } + result = + result || code || (exitSignal ? 128 + constants.signals[exitSignal] : 0); + if (windows && (!commandExited || !commandOutputClosed) && !result) { + result = 127; + record({ event: "windows-command-status-missing" }); + } + record({ + event: windows ? "supervisor-exit" : "command-exit", + exitCode: code, + signal: exitSignal, + }); phase = "output-drain"; }); child.on("close", () => { @@ -263,11 +355,27 @@ export async function runWithWatchdog({ const heartbeat = setInterval(() => { void snapshot(); }, intervalMs); - const deadline = setTimeout(() => { - void stop("deadline-exceeded", 124); - }, timeoutMs); + const statusPoll = + windows && + setInterval(() => { + try { + windowsStatus(); + } catch (error) { + record({ + event: "windows-status-unavailable", + code: error.code ?? error.name, + }); + } + }, 250); + const deadline = setTimeout( + () => { + void stop("deadline-exceeded", 124); + }, + Math.max(0, timeoutMs - (performance.now() - started)), + ); await completed; clearInterval(heartbeat); + clearInterval(statusPoll); clearTimeout(deadline); process.off("SIGINT", onInterrupt); process.off("SIGTERM", onTerminate); @@ -298,10 +406,10 @@ if ( if ( !Number.isFinite(deadline) || deadline <= 0 || - process.platform !== "darwin" + !["darwin", "win32"].includes(process.platform) ) { throw new Error( - "The .NET CI watchdog requires macOS and DOTNET_TEST_DEADLINE", + "The .NET CI watchdog requires macOS or Windows and DOTNET_TEST_DEADLINE", ); } process.exitCode = await runWithWatchdog({ diff --git a/dotnet/ci/test-watchdog.test.mjs b/dotnet/ci/test-watchdog.test.mjs index 4624e6f089..e5555b34ba 100644 --- a/dotnet/ci/test-watchdog.test.mjs +++ b/dotnet/ci/test-watchdog.test.mjs @@ -52,7 +52,7 @@ test("recognizes build, provisioning, test and shutdown without recording raw ou [" _DownloadCopilotCli:", "runtime-provisioning"], [" CoreCompile:", "compilation"], [" _CopyCopilotCliToOutput:", "runtime-copy"], - ["Test run for /private/path.dll (net8.0)", "testhost-startup"], + ["Test run for /private/path.dll", "testhost-startup"], ["Starting test execution, please wait...", "test-discovery"], [ "[xUnit.net 00:00:00.10] Starting: GitHub.Copilot.SDK.Test", @@ -66,6 +66,16 @@ test("recognizes build, provisioning, test and shutdown without recording raw ou ["Test Run Aborted.", "test-command-shutdown"], ]) assert.deepEqual(progressMarker(line), { phase }); + for (const [suffix, framework] of [ + ["(.NETCoreApp,Version=v8.0)", "net8.0"], + ["(net8.0)", "net8.0"], + ["(.NETFramework,Version=v4.7.2)", "net472"], + ]) { + assert.deepEqual(progressMarker(`Test run for private.dll ${suffix}`), { + phase: "testhost-startup", + framework, + }); + } assert.deepEqual( progressMarker( ' Passed GitHub.Copilot.Test.E2E.Example.Test(token: "secret") [1 s]', @@ -111,6 +121,82 @@ test("process snapshots contain only owned numeric metadata and known executable }, ], ); + assert.deepEqual( + ownedProcesses( + ` + 126 123 123 S 0.0 1024 00:01 /private/go + 127 126 123 S 0.0 1024 00:01 /private/e2e.test + 128 126 123 S 0.0 1024 00:01 /private/e2eXtest + 129 1 129 S 0.0 1024 00:01 /private/go +`, + 123, + ).map(({ pid, role }) => ({ pid, role })), + [ + { pid: 126, role: "go" }, + { pid: 127, role: "e2e.test" }, + { pid: 128, role: "other" }, + ], + ); +}); + +test("custom markers and labels replace .NET progress without persisting raw output", async (t) => { + const messages = []; + t.mock.method(console, "error", (message) => messages.push(message)); + const { directory, result } = run( + t, + ` + console.log("=== RUN TestFixture"); + console.log("CoreCompile:"); + setInterval(() => {}, 1000); + `, + { + timeoutMs: process.platform === "win32" ? 5_000 : 1_000, + marker: (line) => + line === "=== RUN TestFixture" ? { phase: "go-tests" } : null, + label: "Go", + forwardOutput: true, + }, + ); + assert.equal(await result, 124); + assert.deepEqual( + events(directory) + .filter(({ event }) => event === "progress") + .map(({ phase }) => phase), + ["go-tests"], + ); + assert.deepEqual(messages, [ + "[Go watchdog] deadline-exceeded during go-tests; preserving diagnostics.", + ]); + assert.ok( + !readFileSync(join(directory, "watchdog.jsonl"), "utf8").includes( + "TestFixture", + ), + ); +}); + +test("custom markers parse short-lived failures with output forwarding disabled", async (t) => { + for (const exit of ["process.exitCode = 37", "process.exit(37)"]) { + const { directory, result } = run( + t, + `console.log("go-package-finished"); ${exit};`, + { + marker: (line) => + line === "go-package-finished" + ? { phase: "go-package-complete" } + : null, + label: "Go", + forwardOutput: false, + }, + ); + assert.equal(await result, 37); + assert.ok( + events(directory).some( + ({ event, phase }) => + event === "progress" && phase === "go-package-complete", + ), + JSON.stringify(events(directory)), + ); + } }); test("samples omit headers and image paths and redact credentials", (t) => { @@ -129,12 +215,19 @@ test("samples omit headers and image paths and redact credentials", (t) => { }); test("forwards arguments, preserves success and failure, and records split output markers", async (t) => { + const previous = process.env.DOTNET_WATCHDOG_TEST_ENV; + process.env.DOTNET_WATCHDOG_TEST_ENV = "inherited-value"; + t.after(() => { + if (previous === undefined) delete process.env.DOTNET_WATCHDOG_TEST_ENV; + else process.env.DOTNET_WATCHDOG_TEST_ENV = previous; + }); for (const code of [0, 23]) { const { directory, result } = run( t, ` const assert = require("node:assert/strict"); assert.deepEqual(process.argv.slice(1), ["--filter", "(A|B)&C", "--blame-hang"]); + assert.equal(process.env.DOTNET_WATCHDOG_TEST_ENV, "inherited-value"); process.stdout.write(" _DownloadCopilot"); setTimeout(() => { console.log("Cli:"); @@ -172,7 +265,7 @@ test("a hung command is sampled before termination and fails within the inner bu setInterval(() => {}, 1000); `, { - timeoutMs: 1_000, + timeoutMs: process.platform === "win32" ? 5_000 : 1_000, sample: async () => { sampled = true; throw new Error("unavailable"); diff --git a/dotnet/ci/windows-watchdog.mjs b/dotnet/ci/windows-watchdog.mjs new file mode 100644 index 0000000000..45c62ef861 --- /dev/null +++ b/dotnet/ci/windows-watchdog.mjs @@ -0,0 +1,203 @@ +import { spawn } from "node:child_process"; +import { + closeSync, + mkdirSync, + openSync, + readSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const windowsSupervisorPath = fileURLToPath( + new URL("./bin/Release/net10.0/WindowsWatchdog.dll", import.meta.url), +); + +export function startWindowsJob(command, args, directory, cwd) { + const statusPath = join(directory, "windows-job.jsonl"); + writeFileSync(statusPath, ""); + const child = spawn("dotnet", [windowsSupervisorPath, statusPath], { + stdio: ["pipe", "pipe", "pipe"], + cwd, + windowsHide: true, + }); + // Arguments are transmitted in memory, not persisted or shell-interpolated. + child.stdin.on("error", (error) => { + if (error.code !== "EPIPE" && error.code !== "EOF") + child.emit("error", error); + }); + child.stdin.end(`${JSON.stringify({ command, args })}\n`); + let offset = 0; + let pending = ""; + let last = { processes: [] }; + return { + child, + status() { + let fd; + try { + fd = openSync(statusPath, "r"); + } catch (error) { + if (error.code === "ENOENT") return last; + throw error; + } + try { + const buffer = Buffer.alloc(1024 * 1024); + const count = readSync(fd, buffer, 0, buffer.length, offset); + offset += count; + pending += buffer.toString("utf8", 0, count); + const end = pending.lastIndexOf("\n"); + if (end !== -1) { + const start = pending.lastIndexOf("\n", end - 1) + 1; + last = JSON.parse(pending.slice(start, end)); + pending = pending.slice(end + 1); + } + if (pending.length > buffer.length) + throw new Error("Windows status exceeded its size limit"); + return last; + } finally { + closeSync(fd); + } + }, + }; +} + +async function stackReport(pid, directory) { + const started = performance.now(); + const statusDirectory = join(directory, `stack-collector-${pid}`); + mkdirSync(statusDirectory, { recursive: true }); + const { child, status } = startWindowsJob( + "dotnet", + [ + "tool", + "run", + "dotnet-stack", + "--", + "report", + "--process-id", + String(pid), + ], + statusDirectory, + import.meta.dirname, + ); + // The tool launcher can also retain a child's pipes. Use the same owned job + // as the test command instead of execFile's parent-only Windows timeout kill. + return await new Promise((resolve, reject) => { + let output = ""; + let size = 0; + let failure; + let forceClose; + let settled = false; + const finish = (error) => { + if (settled) return; + settled = true; + clearTimeout(deadline); + clearTimeout(forceClose); + if (error) reject(error); + else resolve(output); + }; + const stop = (code) => { + failure ??= Object.assign(new Error(code), { code }); + child.kill("SIGKILL"); + forceClose ??= setTimeout(() => { + child.stdout.destroy(); + child.stderr.destroy(); + child.unref(); + finish(failure); + }, 1_000); + }; + const deadline = setTimeout( + () => stop("STACK_TIMEOUT"), + Math.max(0, 10_000 - (performance.now() - started)), + ); + for (const stream of [child.stdout, child.stderr]) { + stream.setEncoding("utf8"); + stream.on("data", (chunk) => { + size += Buffer.byteLength(chunk); + if (size > 4 * 1024 * 1024) stop("STACK_OUTPUT_LIMIT"); + else if (stream === child.stdout) output += chunk; + }); + } + child.on("error", (error) => { + failure ??= error; + }); + child.on("close", (code) => { + try { + if (failure) finish(failure); + else if (code !== 0 || status().exitCode !== 0) + finish(Object.assign(new Error("Stack collector failed"), { code })); + else finish(); + } catch (error) { + finish(error); + } + }); + }); +} + +export function managedStacks(output) { + // dotnet-stack prints type/method signatures, never values. Retain only + // thread IDs, native boundaries and module!method names, excluding headers, + // paths and even parameter signatures. Do not persist the raw EventPipe data. + const lines = output.split(/\r?\n/).flatMap((line) => { + if (/^Thread \(0x[0-9a-f]+\):$/i.test(line)) return [line]; + if (/^\s+\[Native Frames\]$/.test(line)) return [line.trim()]; + const frame = + /^\s+([A-Za-z0-9_.$+`<>,[\]:-]+![A-Za-z0-9_.$+`<>,[\]:-]+)(?:\(|$)/.exec( + line, + ); + return frame ? [` ${frame[1]}`] : []; + }); + let result = lines.slice(0, 4096).join("\n").slice(0, 65_536); + if (lines.length > 4096 || lines.join("\n").length > 65_536) + result = `${result.slice(0, 65_500)}\n[Stack output truncated]`; + for (const name of [ + "COPILOT_HMAC_KEY", + "GH_TOKEN", + "GITHUB_TOKEN", + "COPILOT_GITHUB_TOKEN", + ]) { + if (process.env[name]) + result = result.replaceAll(process.env[name], "[REDACTED]"); + } + return result; +} + +export async function collectWindowsStacks(processes, directory, record) { + const managed = processes.filter(({ runtime }) => runtime === "core"); + for (const { pid, runtime } of processes) { + if (runtime === "framework") + record({ event: "managed-stack-unsupported", pid, runtime }); + } + // Testhosts first, then the build/test orchestrators. Four ten-second caps + // leave most of the four-minute job reserve for cleanup and artifact upload. + managed.sort( + (a, b) => + Number(b.role.startsWith("testhost")) - + Number(a.role.startsWith("testhost")), + ); + if (managed.length > 4) + record({ + event: "managed-stack-target-limit", + available: managed.length, + limit: 4, + }); + for (const { pid } of managed.slice(0, 4)) { + try { + const stdout = await stackReport(pid, directory); + const stacks = managedStacks(stdout); + if (!stacks) { + record({ event: "managed-stack-empty", pid }); + continue; + } + writeFileSync(join(directory, `managed-stack-${pid}.txt`), stacks); + record({ event: "managed-stack", pid }); + } catch (error) { + record({ + event: "managed-stack-unavailable", + pid, + code: error.code, + signal: error.signal, + killed: error.killed, + }); + } + } +} diff --git a/go/.gitignore b/go/.gitignore index 266339f383..8b305cce00 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -22,3 +22,6 @@ go.work # env file .env + +# CI diagnostic artifacts +TestResults/ diff --git a/go/README.md b/go/README.md index 2eb2c720fc..b4e1cc0cbc 100644 --- a/go/README.md +++ b/go/README.md @@ -1075,6 +1075,19 @@ cd go ./test.sh ``` +The macOS in-process CI job wraps this same command with the shared test +watchdog. It captures process metadata and native call stacks before the +20-minute job deadline, then fails and terminates its owned process group if the +command or its output pipes remain stuck. The Go test selection, race detector, +and per-package timeout are unchanged. + +The E2E test process also records startup/completion and all goroutine stacks +one minute before the watchdog deadline, independently of `go test`'s buffered +package output. CI uploads these files from `go/TestResults/`. They contain call +frames, not heap dumps, RPC payloads, or arbitrary test logs. Diagnostics are +opt-in through `GO_TEST_DIAGNOSTIC_DIRECTORY` and the epoch-millisecond +`GO_TEST_DIAGNOSTIC_CAPTURE_AT`; ordinary local tests are unaffected. + ## License MIT diff --git a/go/ci/test-watchdog.mjs b/go/ci/test-watchdog.mjs new file mode 100644 index 0000000000..65308746d2 --- /dev/null +++ b/go/ci/test-watchdog.mjs @@ -0,0 +1,69 @@ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + collectSamples, + runWithWatchdog, +} from "../../dotnet/ci/test-watchdog.mjs"; + +export function goProgressMarker(line) { + if (line === "=== Running Go SDK E2E Tests ===") + return { phase: "go-build-and-test" }; + // Go buffers a package's verbose output until it exits. In-process TestMain + // writes its own phase and goroutine artifacts without relying on this pipe. + const result = + /^(ok|FAIL|\?)\s+(github\.com\/github\/copilot-sdk\/go(?:\/[A-Za-z0-9_.-]+)*)(?=\s|$)/.exec( + line, + ); + return result ? { outcome: result[1], package: result[2] } : null; +} + +export function diagnosticBudget(deadline, now = Date.now()) { + if (!Number.isFinite(deadline) || deadline <= 0) + throw new Error("The Go CI watchdog requires GO_TEST_DEADLINE"); + const timeoutMs = Math.min(15 * 60_000, deadline - now); + return { timeoutMs, captureAt: now + timeoutMs - 60_000 }; +} + +export function prioritizeGoProcesses(processes) { + const rank = ({ role }) => (role === "e2e.test" ? 0 : role === "go" ? 1 : 2); + return [...processes].sort((a, b) => rank(a) - rank(b)); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + if (process.platform !== "darwin") + throw new Error("The Go CI watchdog requires macOS"); + const directory = resolve("TestResults"); + const { timeoutMs, captureAt } = diagnosticBudget( + Number(process.env.GO_TEST_DEADLINE), + ); + mkdirSync(directory, { recursive: true }); + const { copilotCliVersion } = JSON.parse( + readFileSync(new URL("../../nodejs/package.json", import.meta.url)), + ); + writeFileSync( + join(directory, "watchdog-runtime.json"), + JSON.stringify({ + copilotCliVersion, + platform: process.platform, + arch: process.arch, + node: process.version, + captureAt, + }), + ); + process.env.GO_TEST_DIAGNOSTIC_DIRECTORY = directory; + process.env.GO_TEST_DIAGNOSTIC_CAPTURE_AT = String(captureAt); + process.exitCode = await runWithWatchdog({ + command: "/bin/bash", + args: ["test.sh"], + directory, + timeoutMs, + marker: goProgressMarker, + label: "Go", + sample: (processes, directory, record) => + collectSamples(prioritizeGoProcesses(processes), directory, record), + }); +} diff --git a/go/ci/test-watchdog.test.mjs b/go/ci/test-watchdog.test.mjs new file mode 100644 index 0000000000..aa974cb40d --- /dev/null +++ b/go/ci/test-watchdog.test.mjs @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { runWithWatchdog } from "../../dotnet/ci/test-watchdog.mjs"; +import { + diagnosticBudget, + goProgressMarker, + prioritizeGoProcesses, +} from "./test-watchdog.mjs"; + +test("only package progress is retained, without arbitrary output or arguments", () => { + assert.deepEqual(goProgressMarker("=== Running Go SDK E2E Tests ==="), { + phase: "go-build-and-test", + }); + for (const outcome of ["ok", "FAIL", "?"]) { + assert.deepEqual( + goProgressMarker( + `${outcome}\tgithub.com/github/copilot-sdk/go/internal/e2e\tsecret-value`, + ), + { outcome, package: "github.com/github/copilot-sdk/go/internal/e2e" }, + ); + } + for (const line of [ + "secret-value", + "=== RUN TestWithSecret/secret-value", + "ok github.com/github/copilot-sdk/go/internal/e2e?secret-value", + ]) { + assert.equal(goProgressMarker(line), null); + } +}); + +test("capture precedes the earlier of the command and absolute job deadlines", () => { + const now = 2_000_000; + assert.deepEqual(diagnosticBudget(now + 18 * 60_000, now), { + timeoutMs: 15 * 60_000, + captureAt: now + 14 * 60_000, + }); + + assert.deepEqual(diagnosticBudget(now + 5 * 60_000, now), { + timeoutMs: 5 * 60_000, + captureAt: now + 4 * 60_000, + }); + assert.equal(diagnosticBudget(now - 1, now).timeoutMs, -1); + for (const deadline of [NaN, Infinity, 0, -1]) + assert.throws(() => diagnosticBudget(deadline, now), /GO_TEST_DEADLINE/); +}); + +test("the shared watchdog uses Go markers and preserves the command failure", async () => { + const directory = mkdtempSync(join(tmpdir(), "go-watchdog-")); + try { + const code = await runWithWatchdog({ + command: process.execPath, + args: [ + "-e", + "console.log('=== Running Go SDK E2E Tests ==='); console.log('secret-value'); process.exitCode = 37;", + ], + directory, + timeoutMs: 10_000, + forwardOutput: false, + inspect: async () => [], + marker: goProgressMarker, + label: "Go", + }); + + assert.equal(code, 37); + const timeline = readFileSync(join(directory, "watchdog.jsonl"), "utf8"); + assert.match(timeline, /"phase":"go-build-and-test"/); + assert.doesNotMatch(timeline, /secret-value/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("native sampling prioritizes the in-process E2E host over launcher processes", () => { + const processes = [ + ...Array.from({ length: 8 }, (_, pid) => ({ pid, role: "other" })), + { pid: 100, role: "go" }, + { pid: 101, role: "e2e.test" }, + ]; + assert.deepEqual(prioritizeGoProcesses(processes).slice(0, 2), [ + { pid: 101, role: "e2e.test" }, + { pid: 100, role: "go" }, + ]); + assert.equal(processes[0].pid, 0); +}); diff --git a/go/internal/e2e/main_test.go b/go/internal/e2e/main_test.go new file mode 100644 index 0000000000..45702355de --- /dev/null +++ b/go/internal/e2e/main_test.go @@ -0,0 +1,12 @@ +package e2e + +import ( + "os" + "testing" + + "github.com/github/copilot-sdk/go/internal/testdiagnostics" +) + +func TestMain(m *testing.M) { + os.Exit(testdiagnostics.Run(m.Run)) +} diff --git a/go/internal/testdiagnostics/diagnostics.go b/go/internal/testdiagnostics/diagnostics.go new file mode 100644 index 0000000000..68202af4d4 --- /dev/null +++ b/go/internal/testdiagnostics/diagnostics.go @@ -0,0 +1,100 @@ +// Package testdiagnostics captures test-process state before the CI job deadline. +package testdiagnostics + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" +) + +// Run preserves the test runner's result and optionally records pre-timeout +// goroutine stacks. The external CI watchdog owns termination and native samples. +func Run(run func() int) int { + directory := os.Getenv("GO_TEST_DIAGNOSTIC_DIRECTORY") + if directory == "" { + return run() + } + captureAt, err := strconv.ParseInt(os.Getenv("GO_TEST_DIAGNOSTIC_CAPTURE_AT"), 10, 64) + if err != nil || captureAt <= 0 { + fmt.Fprintln(os.Stderr, "Go diagnostics require a positive GO_TEST_DIAGNOSTIC_CAPTURE_AT timestamp") + return 1 + } + directory, err = filepath.Abs(directory) + if err != nil { + fmt.Fprintln(os.Stderr, "Go diagnostic directory:", err) + return 1 + } + return runMonitored(run, directory, time.UnixMilli(captureAt), func() error { + return captureStacks(directory) + }) +} + +func runMonitored(run func() int, directory string, captureAt time.Time, capture func() error) int { + if err := os.MkdirAll(directory, 0700); err != nil { + fmt.Fprintln(os.Stderr, "Creating Go diagnostic directory:", err) + return 1 + } + record := func(phase string, code *int) error { + data, err := json.Marshal(struct { + Phase string `json:"phase"` + At time.Time `json:"at"` + PID int `json:"pid"` + Go string `json:"go"` + ExitCode *int `json:"exitCode,omitempty"` + }{phase, time.Now().UTC(), os.Getpid(), runtime.Version(), code}) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(directory, "go-test-process.json"), data, 0600) + } + if err := record("tests-started", nil); err != nil { + fmt.Fprintln(os.Stderr, "Recording Go test startup:", err) + return 1 + } + captured := make(chan error, 1) + timer := time.AfterFunc(time.Until(captureAt), func() { + err := capture() + if err != nil { + fmt.Fprintln(os.Stderr, "Capturing Go goroutines:", err) + } + captured <- err + }) + code := run() + if !timer.Stop() { + if err := <-captured; err != nil && code == 0 { + code = 1 + } + } + if err := record("tests-finished", &code); err != nil { + fmt.Fprintln(os.Stderr, "Recording Go test completion:", err) + if code == 0 { + code = 1 + } + } + return code +} + +func captureStacks(directory string) error { + // runtime.Stack reports call frames and numeric arguments, not heap contents, + // RPC payloads, environment variables, or arbitrary test output. + for size := 64 * 1024; size <= 16*1024*1024; size *= 2 { + buffer := make([]byte, size) + n := runtime.Stack(buffer, true) + if n == len(buffer) { + continue + } + stacks := string(buffer[:n]) + for _, name := range []string{"COPILOT_HMAC_KEY", "GH_TOKEN", "GITHUB_TOKEN", "COPILOT_GITHUB_TOKEN"} { + if value := os.Getenv(name); value != "" { + stacks = strings.ReplaceAll(stacks, value, "[REDACTED]") + } + } + return os.WriteFile(filepath.Join(directory, "go-goroutines.txt"), []byte(stacks), 0600) + } + return fmt.Errorf("goroutine stacks exceeded the 16 MiB diagnostic limit") +} diff --git a/go/internal/testdiagnostics/diagnostics_test.go b/go/internal/testdiagnostics/diagnostics_test.go new file mode 100644 index 0000000000..b3cea8a05a --- /dev/null +++ b/go/internal/testdiagnostics/diagnostics_test.go @@ -0,0 +1,141 @@ +package testdiagnostics + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestDisabledDiagnosticsPreserveResult(t *testing.T) { + t.Setenv("GO_TEST_DIAGNOSTIC_DIRECTORY", "") + if code := Run(func() int { return 37 }); code != 37 { + t.Fatalf("Run returned %d, want 37", code) + } +} + +func TestInvalidConfigurationFailsBeforeTests(t *testing.T) { + for _, timestamp := range []string{"", "invalid", "0", "-1"} { + t.Run(timestamp, func(t *testing.T) { + t.Setenv("GO_TEST_DIAGNOSTIC_DIRECTORY", t.TempDir()) + t.Setenv("GO_TEST_DIAGNOSTIC_CAPTURE_AT", timestamp) + called := false + code := Run(func() int { called = true; return 0 }) + if code != 1 || called { + t.Fatalf("code=%d, called=%v; invalid configuration must fail", code, called) + } + }) + } +} + +func TestConfiguredDiagnosticsPreserveResult(t *testing.T) { + directory := t.TempDir() + t.Setenv("GO_TEST_DIAGNOSTIC_DIRECTORY", directory) + t.Setenv("GO_TEST_DIAGNOSTIC_CAPTURE_AT", "4102444800000") + if code := Run(func() int { return 37 }); code != 37 { + t.Fatalf("configured Run returned %d, want 37", code) + } + if _, err := os.Stat(filepath.Join(directory, "go-test-process.json")); err != nil { + t.Fatalf("configured Run did not write process state: %v", err) + } +} + +func TestCompletionPreservesExitAndCancelsCapture(t *testing.T) { + for _, expected := range []int{0, 37} { + directory := t.TempDir() + code := runMonitored(func() int { return expected }, directory, time.Now().Add(time.Hour), func() error { + t.Error("capture ran after normal completion") + return nil + }) + data, err := os.ReadFile(filepath.Join(directory, "go-test-process.json")) + if err != nil { + t.Fatal(err) + } + var state struct { + Phase string `json:"phase"` + ExitCode int `json:"exitCode"` + } + if err := json.Unmarshal(data, &state); err != nil { + t.Fatal(err) + } + if code != expected || state.ExitCode != expected || state.Phase != "tests-finished" { + t.Fatalf("code=%d, state=%+v, want exit %d", code, state, expected) + } + if _, err := os.Stat(filepath.Join(directory, "go-goroutines.txt")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected stack artifact: %v", err) + } + } +} + +func TestCaptureRunsBeforeTestCleanup(t *testing.T) { + directory := t.TempDir() + captured := make(chan struct{}) + code := runMonitored(func() int { + <-captured + return 37 + }, directory, time.Now(), func() error { + defer close(captured) + if err := captureStacks(directory); err != nil { + return err + } + data, err := os.ReadFile(filepath.Join(directory, "go-test-process.json")) + if err == nil && !strings.Contains(string(data), `"phase":"tests-started"`) { + t.Errorf("capture did not precede cleanup: %s", data) + } + return err + }) + if code != 37 { + t.Fatalf("capture changed first failure to %d", code) + } + data, err := os.ReadFile(filepath.Join(directory, "go-goroutines.txt")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), ".TestCaptureRunsBeforeTestCleanup.func1(") { + t.Fatalf("timed capture did not include the blocked test runner: %s", data) + } +} + +func TestCaptureFailureIsNotSuccess(t *testing.T) { + for _, expected := range []int{0, 37} { + captured := make(chan struct{}) + code := runMonitored(func() int { <-captured; return expected }, t.TempDir(), time.Now(), func() error { + defer close(captured) + return errors.New("controlled capture failure") + }) + want := expected + if want == 0 { + want = 1 + } + if code != want { + t.Fatalf("code=%d, want %d", code, want) + } + } +} + +func diagnosticBlockedRoutine(ready chan<- struct{}, release <-chan struct{}, done chan<- struct{}) { + close(ready) + <-release + close(done) +} + +func TestCaptureIncludesBlockedGoroutine(t *testing.T) { + directory := t.TempDir() + ready, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + go diagnosticBlockedRoutine(ready, release, done) + <-ready + defer func() { close(release); <-done }() + if err := captureStacks(directory); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(directory, "go-goroutines.txt")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), ".diagnosticBlockedRoutine(") || !strings.Contains(string(data), "[chan receive]") { + t.Fatalf("missing blocked goroutine in stack capture: %s", data) + } +} From 77127f86dec3990e7bbfe809f177a3d94e63d6d6 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 18:30:50 -0400 Subject: [PATCH 13/14] Complete the Python persisted-session fixture before cleanup Use the existing synthetic inference response helpers and observe the completed turn before saving and listing session metadata. Preserve nonempty discriminator assertions and always stop the per-test client when detach fails. Add shared-scenario ordering and cleanup controls. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- python/e2e/test_rpc_server_e2e.py | 90 ++++++++++++++++------------ python/test_rpc_server_fixture.py | 97 +++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 37 deletions(-) create mode 100644 python/test_rpc_server_fixture.py diff --git a/python/e2e/test_rpc_server_e2e.py b/python/e2e/test_rpc_server_e2e.py index 83dc4a01e2..044121e0cb 100644 --- a/python/e2e/test_rpc_server_e2e.py +++ b/python/e2e/test_rpc_server_e2e.py @@ -11,9 +11,10 @@ from datetime import UTC, datetime from pathlib import Path +import httpx import pytest -from copilot import CopilotClient, RuntimeConnection +from copilot import CopilotClient, CopilotRequestContext, CopilotRequestHandler, RuntimeConnection from copilot.rpc import ( AccountGetQuotaRequest, AgentsDiscoverRequest, @@ -55,7 +56,14 @@ ) from copilot.session import PermissionHandler -from .testharness import E2ETestContext, is_inprocess_transport, wait_for_condition +from ._copilot_request_helpers import ( + SYNTHETIC_TEXT, + assistant_text, + build_inference_response, + build_non_inference_response, + is_inference_url, +) +from .testharness import E2ETestContext, is_inprocess_transport pytestmark = pytest.mark.asyncio(loop_scope="module") @@ -89,7 +97,12 @@ async def authed_ctx(ctx: E2ETestContext): return ctx -def _make_authed_client(ctx: E2ETestContext, token: str) -> CopilotClient: +def _make_authed_client( + ctx: E2ETestContext, + token: str, + *, + request_handler: CopilotRequestHandler | None = None, +) -> CopilotClient: env = ctx.get_env() env["COPILOT_DEBUG_GITHUB_API_URL"] = ctx.proxy_url return CopilotClient( @@ -97,9 +110,21 @@ def _make_authed_client(ctx: E2ETestContext, token: str) -> CopilotClient: working_directory=ctx.work_dir, env=env, github_token=token, + request_handler=request_handler, ) +class _PersistedSessionRequestHandler(CopilotRequestHandler): + """Complete the metadata fixture's real turn without a live inference request.""" + + async def send_request( + self, request: httpx.Request, ctx: CopilotRequestContext + ) -> httpx.Response: + if is_inference_url(str(request.url)): + return build_inference_response(request) + return build_non_inference_response(str(request.url), supported_endpoints=["/responses"]) + + def _make_client_with_env(ctx: E2ETestContext, env_overrides: dict[str, str]) -> CopilotClient: env = ctx.get_env() env.update(env_overrides) @@ -295,7 +320,9 @@ async def test_should_list_find_and_inspect_persisted_session_state( ): token = os.environ.get("GITHUB_TOKEN", "fakevalue") await _configure_user(authed_ctx, token) - client = _make_authed_client(authed_ctx, token) + client = _make_authed_client( + authed_ctx, token, request_handler=_PersistedSessionRequestHandler() + ) session_id = str(uuid.uuid4()) working_directory = Path(authed_ctx.work_dir) / f"server-rpc-list-{uuid.uuid4().hex}" @@ -311,33 +338,20 @@ async def test_should_list_find_and_inspect_persisted_session_state( on_permission_request=PermissionHandler.approve_all, ) - await session.send( - "Record a turn for sessions.list discriminator coverage", mode="enqueue" + # A user turn makes sessions.list nonempty. Finish a synthetic turn + # before inspecting persistence or detaching; enqueue alone leaves + # unobserved inference racing cleanup. + message = await session.send_and_wait( + "Record a turn for sessions.list discriminator coverage", timeout=60.0 ) - - listed = None - - async def session_is_listed() -> bool: - nonlocal listed - # Re-save on every attempt: on slower runners the enqueued turn is not - # necessarily recorded yet when the first save runs, so a single save - # followed by a fixed sleep races the CLI's own persistence. - save = await client.rpc.sessions.save(SessionsSaveRequest(session_id=session_id)) - assert save is not None - listed = await client.rpc.sessions.list( - SessionsListRequest( - filter=SessionListFilter(cwd=str(working_directory)), - metadata_limit=0, - ) + assert assistant_text(message) == SYNTHETIC_TEXT + save = await client.rpc.sessions.save(SessionsSaveRequest(session_id=session_id)) + assert save is not None + listed = await client.rpc.sessions.list( + SessionsListRequest( + filter=SessionListFilter(cwd=str(working_directory)), + metadata_limit=0, ) - return any(item.session_id == session_id for item in listed.sessions or []) - - await wait_for_condition( - session_is_listed, - timeout=60.0, - timeout_message=( - "Timed out waiting for the saved session to be returned by sessions.list." - ), ) assert listed is not None @@ -379,15 +393,17 @@ async def session_is_listed() -> bool: ) assert missing_session_id not in in_use.in_use finally: - if session is not None: - await session.disconnect() try: - await client.stop() - except ExceptionGroup: - # Intentional: shutting down the per-test client can race the - # CLI's own teardown and surface as an aggregated cancellation - # error from anyio. We don't want it to fail the test. - pass + if session is not None: + await session.disconnect() + finally: + try: + await client.stop() + except ExceptionGroup: + # Intentional: shutting down the per-test client can race the + # CLI's own teardown and surface as an aggregated cancellation + # error from anyio. We don't want it to fail the test. + pass async def test_should_enrich_basic_session_metadata(self, ctx: E2ETestContext): session_id = str(uuid.uuid4()) diff --git a/python/test_rpc_server_fixture.py b/python/test_rpc_server_fixture.py new file mode 100644 index 0000000000..4467eba68c --- /dev/null +++ b/python/test_rpc_server_fixture.py @@ -0,0 +1,97 @@ +"""Regression controls for the persisted-session E2E fixture's completion fence.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from copilot.generated.session_events import AssistantMessageData +from copilot.rpc import LocalSessionMetadataValue, SessionContext +from e2e import test_rpc_server_e2e as scenario + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disconnect_fails", [False, True]) +async def test_persisted_session_fixture_observes_completion_before_save_and_cleanup( + tmp_path, monkeypatch, disconnect_fails +): + state = {} + calls = [] + + async def send(*args, **kwargs): + calls.append("send-without-completion") + + async def send_and_wait(prompt, timeout): + assert prompt == "Record a turn for sessions.list discriminator coverage" + assert timeout == 60.0 + calls.append("completed-turn") + return SimpleNamespace( + data=AssistantMessageData( + content=scenario.SYNTHETIC_TEXT, message_id="fixture-assistant" + ) + ) + + async def save(request): + assert calls == ["completed-turn"], "persistence must follow observed turn completion" + assert request.session_id == state["session_id"] + calls.append("save") + return object() + + async def listed(request): + assert calls == ["completed-turn", "save"] + return SimpleNamespace( + sessions=[ + LocalSessionMetadataValue( + session_id=state["session_id"], + is_remote=False, + start_time="2026-01-01T00:00:00Z", + modified_time="2026-01-01T00:00:00Z", + context=SessionContext(cwd=state["working_directory"]), + ) + ] + ) + + async def disconnect(): + calls.append("disconnect") + if disconnect_fails: + raise RuntimeError("controlled detach failure") + + session = SimpleNamespace(send=send, send_and_wait=send_and_wait, disconnect=disconnect) + + async def create_session(**kwargs): + state.update(kwargs) + return session + + client = SimpleNamespace( + start=AsyncMock(), + stop=AsyncMock(), + create_session=create_session, + rpc=SimpleNamespace( + sessions=SimpleNamespace( + save=save, + list=listed, + find_by_prefix=AsyncMock(return_value=SimpleNamespace(session_id=None)), + find_by_task_id=AsyncMock(return_value=SimpleNamespace(session_id=None)), + get_last_for_context=AsyncMock(return_value=SimpleNamespace(session_id=None)), + get_sizes=AsyncMock(return_value=SimpleNamespace(sizes={})), + check_in_use=AsyncMock(return_value=SimpleNamespace(in_use=[])), + ) + ), + ) + + def make_client(ctx, token, **kwargs): + if "request_handler" in kwargs: + assert isinstance(kwargs["request_handler"], scenario._PersistedSessionRequestHandler) + return client + + monkeypatch.setattr(scenario, "_configure_user", AsyncMock()) + monkeypatch.setattr(scenario, "_make_authed_client", make_client) + run = scenario.TestRpcServer().test_should_list_find_and_inspect_persisted_session_state + ctx = SimpleNamespace(work_dir=str(tmp_path)) + if disconnect_fails: + with pytest.raises(RuntimeError, match="controlled detach failure"): + await run(ctx) + else: + await run(ctx) + assert calls == ["completed-turn", "save", "disconnect"] + client.stop.assert_awaited_once() From 7f5737e9087f3a1b74d53a6297f66ef504a80acd Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Sun, 13 Sep 2026 19:15:50 -0400 Subject: [PATCH 14/14] Remove investigation-only CI instrumentation Restore all three SDK workflows to the PR base and remove the now-unused .NET and Go watchdogs, stack collectors, diagnostic TestMain, supporting tests and documentation. Remove the Python manual reproduction dispatch documentation. Keep the completion fixes, Python interrupted-task cleanup and their regression coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/dotnet-sdk-tests.yml | 35 +- .github/workflows/go-sdk-tests.yml | 25 +- .github/workflows/python-sdk-tests.yml | 70 +-- dotnet/ci/README.md | 86 ---- dotnet/ci/WindowsJob.cs | 126 ------ dotnet/ci/WindowsWatchdog.cs | 205 --------- dotnet/ci/WindowsWatchdog.csproj | 6 - dotnet/ci/dotnet-tools.json | 11 - dotnet/ci/test-watchdog-windows.test.mjs | 312 ------------- dotnet/ci/test-watchdog.mjs | 420 ------------------ dotnet/ci/test-watchdog.test.mjs | 409 ----------------- dotnet/ci/windows-watchdog.mjs | 203 --------- go/.gitignore | 3 - go/README.md | 13 - go/ci/test-watchdog.mjs | 69 --- go/ci/test-watchdog.test.mjs | 87 ---- go/internal/e2e/main_test.go | 12 - go/internal/testdiagnostics/diagnostics.go | 100 ----- .../testdiagnostics/diagnostics_test.go | 141 ------ python/README.md | 17 +- 20 files changed, 8 insertions(+), 2342 deletions(-) delete mode 100644 dotnet/ci/README.md delete mode 100644 dotnet/ci/WindowsJob.cs delete mode 100644 dotnet/ci/WindowsWatchdog.cs delete mode 100644 dotnet/ci/WindowsWatchdog.csproj delete mode 100644 dotnet/ci/dotnet-tools.json delete mode 100644 dotnet/ci/test-watchdog-windows.test.mjs delete mode 100644 dotnet/ci/test-watchdog.mjs delete mode 100644 dotnet/ci/test-watchdog.test.mjs delete mode 100644 dotnet/ci/windows-watchdog.mjs delete mode 100644 go/ci/test-watchdog.mjs delete mode 100644 go/ci/test-watchdog.test.mjs delete mode 100644 go/internal/e2e/main_test.go delete mode 100644 go/internal/testdiagnostics/diagnostics.go delete mode 100644 go/internal/testdiagnostics/diagnostics_test.go diff --git a/.github/workflows/dotnet-sdk-tests.yml b/.github/workflows/dotnet-sdk-tests.yml index 671118eeb7..f12f53bd96 100644 --- a/.github/workflows/dotnet-sdk-tests.yml +++ b/.github/workflows/dotnet-sdk-tests.yml @@ -193,12 +193,6 @@ jobs: working-directory: ./dotnet steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - # Reserve time for bounded sampling, process cleanup and artifact upload - # before the job deadline, including time spent in setup below. - - name: Reserve .NET diagnostic budget - if: (runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1' - run: echo "DOTNET_TEST_DEADLINE=$(( ($(date +%s) + 16 * 60) * 1000 ))" >> "$GITHUB_ENV" - - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 with: dotnet-version: "10.0.x" @@ -225,28 +219,11 @@ jobs: if: runner.os == 'Windows' run: pwsh.exe -Command "Write-Host 'PowerShell ready'" - - name: Restore Windows managed-stack diagnostics - if: runner.os == 'Windows' && matrix.shard == '1' - run: | - dotnet tool restore --tool-manifest ci/dotnet-tools.json - dotnet build ci/WindowsWatchdog.csproj -c Release -p:UseSharedCompilation=false - - name: Select inprocess transport if: matrix.transport == 'inprocess' run: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" - - name: Validate .NET watchdog - if: (runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1' - timeout-minutes: 1 - run: node --test --test-timeout=30000 ci/test-watchdog.test.mjs - - - name: Validate Windows .NET diagnostics - if: runner.os == 'Windows' && matrix.shard == '1' - timeout-minutes: 2 - run: node --test --test-timeout=60000 ci/test-watchdog-windows.test.mjs - - name: Run .NET SDK tests - timeout-minutes: ${{ (runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1' && 16 || 20 }} env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} DOTNET_TEST_SHARD: ${{ matrix.shard }} @@ -362,18 +339,10 @@ jobs: if [[ -n "$filter" ]]; then args+=(--filter "$filter") fi - if [[ ("$RUNNER_OS" == "macOS" || "$RUNNER_OS" == "Windows") && "$DOTNET_TEST_SHARD" == "1" ]]; then - # Preserve the exact command. The watchdog observes known progress - # markers and samples only its process group / Windows Job Object; - # it never saves raw logs or process memory. - node ci/test-watchdog.mjs test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" - else - dotnet test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" - fi + dotnet test test/GitHub.Copilot.SDK.Test.csproj "${args[@]}" - name: Upload .NET test diagnostics - if: failure() || cancelled() || ((runner.os == 'macOS' || runner.os == 'Windows') && matrix.shard == '1') - timeout-minutes: 2 + if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: dotnet-test-diagnostics-${{ matrix.os }}-${{ matrix.transport }}-${{ matrix.backend }}-${{ matrix.shard }}-${{ github.run_attempt }} diff --git a/.github/workflows/go-sdk-tests.yml b/.github/workflows/go-sdk-tests.yml index da999078a0..cde400615a 100644 --- a/.github/workflows/go-sdk-tests.yml +++ b/.github/workflows/go-sdk-tests.yml @@ -56,9 +56,6 @@ jobs: working-directory: ./go steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Reserve Go diagnostic budget - if: matrix.os == 'macos-latest' && matrix.transport == 'inprocess' - run: echo "GO_TEST_DEADLINE=$(($(date +%s) * 1000 + 16 * 60 * 1000))" >> "$GITHUB_ENV" - uses: ./.github/actions/setup-copilot id: setup-copilot - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 @@ -79,31 +76,11 @@ jobs: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" echo "GOFLAGS=-tags=copilot_inprocess" >> "$GITHUB_ENV" - - name: Validate Go watchdog - if: matrix.os == 'macos-latest' && matrix.transport == 'inprocess' - run: | - node --test ci/test-watchdog.test.mjs ../dotnet/ci/test-watchdog.test.mjs - go test ./internal/testdiagnostics -race -count=1 - - name: Run Go SDK tests env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} COPILOT_CLI_PATH: ${{ steps.setup-copilot.outputs.cli-path }} - run: | - if [ "$RUNNER_OS" = "macOS" ] && [ "$COPILOT_SDK_DEFAULT_CONNECTION" = "inprocess" ]; then - node ci/test-watchdog.mjs - else - /bin/bash test.sh - fi - - - name: Upload Go test diagnostics - if: always() && matrix.os == 'macos-latest' && matrix.transport == 'inprocess' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: go-test-diagnostics-${{ matrix.os }}-${{ matrix.transport }}-${{ github.run_attempt }} - path: go/TestResults/ - if-no-files-found: warn - retention-days: 7 + run: /bin/bash test.sh # JavaScript actions use a glibc-linked Node runtime, so Alpine runs through Docker. test-musl-arm64: diff --git a/.github/workflows/python-sdk-tests.yml b/.github/workflows/python-sdk-tests.yml index 26fc4f837a..e51ee3f3c8 100644 --- a/.github/workflows/python-sdk-tests.yml +++ b/.github/workflows/python-sdk-tests.yml @@ -5,18 +5,6 @@ env: on: workflow_dispatch: - inputs: - reproduce_timeout: - description: "Reproduce macOS/inprocess timeouts, stopping on the first failure" - type: boolean - default: false - reproduction_scope: - description: "Diagnostic scope (used only when reproduce_timeout is enabled)" - type: choice - default: full - options: - - full - - session-config workflow_call: permissions: @@ -25,7 +13,7 @@ permissions: jobs: validate: name: "Python SDK Format and Typecheck" - if: github.event.repository.fork == false && !inputs.reproduce_timeout + if: github.event.repository.fork == false runs-on: ubuntu-latest timeout-minutes: 20 defaults: @@ -58,10 +46,10 @@ jobs: strategy: fail-fast: false matrix: - os: ${{ fromJSON(inputs.reproduce_timeout && '["macos-latest"]' || '["ubuntu-latest", "macos-latest", "windows-latest"]') }} + os: [ubuntu-latest, macos-latest, windows-latest] # Test the oldest supported Python version to make sure compatibility is maintained. python-version: ["3.11"] - transport: ${{ fromJSON(inputs.reproduce_timeout && '["inprocess"]' || '["default", "inprocess"]') }} + transport: ["default", "inprocess"] runs-on: ${{ matrix.os }} timeout-minutes: 20 defaults: @@ -105,66 +93,16 @@ jobs: echo "COPILOT_SDK_DEFAULT_CONNECTION=inprocess" >> "$GITHUB_ENV" - name: Run Python SDK tests - if: ${{ !inputs.reproduce_timeout }} env: COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} # Keep each module's shared E2E client and proxy on one process while # running independent modules concurrently in isolated workers. run: uv run pytest -v -s -n 2 --dist=loadfile - - name: Reproduce macOS inprocess timeout - if: inputs.reproduce_timeout - env: - COPILOT_HMAC_KEY: ${{ secrets.COPILOT_DEVELOPER_CLI_INTEGRATION_HMAC_KEY }} - REPRODUCTION_SCOPE: ${{ inputs.reproduction_scope || 'full' }} - run: | - case "$REPRODUCTION_SCOPE" in - full) - attempts=2 - targets=() - ;; - session-config) - attempts=10 - targets=(e2e/test_session_config_e2e.py) - ;; - *) - echo "::error::Unsupported reproduction scope." - exit 64 - ;; - esac - mkdir -p .pytest-diagnostics - summary=.pytest-diagnostics/reproduction-summary.txt - printf 'scope=%s\nplanned_invocations=%s\n' "$REPRODUCTION_SCOPE" "$attempts" > "$summary" - for ((attempt=1; attempt<=attempts; attempt++)); do - printf 'started_invocation=%s\n' "$attempt" >> "$summary" - echo "::group::Reproduction ($REPRODUCTION_SCOPE) invocation $attempt/$attempts" - if uv run pytest -v -s -n 2 --dist=loadfile "${targets[@]}"; then - printf 'completed_invocation=%s\n' "$attempt" >> "$summary" - echo "::endgroup::" - else - status=$? - printf 'failed_invocation=%s\nexit_code=%s\n' "$attempt" "$status" >> "$summary" - echo "::endgroup::" - echo "::error::Invocation $attempt failed (exit $status); stopping reproduction." - exit "$status" - fi - done - echo 'reproduction_complete=true' >> "$summary" - echo "::notice::No failure reproduced in $attempts completed $REPRODUCTION_SCOPE invocations." - - - name: Upload Python timeout diagnostics - if: ${{ always() && (inputs.reproduce_timeout || failure()) }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: python-timeout-${{ matrix.os }}-${{ matrix.transport }} - path: python/.pytest-diagnostics/ - include-hidden-files: true - if-no-files-found: ignore - # JavaScript actions use a glibc-linked Node runtime, so Alpine runs through Docker. test-musl-arm64: name: "Python SDK Tests (Alpine ARM64, ${{ matrix.transport }})" - if: github.event.repository.fork == false && !inputs.reproduce_timeout + if: github.event.repository.fork == false strategy: fail-fast: false matrix: diff --git a/dotnet/ci/README.md b/dotnet/ci/README.md deleted file mode 100644 index 7e4cecb745..0000000000 --- a/dotnet/ci/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# .NET CI hang diagnostics - -The macOS and Windows default/CAPI shard 1 jobs run the **unchanged** `dotnet test` -command through `test-watchdog.mjs`. Frameworks, filters, environment, and the -10-minute per-test blame timeout are unchanged. The watchdog deadline is the -earlier of 15 minutes after command launch or 16 minutes after checkout. The -20-minute job limit is unchanged; the remaining time is for diagnostics, cleanup, -and the two-minute artifact upload. - -On Windows, the workflow builds `WindowsWatchdog.csproj` with the already -installed .NET 10 SDK and restores the pinned Microsoft `dotnet-stack` tool from -`dotnet-tools.json`. The helper joins a kill-on-close Windows Job Object **before** -starting the command. Nested jobs contain grandchildren even when ancestors exit -between snapshots. Cleanup never uses executable-name matching or an unrelated -process search. Held process handles prevent sampled PIDs from being reused. - -The root command exiting is not sufficient: the helper also waits for stdout and -stderr EOF. A descendant retaining either pipe therefore still reaches the -watchdog deadline, preserving an earlier command failure (otherwise exit 124). -After a normal root exit and EOF, any remaining background servers are cleaned -up without changing the command's result. Killing the supervisor also closes its -job and terminates its owned descendants. Supervisor/launch failures stay failures. - -## Reading the next CI artifact - -Download `dotnet-test-diagnostics-windows-latest-default-capi-1-`: - -- `watchdog.jsonl`: allowlisted build/provisioning/test/shutdown phase markers, - recognized target framework, completed test method names (no argument values), - exit-versus-output-drain timing, deadline, inspection errors, and final status. -- `windows-job.jsonl`: append-only snapshots on a five-second cadence, plus - lifecycle changes. Only owned PIDs, known executable roles, runtime kind, - CPU/RSS, process start times, and numeric thread states/wait reasons are stored. - Snapshots cover at most 128 processes and 64 threads per process; total - `processCount`/`threadCount` values expose truncation. -- `managed-stack-.txt`: at the watchdog deadline, readable .NET managed - thread stacks for up to four owned CoreCLR processes, prioritizing testhosts. - Each collector has a ten-second deadline (including startup), a one-second - forced-close bound, and its own Job Object. Collector stdout/stderr are captured - in memory only, capped at 4 MiB; artifacts retain only thread IDs, native boundaries, and - module/method names, capped at 4,096 lines / 64 KiB. Truncation and unavailable - captures are explicit. `stack-collector-/windows-job.jsonl` diagnoses the - collector itself. -- Existing TRX and blame sequence files: correlate the framework and last - completed test with the test host's existing failure/active-test evidence. -- `watchdog-runtime.json`: pinned CLI version, platform, architecture, and Node. - -`dotnet-stack` supports CoreCLR, not .NET Framework. Framework processes get an -explicit `managed-stack-unsupported` event, numeric thread information, and the -existing blame/TRX diagnostics. Native CLI stacks are not collected on Windows. -Thread stacks are not a dump of suspended async state machines; an off-thread -await may still need follow-up investigation. No heap/process dumps, environment, -command lines, arbitrary console output, or locals are added to diagnostic -artifacts. The existing TRX/blame artifact behavior is unchanged. - -Instrumentation does not establish the cause of the original Windows timeout. -Use the next failure's phase, framework, exit/EOF timing, test sequence and stacks -to distinguish provisioning, test execution, fixture disposal and pipe retention -before attributing it or changing SDK behavior. - -## Focused local validation (Windows) - -From `dotnet`: - -```powershell -dotnet tool restore --tool-manifest ci\dotnet-tools.json -dotnet build ci\WindowsWatchdog.csproj -c Release -p:UseSharedCompilation=false -dotnet format ci\WindowsWatchdog.csproj --no-restore --verify-no-changes -node --test --test-timeout=30000 ci\test-watchdog.test.mjs -node --test --test-timeout=60000 ci\test-watchdog-windows.test.mjs -``` - -The Windows-only controls cover orphaned pipe holders, original failure -preservation, unrelated-process survival, supervisor termination, normal EOF with -background servers, and real managed waiting-stack capture. `--stack-probe` on the -helper is their small managed fixture, not part of SDK test selection. The shared -suite retains its existing POSIX-only controls; run it on macOS to exercise native -sampling and process-group behavior. - -Other CI entry points may import `runWithWatchdog` and supply `command`, `args`, -an absolute artifact `directory`, and `timeoutMs`. Optional `marker` and `label` -parameters default to `progressMarker` and `".NET"`; a custom marker must return -only allowlisted metadata or `null`, never raw console output or argument values. -This lets the Go macOS entry point reuse process-group cleanup and native sampling -without duplicating the watchdog. Importing the module does not execute its .NET -CLI entry point. diff --git a/dotnet/ci/WindowsJob.cs b/dotnet/ci/WindowsJob.cs deleted file mode 100644 index c2ae086f84..0000000000 --- a/dotnet/ci/WindowsJob.cs +++ /dev/null @@ -1,126 +0,0 @@ -using System; -using System.ComponentModel; -using System.Diagnostics; -using System.Runtime.InteropServices; - -namespace GitHub.Copilot.Ci; - -// The supervisor joins before spawning anything. All descendants inherit this -// nested job, including children whose parent exits before the next snapshot. -internal sealed class WindowsJob : IDisposable -{ - private readonly IntPtr handle; - - public WindowsJob() - { - handle = CreateJobObject(IntPtr.Zero, null); - Check(handle != IntPtr.Zero); - SetKillOnClose(true); - Check(AssignProcessToJobObject(handle, Process.GetCurrentProcess().Handle)); - } - - public int[] ProcessIds() - { - // A bounded allocation, with a visible error rather than a truncated tree. - const int capacity = 4096; - IntPtr buffer = Marshal.AllocHGlobal(8 + capacity * IntPtr.Size); - try - { - Check(QueryInformationJobObject(handle, 3, buffer, 8 + capacity * IntPtr.Size, IntPtr.Zero)); - int count = Marshal.ReadInt32(buffer, 4); - int[] ids = new int[count]; - for (int i = 0; i < count; i++) - ids[i] = checked((int)Marshal.ReadIntPtr(buffer, 8 + i * IntPtr.Size)); - return ids; - } - finally - { - Marshal.FreeHGlobal(buffer); - } - } - - public void Complete() - { - // Only the supervisor may remain when disabling kill-on-close. - int[] ids = ProcessIds(); - if (ids.Length != 1 || ids[0] != Environment.ProcessId) - throw new InvalidOperationException("The owned job is not empty."); - SetKillOnClose(false); - } - - public bool Owns(IntPtr process) - { - Check(IsProcessInJob(process, handle, out bool owned)); - return owned; - } - - public void Abort(int exitCode) => Check(TerminateJobObject(handle, unchecked((uint)exitCode))); - - private void SetKillOnClose(bool enabled) - { - var limits = new ExtendedLimits(); - limits.Basic.LimitFlags = enabled ? 0x2000u : 0; - Check(SetInformationJobObject(handle, 9, ref limits, Marshal.SizeOf())); - } - - private static void Check(bool succeeded) - { - if (!succeeded) - throw new Win32Exception(Marshal.GetLastWin32Error()); - } - - public void Dispose() => Check(CloseHandle(handle)); - - [StructLayout(LayoutKind.Sequential)] - private struct BasicLimits - { - public long PerProcessUserTime, PerJobUserTime; - public uint LimitFlags; - public UIntPtr MinimumWorkingSet, MaximumWorkingSet; - public uint ActiveProcessLimit; - public UIntPtr Affinity; - public uint PriorityClass, SchedulingClass; - } - - [StructLayout(LayoutKind.Sequential)] - private struct IoCounters - { - public ulong ReadOperations, WriteOperations, OtherOperations; - public ulong ReadBytes, WriteBytes, OtherBytes; - } - - [StructLayout(LayoutKind.Sequential)] - private struct ExtendedLimits - { - public BasicLimits Basic; - public IoCounters Io; - public UIntPtr ProcessMemory, JobMemory, PeakProcessMemory, PeakJobMemory; - } - - [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] - private static extern IntPtr CreateJobObject(IntPtr attributes, string? name); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool SetInformationJobObject(IntPtr job, int infoClass, ref ExtendedLimits info, int length); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool AssignProcessToJobObject(IntPtr job, IntPtr process); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool IsProcessInJob(IntPtr process, IntPtr job, [MarshalAs(UnmanagedType.Bool)] out bool owned); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool TerminateJobObject(IntPtr job, uint exitCode); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool QueryInformationJobObject(IntPtr job, int infoClass, IntPtr info, int length, IntPtr returnedLength); - - [DllImport("kernel32.dll", SetLastError = true)] - [return: MarshalAs(UnmanagedType.Bool)] - private static extern bool CloseHandle(IntPtr handle); -} diff --git a/dotnet/ci/WindowsWatchdog.cs b/dotnet/ci/WindowsWatchdog.cs deleted file mode 100644 index 95a959e13f..0000000000 --- a/dotnet/ci/WindowsWatchdog.cs +++ /dev/null @@ -1,205 +0,0 @@ -using System.ComponentModel; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.Versioning; -using System.Text.Json; -using System.Text.Json.Serialization; - -[assembly: SupportedOSPlatform("windows")] - -namespace GitHub.Copilot.Ci; - -internal static class WindowsWatchdog -{ - private static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private static int Main(string[] args) - { - if (args is ["--stack-probe"]) - { - Console.WriteLine("secret test payload"); - WaitForStackProbe(); - return 0; - } - - var state = new Status(); - var tracked = new Dictionary(); - WindowsJob? job = null; - bool completed = false; - // Append-only IPC avoids Windows rename/delete sharing races with readers. - using var output = new StreamWriter(new FileStream(args[0], FileMode.Append, FileAccess.Write, FileShare.Read)); - output.AutoFlush = true; - try - { - job = new WindowsJob(); - var request = JsonSerializer.Deserialize(Console.ReadLine()!, JsonOptions)!; - var start = new ProcessStartInfo(request.Command) - { - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - }; - foreach (string argument in request.Args) - start.ArgumentList.Add(argument); - using var root = Process.Start(start)!; - root.StandardInput.Close(); - state.RootPid = root.Id; - Task drained = Task.WhenAll( - root.StandardOutput.BaseStream.CopyToAsync(Console.OpenStandardOutput()), - root.StandardError.BaseStream.CopyToAsync(Console.OpenStandardError())); - long nextSnapshot = 0; - do - { - int? previousExitCode = state.ExitCode; - if (root.HasExited) - state.ExitCode = root.ExitCode; - int[] ids = job.ProcessIds().Where(id => id != Environment.ProcessId).ToArray(); - bool closing = state.ExitCode.HasValue && drained.IsCompleted; - if (closing) - drained.GetAwaiter().GetResult(); - if (closing || state.ExitCode != previousExitCode || Environment.TickCount64 >= nextSnapshot) - { - state.ProcessCount = ids.Length; - state.Processes = ids.Take(128).Select(id => Snapshot(id, job, tracked)).ToArray(); - state.OutputClosed = closing; - output.WriteLine(JsonSerializer.Serialize(state, JsonOptions)); - nextSnapshot = Environment.TickCount64 + 5_000; - } - if (closing) - { - // Compiler servers may outlive a successful command without - // holding its pipes. Clean them up, but only after BOTH the - // actual root exit and output EOF, never merely root exit. - if (ids.Length != 0) - job.Abort(state.ExitCode!.Value); - break; - } - Thread.Sleep(250); - } while (true); - job.Complete(); - completed = true; - return state.ExitCode!.Value; - } - catch (Exception error) when (error is Win32Exception or InvalidOperationException or IOException or JsonException or ArgumentException) - { - state.Error = error.GetType().Name; - state.ErrorCode = error.HResult; - state.ExitCode = state.ExitCode is null or 0 ? 127 : state.ExitCode; - Console.Error.WriteLine($"[.NET watchdog] Windows supervisor failed: {state.Error}"); - output.WriteLine(JsonSerializer.Serialize(state, JsonOptions)); - return state.ExitCode.Value; - } - finally - { - foreach (Process process in tracked.Values) - process.Dispose(); - if (job is not null) - { - // Never turn an inspector/supervisor failure into success. Abort - // gives the supervisor AND its descendants a nonzero exit code. - if (!completed) - job.Abort(state.ExitCode is null or 0 ? 127 : state.ExitCode.Value); - job.Dispose(); - } - } - } - - private static object Snapshot(int pid, WindowsJob job, Dictionary tracked) - { - try - { - if (!tracked.TryGetValue(pid, out Process? process)) - { - process = Process.GetProcessById(pid); - try - { - if (!job.Owns(process.Handle)) - throw new InvalidOperationException(); - // Retain the handle while sampling so an exited PID cannot - // be recycled into an unrelated diagnostic target. - tracked.Add(pid, process); - } - catch - { - process.Dispose(); - throw; - } - } - process.Refresh(); - string role = process.ProcessName.ToLowerInvariant(); - if (role is not ("dotnet" or "testhost" or "testhost.x86" or "msbuild" or "copilot" or "copilot-runtime" or "node" or "tar" or "pwsh")) - role = "other"; - string runtime = "native"; - foreach (ProcessModule module in process.Modules) - { - if (module.ModuleName.Equals("coreclr.dll", StringComparison.OrdinalIgnoreCase)) - { - runtime = "core"; - break; - } - if (module.ModuleName.Equals("clr.dll", StringComparison.OrdinalIgnoreCase)) - { - runtime = "framework"; - break; - } - } - ProcessThreadCollection threads = process.Threads; - return new - { - pid, - role, - runtime, - cpuMs = Math.Round(process.TotalProcessorTime.TotalMilliseconds), - rssKiB = Math.Round(process.WorkingSet64 / 1024d), - started = process.StartTime.ToUniversalTime().ToString("O"), - threadCount = threads.Count, - threads = threads.Cast().Take(64).Select(ThreadSnapshot).ToArray(), - }; - } - catch (Exception error) when (error is ArgumentException or InvalidOperationException or Win32Exception) - { - return new { pid, unavailable = error.GetType().Name, code = error.HResult }; - } - } - - private static object ThreadSnapshot(ProcessThread thread) - { - try - { - System.Diagnostics.ThreadState state = thread.ThreadState; - return new - { - id = thread.Id, - state = (int)state, - wait = state == System.Diagnostics.ThreadState.Wait ? (int?)thread.WaitReason : null, - }; - } - catch (InvalidOperationException error) - { - return new { id = thread.Id, unavailable = error.GetType().Name }; - } - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void WaitForStackProbe() => Thread.Sleep(Timeout.Infinite); - - private sealed record Request(string Command, string[] Args); - - private sealed class Status - { - public int? RootPid { get; set; } - public int? ExitCode { get; set; } - public int ProcessCount { get; set; } - public bool OutputClosed { get; set; } - public object[] Processes { get; set; } = []; - public string? Error { get; set; } - public int? ErrorCode { get; set; } - } -} diff --git a/dotnet/ci/WindowsWatchdog.csproj b/dotnet/ci/WindowsWatchdog.csproj deleted file mode 100644 index 36a29620ed..0000000000 --- a/dotnet/ci/WindowsWatchdog.csproj +++ /dev/null @@ -1,6 +0,0 @@ - - - Exe - net10.0 - - diff --git a/dotnet/ci/dotnet-tools.json b/dotnet/ci/dotnet-tools.json deleted file mode 100644 index 7aa5c5fc7c..0000000000 --- a/dotnet/ci/dotnet-tools.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "version": 1, - "isRoot": true, - "tools": { - "dotnet-stack": { - "version": "10.0.745401", - "commands": ["dotnet-stack"], - "rollForward": true - } - } -} diff --git a/dotnet/ci/test-watchdog-windows.test.mjs b/dotnet/ci/test-watchdog-windows.test.mjs deleted file mode 100644 index ef3d6b6068..0000000000 --- a/dotnet/ci/test-watchdog-windows.test.mjs +++ /dev/null @@ -1,312 +0,0 @@ -import assert from "node:assert/strict"; -import { spawn } from "node:child_process"; -import { mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; -import { test } from "node:test"; -import { runWithWatchdog } from "./test-watchdog.mjs"; -import { - collectWindowsStacks, - managedStacks, - startWindowsJob, - windowsSupervisorPath, -} from "./windows-watchdog.mjs"; - -assert.equal(process.platform, "win32", "Run these controls on Windows."); - -function outputDirectory(t) { - const directory = join( - import.meta.dirname, - `.watchdog-test-${process.pid}-${crypto.randomUUID()}`, - ); - mkdirSync(directory, { recursive: true }); - t.after(() => rmSync(directory, { recursive: true, force: true })); - return directory; -} - -function events(directory) { - return readFileSync(join(directory, "watchdog.jsonl"), "utf8") - .trim() - .split("\n") - .map(JSON.parse); -} - -function alive(pid) { - try { - process.kill(pid, 0); - return true; - } catch (error) { - if (error.code === "ESRCH") return false; - throw error; - } -} - -async function until(predicate, timeoutMs = 10_000) { - const deadline = Date.now() + timeoutMs; - while (!predicate()) { - assert.ok(Date.now() < deadline, "Condition did not become true in time"); - await delay(50); - } -} - -test("managed-stack artifacts allowlist frames and thread IDs, not paths or values", (t) => { - const previous = process.env.COPILOT_HMAC_KEY; - process.env.COPILOT_HMAC_KEY = "SecretType"; - t.after(() => { - if (previous === undefined) delete process.env.COPILOT_HMAC_KEY; - else process.env.COPILOT_HMAC_KEY = previous; - }); - assert.equal( - managedStacks(` -private-path secret output -Thread (0x1234): - [Native Frames] - System.Private.CoreLib!System.Threading.Thread.Sleep(int32) - Test!SecretType.Wait(class System.String[]) - Unrecognized secret content - C:\\private\\path!Method(value) -`), - "Thread (0x1234):\n[Native Frames]\n System.Private.CoreLib!System.Threading.Thread.Sleep\n Test![REDACTED].Wait", - ); - assert.equal(managedStacks("raw diagnostic error with secret"), ""); - assert.ok(managedStacks(" A!B()\n".repeat(50_000)).length <= 65_536); -}); - -test("orphaned descendants retaining pipes preserve the first failure and die with their owned job", async (t) => { - const unrelated = spawn( - process.execPath, - ["-e", "setInterval(() => {}, 1000)"], - { - stdio: "ignore", - }, - ); - t.after(() => unrelated.kill("SIGKILL")); - for (const code of [0, 37]) { - const directory = outputDirectory(t); - let descendants = []; - // Both ancestors exit before a heartbeat. PID enumeration by surviving - // parent IDs or taskkill /T on the exited root would miss the grandchild. - const grandchild = "setInterval(() => {}, 1000)"; - const intermediate = ` - const { spawn } = require("node:child_process"); - spawn(process.execPath, ["-e", ${JSON.stringify(grandchild)}], - { detached: true, stdio: ["ignore", 1, 2] }).unref(); - `; - const source = ` - const { spawn } = require("node:child_process"); - console.log("Test run for private.dll (.NETCoreApp,Version=v8.0)"); - spawn(process.execPath, ["-e", ${JSON.stringify(intermediate)}], - { detached: true, stdio: ["ignore", 1, 2] }).unref(); - process.exit(${code}); - `; - const started = performance.now(); - const result = await runWithWatchdog({ - command: process.execPath, - args: ["-e", source], - directory, - timeoutMs: 15_000, - intervalMs: 500, - graceMs: 50, - sample: async (processes) => { - descendants = processes; - }, - forwardOutput: false, - }); - assert.equal(result, code || 124, JSON.stringify(events(directory))); - assert.ok(performance.now() - started < 19_000); - const deadline = events(directory).find( - (event) => event.event === "deadline-exceeded", - ); - assert.equal(deadline.phase, "output-drain"); - assert.equal(deadline.framework, "net8.0"); - assert.ok(descendants.some(({ role }) => role === "node")); - assert.ok(descendants.every(({ pid }) => pid !== unrelated.pid)); - await until(() => descendants.every(({ pid }) => !alive(pid)), 3_000); - assert.ok( - alive(unrelated.pid), - "An unrelated process must not be terminated", - ); - const exits = events(directory).filter( - ({ event }) => event === "command-exit", - ); - assert.equal(exits.length, 1); - assert.equal(exits[0].exitCode, code); - } -}); - -test("terminating the supervisor closes its job and kills live descendants", async (t) => { - const directory = outputDirectory(t); - const { child, status } = startWindowsJob( - process.execPath, - [ - "-e", - ` - require("node:child_process").spawn(process.execPath, - ["-e", "setInterval(() => {}, 1000)"], { stdio: "inherit" }); - setInterval(() => {}, 1000); - `, - ], - directory, - ); - t.after(() => child.kill("SIGKILL")); - const closed = new Promise((resolve) => child.once("close", resolve)); - await until( - () => status().processes.filter(({ role }) => role === "node").length === 2, - ); - const owned = status().processes; - assert.ok(owned.every(({ threads }) => threads.length > 0)); - assert.ok(owned.every(({ runtime }) => runtime === "native")); - child.kill("SIGKILL"); - await closed; - await until(() => owned.every(({ pid }) => !alive(pid)), 3_000); -}); - -test("normal output EOF cleans up background servers without hiding the command result", async (t) => { - for (const code of [0, 43]) { - const directory = outputDirectory(t); - const result = await runWithWatchdog({ - command: process.execPath, - args: [ - "-e", - ` - const child = require("node:child_process").spawn(process.execPath, - ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); - require("node:fs").writeFileSync(${JSON.stringify(join(directory, "background-pid"))}, String(child.pid)); - child.unref(); - process.exit(${code}); - `, - ], - directory, - timeoutMs: 15_000, - forwardOutput: false, - }); - assert.equal(result, code); - assert.ok( - !events(directory).some(({ event }) => event === "deadline-exceeded"), - ); - assert.ok( - events(directory).some( - ({ event }) => event === "owned-background-cleanup", - ), - ); - const pid = Number(readFileSync(join(directory, "background-pid"), "utf8")); - await until(() => !alive(pid), 3_000); - } -}); - -test("termination requests retain a failing status and clean the owned Windows job", async (t) => { - const directory = outputDirectory(t); - const result = runWithWatchdog({ - command: process.execPath, - args: ["-e", "setInterval(() => {}, 1000)"], - directory, - timeoutMs: 20_000, - intervalMs: 100, - graceMs: 50, - forwardOutput: false, - }); - let pid; - await until(() => { - pid = events(directory) - .flatMap((event) => event.processes ?? []) - .at(-1)?.pid; - return pid !== undefined; - }); - // Windows TerminateProcess cannot deliver POSIX signals. Exercise the SDK's - // termination handler separately from the real supervisor-kill control above. - process.emit("SIGTERM"); - assert.equal(await result, 143); - assert.ok(events(directory).some(({ event }) => event === "terminated")); - await until(() => !alive(pid), 3_000); -}); - -test("the command-line entry point accepts Windows and records its runtime", async (t) => { - const directory = outputDirectory(t); - const child = spawn( - process.execPath, - [join(import.meta.dirname, "test-watchdog.mjs"), "--version"], - { - cwd: directory, - env: { - ...process.env, - DOTNET_TEST_DEADLINE: String(Date.now() + 15_000), - }, - stdio: "ignore", - }, - ); - t.after(() => child.kill("SIGKILL")); - assert.equal(await new Promise((resolve) => child.once("exit", resolve)), 0); - const artifacts = join(directory, "TestResults"); - assert.equal(events(artifacts).at(-1).exitCode, 0); - assert.equal( - JSON.parse(readFileSync(join(artifacts, "watchdog-runtime.json"), "utf8")) - .platform, - "win32", - ); -}); - -test("Windows captures real managed waiting stacks without a memory dump or raw log", async (t) => { - const directory = outputDirectory(t); - const result = await runWithWatchdog({ - command: "dotnet", - args: [windowsSupervisorPath, "--stack-probe"], - directory, - timeoutMs: 10_000, - intervalMs: 500, - graceMs: 50, - forwardOutput: false, - }); - assert.equal(result, 124); - const captured = events(directory).filter( - ({ event }) => event === "managed-stack", - ); - assert.equal( - captured.length, - 1, - JSON.stringify( - events(directory).filter(({ event }) => event !== "processes"), - ), - ); - const stacks = readFileSync( - join(directory, `managed-stack-${captured[0].pid}.txt`), - "utf8", - ); - assert.match(stacks, /WindowsWatchdog\.WaitForStackProbe/); - assert.match(stacks, /^Thread \(0x[0-9a-f]+\):/im); - assert.ok(!stacks.includes("secret test payload")); - assert.ok( - !readFileSync(join(directory, "watchdog.jsonl"), "utf8").includes( - "secret test payload", - ), - ); - assert.ok( - !readdirSync(directory, { recursive: true }).some((file) => - /\.(dmp|nettrace|log)$/i.test(file), - ), - ); - await until(() => !alive(captured[0].pid), 3_000); -}); - -test("unsupported runtimes and failed stack collection are explicitly recorded", async (t) => { - const directory = outputDirectory(t); - const records = []; - await collectWindowsStacks( - [ - { pid: 2147483647, runtime: "core", role: "dotnet" }, - { pid: 2147483646, runtime: "framework", role: "testhost" }, - ], - directory, - (event) => records.push(event), - ); - assert.deepEqual(records[0], { - event: "managed-stack-unsupported", - pid: 2147483646, - runtime: "framework", - }); - assert.equal(records[1].event, "managed-stack-unavailable"); - assert.equal(records[1].pid, 2147483647); - assert.equal(records[1].code, 4294967295); - assert.ok( - !readdirSync(directory).some((file) => file.startsWith("managed-stack-")), - ); -}); diff --git a/dotnet/ci/test-watchdog.mjs b/dotnet/ci/test-watchdog.mjs deleted file mode 100644 index 9355f5c29d..0000000000 --- a/dotnet/ci/test-watchdog.mjs +++ /dev/null @@ -1,420 +0,0 @@ -import { execFile, spawn } from "node:child_process"; -import { - appendFileSync, - mkdirSync, - readFileSync, - writeFileSync, -} from "node:fs"; -import { basename, join, resolve } from "node:path"; -import { constants } from "node:os"; -import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; -import { collectWindowsStacks, startWindowsJob } from "./windows-watchdog.mjs"; - -const exec = promisify(execFile); - -// Persist only recognized markers, never arbitrary console output (Actions' -// secret masking does not apply to artifact files). -export function progressMarker(line) { - if (/\b_DownloadCopilotCli:/.test(line)) - return { phase: "runtime-provisioning" }; - if (/\bCoreCompile:/.test(line)) return { phase: "compilation" }; - if (/\b_CopyCopilotCliToOutput:/.test(line)) return { phase: "runtime-copy" }; - if (/^Test run for /.test(line)) { - const framework = /\.NETCoreApp,Version=v8\.0|net8\.0/.test(line) - ? "net8.0" - : /\.NETFramework,Version=v4\.7\.2|net472/.test(line) - ? "net472" - : undefined; - return { phase: "testhost-startup", ...(framework && { framework }) }; - } - if (/^Starting test execution,/.test(line)) - return { phase: "test-discovery" }; - if (/^\[xUnit\.net [\d:.]+\]\s+Starting:/.test(line)) { - return { phase: "tests-and-fixture-cleanup" }; - } - if (/^\[xUnit\.net [\d:.]+\]\s+Finished:/.test(line)) { - return { phase: "testhost-shutdown" }; - } - if (/^Test Run (Successful|Failed|Aborted)\./.test(line)) { - return { phase: "test-command-shutdown" }; - } - const test = - /^\s*(Passed|Failed|Skipped) (GitHub\.Copilot\.Test\.[A-Za-z0-9_.]+)(?=[(\s]|$)/.exec( - line, - ); - if (test) return { outcome: test[1], test: test[2] }; - return null; -} - -export function ownedProcesses(output, group) { - return output.split("\n").flatMap((line) => { - const match = - /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+([\d.]+)\s+(\d+)\s+([\d:-]+)\s+(.+?)\s*$/.exec( - line, - ); - if (!match || Number(match[3]) !== group) return []; - const name = basename(match[8]); - return [ - { - pid: Number(match[1]), - ppid: Number(match[2]), - group, - state: match[4], - cpu: Number(match[5]), - rssKiB: Number(match[6]), - elapsed: match[7], - role: /^(dotnet|testhost|copilot|copilot-runtime|node|tar|go|e2e\.test)$/.test( - name, - ) - ? name - : "other", - }, - ]; - }); -} - -export function sampleStacks(output) { - // sample reports native stacks, not memory or local variables. Omit its - // process/path headers and binary-image paths as well. - const graph = - /Call graph:\r?\n([\s\S]*?)(?:\r?\nTotal number in stack|\r?\nBinary Images:|$)/.exec( - output, - ); - let stacks = graph?.[1] ?? "No call graph available"; - for (const name of [ - "COPILOT_HMAC_KEY", - "GH_TOKEN", - "GITHUB_TOKEN", - "COPILOT_GITHUB_TOKEN", - ]) { - if (process.env[name]) - stacks = stacks.replaceAll(process.env[name], "[REDACTED]"); - } - return stacks; -} - -export async function collectProcesses(group) { - const { stdout } = await exec( - "ps", - ["-axo", "pid=,ppid=,pgid=,stat=,%cpu=,rss=,etime=,comm="], - { - timeout: 3_000, - killSignal: "SIGKILL", - maxBuffer: 4 * 1024 * 1024, - env: { ...process.env, LC_ALL: "C" }, - }, - ); - return ownedProcesses(stdout, group); -} - -export async function collectSamples(processes, directory, record) { - if (process.platform !== "darwin") return; - // Bound diagnostics too: eight one-second samples, each capped at five seconds. - for (const { pid } of processes.slice(0, 8)) { - try { - // Explicit stdout avoids sample's default on-disk report. Only the - // filtered call graph below is written to the artifact directory. - const { stdout } = await exec( - "/usr/bin/sample", - [String(pid), "1", "1", "-file", "/dev/stdout"], - { - timeout: 5_000, - killSignal: "SIGKILL", - maxBuffer: 4 * 1024 * 1024, - }, - ); - writeFileSync(join(directory, `sample-${pid}.txt`), sampleStacks(stdout)); - record({ event: "sample", pid }); - } catch { - record({ event: "sample-unavailable", pid }); - } - } -} - -export async function runWithWatchdog({ - command = "dotnet", - args, - directory, - timeoutMs, - intervalMs = 60_000, - graceMs = 5_000, - inspect, - sample = process.platform === "win32" ? collectWindowsStacks : collectSamples, - forwardOutput = true, - marker = progressMarker, - label = ".NET", -}) { - mkdirSync(directory, { recursive: true }); - const started = performance.now(); - let phase = "build-startup"; - let framework; - let finalized = false; - const record = (data) => - !finalized && - appendFileSync( - join(directory, "watchdog.jsonl"), - `${JSON.stringify({ at: new Date().toISOString(), elapsedMs: Math.round(performance.now() - started), phase, framework, ...data })}\n`, - ); - record({ event: "start", timeoutMs }); - if (timeoutMs <= 0) { - record({ event: "deadline-expired-before-start" }); - return 124; - } - - // POSIX uses an owned process group. Windows uses a supervisor in a nested - // kill-on-close Job Object; killing it also kills orphaned pipe holders. - const windows = - process.platform === "win32" - ? startWindowsJob(command, args, directory) - : undefined; - const child = - windows?.child ?? - spawn(command, args, { - detached: process.platform !== "win32", - stdio: ["ignore", "pipe", "pipe"], - }); - let result; - let stopping = false; - let commandExited = false; - let commandOutputClosed = false; - let backgroundCleaned = false; - let supervisorErrorRecorded = false; - const windowsStatus = () => { - const status = windows.status(); - commandOutputClosed ||= status.outputClosed === true; - if (status.exitCode !== undefined && !commandExited) { - commandExited = true; - result ??= status.exitCode; - record({ - event: "command-exit", - exitCode: status.exitCode, - pid: status.rootPid, - }); - phase = "output-drain"; - } - if (status.error && !supervisorErrorRecorded) { - supervisorErrorRecorded = true; - record({ - event: "windows-supervisor-error", - error: status.error, - code: status.errorCode, - }); - } - if (status.outputClosed && status.processCount > 0 && !backgroundCleaned) { - backgroundCleaned = true; - record({ - event: "owned-background-cleanup", - processCount: status.processCount, - }); - } - return status; - }; - inspect ??= windows - ? async () => windowsStatus().processes - : collectProcesses; - let finish; - const completed = new Promise((resolve) => { - finish = resolve; - }); - const signal = (name) => { - try { - if (process.platform === "win32") child.kill(name); - else process.kill(-child.pid, name); - } catch (error) { - if (error.code !== "ESRCH") - record({ event: "signal-failed", signal: name }); - } - }; - const snapshot = async () => { - try { - const processes = await inspect(child.pid); - record({ event: "processes", processes }); - return processes; - } catch (error) { - record({ - event: "process-snapshot-unavailable", - code: error.code ?? error.name, - }); - return []; - } - }; - const stop = async (reason, code) => { - if (stopping) return; - stopping = true; - if (windows) { - try { - windowsStatus(); - } catch (error) { - record({ - event: "windows-status-unavailable", - code: error.code ?? error.name, - }); - } - } - result = result || code; - record({ event: reason }); - if (forwardOutput) - console.error( - `[${label} watchdog] ${reason} during ${phase}; preserving diagnostics.`, - ); - clearInterval(heartbeat); - clearTimeout(deadline); - const processes = await snapshot(); - // Actions allows only a short signal grace period on cancellation. Keep - // the already-written timeline and snapshot; sample only our own deadline. - if (reason === "deadline-exceeded") { - try { - await sample(processes, directory, record); - } catch (error) { - record({ - event: "samples-unavailable", - code: error.code ?? error.name, - }); - } - } - signal("SIGTERM"); - await new Promise((resolve) => - setTimeout( - resolve, - reason === "deadline-exceeded" ? graceMs : Math.min(graceMs, 1_000), - ), - ); - signal("SIGKILL"); - child.stdout.destroy(); - child.stderr.destroy(); - child.unref(); - record({ event: "stopped", exitCode: result }); - finish(); - }; - const onInterrupt = () => { - void stop("interrupted", 130); - }; - const onTerminate = () => { - void stop("terminated", 143); - }; - process.on("SIGINT", onInterrupt); - process.on("SIGTERM", onTerminate); - - for (const [stream, destination] of [ - [child.stdout, process.stdout], - [child.stderr, process.stderr], - ]) { - if (forwardOutput) stream.pipe(destination); - let pending = ""; - stream.setEncoding("utf8"); - stream.on("data", (chunk) => { - pending += chunk; - let newline; - while ((newline = pending.indexOf("\n")) !== -1) { - const progress = marker(pending.slice(0, newline)); - if (progress) { - if (progress.phase) phase = progress.phase; - if (progress.framework) framework = progress.framework; - record({ event: "progress", ...progress }); - } - pending = pending.slice(newline + 1); - } - // Compiler invocations can be very long; none are diagnostic markers. - if (pending.length > 16_384) pending = ""; - }); - } - child.on("spawn", () => record({ event: "spawn", pid: child.pid })); - child.on("error", () => { - result ??= 127; - record({ event: "spawn-error" }); - }); - child.on("exit", (code, exitSignal) => { - if (windows) { - try { - windowsStatus(); - } catch (error) { - record({ - event: "windows-status-unavailable", - code: error.code ?? error.name, - }); - } - } - result = - result || code || (exitSignal ? 128 + constants.signals[exitSignal] : 0); - if (windows && (!commandExited || !commandOutputClosed) && !result) { - result = 127; - record({ event: "windows-command-status-missing" }); - } - record({ - event: windows ? "supervisor-exit" : "command-exit", - exitCode: code, - signal: exitSignal, - }); - phase = "output-drain"; - }); - child.on("close", () => { - record({ event: "output-closed" }); - if (!stopping) finish(); - }); - const heartbeat = setInterval(() => { - void snapshot(); - }, intervalMs); - const statusPoll = - windows && - setInterval(() => { - try { - windowsStatus(); - } catch (error) { - record({ - event: "windows-status-unavailable", - code: error.code ?? error.name, - }); - } - }, 250); - const deadline = setTimeout( - () => { - void stop("deadline-exceeded", 124); - }, - Math.max(0, timeoutMs - (performance.now() - started)), - ); - await completed; - clearInterval(heartbeat); - clearInterval(statusPoll); - clearTimeout(deadline); - process.off("SIGINT", onInterrupt); - process.off("SIGTERM", onTerminate); - record({ event: "finish", exitCode: result }); - finalized = true; - return result ?? 1; -} - -if ( - process.argv[1] && - resolve(process.argv[1]) === fileURLToPath(import.meta.url) -) { - const directory = resolve("TestResults"); - mkdirSync(directory, { recursive: true }); - const { copilotCliVersion } = JSON.parse( - readFileSync(new URL("../../nodejs/package.json", import.meta.url)), - ); - writeFileSync( - join(directory, "watchdog-runtime.json"), - JSON.stringify({ - copilotCliVersion, - platform: process.platform, - arch: process.arch, - node: process.version, - }), - ); - const deadline = Number(process.env.DOTNET_TEST_DEADLINE); - if ( - !Number.isFinite(deadline) || - deadline <= 0 || - !["darwin", "win32"].includes(process.platform) - ) { - throw new Error( - "The .NET CI watchdog requires macOS or Windows and DOTNET_TEST_DEADLINE", - ); - } - process.exitCode = await runWithWatchdog({ - args: process.argv.slice(2), - directory, - timeoutMs: Math.min(15 * 60_000, deadline - Date.now()), - }); -} diff --git a/dotnet/ci/test-watchdog.test.mjs b/dotnet/ci/test-watchdog.test.mjs deleted file mode 100644 index e5555b34ba..0000000000 --- a/dotnet/ci/test-watchdog.test.mjs +++ /dev/null @@ -1,409 +0,0 @@ -import assert from "node:assert/strict"; -import { spawn } from "node:child_process"; -import { mkdirSync, readFileSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { test } from "node:test"; -import { - collectProcesses, - collectSamples, - ownedProcesses, - progressMarker, - runWithWatchdog, - sampleStacks, -} from "./test-watchdog.mjs"; - -function outputDirectory(t) { - const directory = join( - import.meta.dirname, - `.watchdog-test-${process.pid}-${crypto.randomUUID()}`, - ); - mkdirSync(directory, { recursive: true }); - t.after(() => rmSync(directory, { recursive: true, force: true })); - return directory; -} - -function events(directory) { - return readFileSync(join(directory, "watchdog.jsonl"), "utf8") - .trim() - .split("\n") - .map(JSON.parse); -} - -function run(t, source, options = {}) { - const directory = outputDirectory(t); - return { - directory, - result: runWithWatchdog({ - command: process.execPath, - args: ["-e", source, "--", ...(options.extraArgs ?? [])], - directory, - timeoutMs: 5_000, - graceMs: 25, - inspect: async () => [], - sample: async () => {}, - forwardOutput: false, - ...options, - }), - }; -} - -test("recognizes build, provisioning, test and shutdown without recording raw output", () => { - for (const [line, phase] of [ - [" _DownloadCopilotCli:", "runtime-provisioning"], - [" CoreCompile:", "compilation"], - [" _CopyCopilotCliToOutput:", "runtime-copy"], - ["Test run for /private/path.dll", "testhost-startup"], - ["Starting test execution, please wait...", "test-discovery"], - [ - "[xUnit.net 00:00:00.10] Starting: GitHub.Copilot.SDK.Test", - "tests-and-fixture-cleanup", - ], - [ - "[xUnit.net 00:01:50.44] Finished: GitHub.Copilot.SDK.Test", - "testhost-shutdown", - ], - ["Test Run Successful.", "test-command-shutdown"], - ["Test Run Aborted.", "test-command-shutdown"], - ]) - assert.deepEqual(progressMarker(line), { phase }); - for (const [suffix, framework] of [ - ["(.NETCoreApp,Version=v8.0)", "net8.0"], - ["(net8.0)", "net8.0"], - ["(.NETFramework,Version=v4.7.2)", "net472"], - ]) { - assert.deepEqual(progressMarker(`Test run for private.dll ${suffix}`), { - phase: "testhost-startup", - framework, - }); - } - assert.deepEqual( - progressMarker( - ' Passed GitHub.Copilot.Test.E2E.Example.Test(token: "secret") [1 s]', - ), - { - outcome: "Passed", - test: "GitHub.Copilot.Test.E2E.Example.Test", - }, - ); - assert.equal(progressMarker("secret output"), null); -}); - -test("process snapshots contain only owned numeric metadata and known executable roles", () => { - assert.deepEqual( - ownedProcesses( - ` - 123 1 123 S 0.1 4096 01:02 /private/dotnet - 124 123 123 R+ 10.0 2048 00:01 /private/unrecognized-secret - 125 1 125 S 0.0 1024 01:00 /private/node -`, - 123, - ), - [ - { - pid: 123, - ppid: 1, - group: 123, - state: "S", - cpu: 0.1, - rssKiB: 4096, - elapsed: "01:02", - role: "dotnet", - }, - { - pid: 124, - ppid: 123, - group: 123, - state: "R+", - cpu: 10, - rssKiB: 2048, - elapsed: "00:01", - role: "other", - }, - ], - ); - assert.deepEqual( - ownedProcesses( - ` - 126 123 123 S 0.0 1024 00:01 /private/go - 127 126 123 S 0.0 1024 00:01 /private/e2e.test - 128 126 123 S 0.0 1024 00:01 /private/e2eXtest - 129 1 129 S 0.0 1024 00:01 /private/go -`, - 123, - ).map(({ pid, role }) => ({ pid, role })), - [ - { pid: 126, role: "go" }, - { pid: 127, role: "e2e.test" }, - { pid: 128, role: "other" }, - ], - ); -}); - -test("custom markers and labels replace .NET progress without persisting raw output", async (t) => { - const messages = []; - t.mock.method(console, "error", (message) => messages.push(message)); - const { directory, result } = run( - t, - ` - console.log("=== RUN TestFixture"); - console.log("CoreCompile:"); - setInterval(() => {}, 1000); - `, - { - timeoutMs: process.platform === "win32" ? 5_000 : 1_000, - marker: (line) => - line === "=== RUN TestFixture" ? { phase: "go-tests" } : null, - label: "Go", - forwardOutput: true, - }, - ); - assert.equal(await result, 124); - assert.deepEqual( - events(directory) - .filter(({ event }) => event === "progress") - .map(({ phase }) => phase), - ["go-tests"], - ); - assert.deepEqual(messages, [ - "[Go watchdog] deadline-exceeded during go-tests; preserving diagnostics.", - ]); - assert.ok( - !readFileSync(join(directory, "watchdog.jsonl"), "utf8").includes( - "TestFixture", - ), - ); -}); - -test("custom markers parse short-lived failures with output forwarding disabled", async (t) => { - for (const exit of ["process.exitCode = 37", "process.exit(37)"]) { - const { directory, result } = run( - t, - `console.log("go-package-finished"); ${exit};`, - { - marker: (line) => - line === "go-package-finished" - ? { phase: "go-package-complete" } - : null, - label: "Go", - forwardOutput: false, - }, - ); - assert.equal(await result, 37); - assert.ok( - events(directory).some( - ({ event, phase }) => - event === "progress" && phase === "go-package-complete", - ), - JSON.stringify(events(directory)), - ); - } -}); - -test("samples omit headers and image paths and redact credentials", (t) => { - const previous = process.env.COPILOT_HMAC_KEY; - process.env.COPILOT_HMAC_KEY = "watchdog-test-secret"; - t.after(() => { - if (previous === undefined) delete process.env.COPILOT_HMAC_KEY; - else process.env.COPILOT_HMAC_KEY = previous; - }); - assert.equal( - sampleStacks( - "Path: private\nCall graph:\n wait watchdog-test-secret\nBinary Images:\nprivate", - ), - " wait [REDACTED]", - ); -}); - -test("forwards arguments, preserves success and failure, and records split output markers", async (t) => { - const previous = process.env.DOTNET_WATCHDOG_TEST_ENV; - process.env.DOTNET_WATCHDOG_TEST_ENV = "inherited-value"; - t.after(() => { - if (previous === undefined) delete process.env.DOTNET_WATCHDOG_TEST_ENV; - else process.env.DOTNET_WATCHDOG_TEST_ENV = previous; - }); - for (const code of [0, 23]) { - const { directory, result } = run( - t, - ` - const assert = require("node:assert/strict"); - assert.deepEqual(process.argv.slice(1), ["--filter", "(A|B)&C", "--blame-hang"]); - assert.equal(process.env.DOTNET_WATCHDOG_TEST_ENV, "inherited-value"); - process.stdout.write(" _DownloadCopilot"); - setTimeout(() => { - console.log("Cli:"); - console.log("secret output"); - console.log("Test Run Successful."); - process.exitCode = ${code}; - }, 20); - `, - { extraArgs: ["--filter", "(A|B)&C", "--blame-hang"] }, - ); - assert.equal(await result, code); - assert.ok( - events(directory).some((event) => event.phase === "runtime-provisioning"), - ); - assert.ok( - !readFileSync(join(directory, "watchdog.jsonl"), "utf8").includes( - "secret output", - ), - ); - } -}); - -test("expired job budget never starts a command", async (t) => { - const { directory, result } = run(t, "process.exit(99)", { timeoutMs: 0 }); - assert.equal(await result, 124); - assert.ok(!events(directory).some((event) => event.event === "spawn")); -}); - -test("a hung command is sampled before termination and fails within the inner budget", async (t) => { - let sampled = false; - const { directory, result } = run( - t, - ` - console.log("Test run for test.dll"); - setInterval(() => {}, 1000); - `, - { - timeoutMs: process.platform === "win32" ? 5_000 : 1_000, - sample: async () => { - sampled = true; - throw new Error("unavailable"); - }, - inspect: async () => { - throw new Error("unavailable"); - }, - }, - ); - assert.equal(await result, 124); - assert.ok(sampled); - assert.equal( - events(directory).find((event) => event.event === "deadline-exceeded") - .phase, - "testhost-startup", - ); - assert.ok( - events(directory).some( - (event) => event.event === "process-snapshot-unavailable", - ), - ); - assert.ok( - events(directory).some((event) => event.event === "samples-unavailable"), - ); -}); - -test("missing executables preserve spawn failure", async (t) => { - const { result } = run(t, "", { - command: "nonexistent-dotnet-watchdog-test-command", - }); - assert.equal(await result, 127); -}); - -test( - "signal exits preserve the shell exit status", - { - skip: process.platform === "win32", - }, - async (t) => { - const { result } = run(t, "process.kill(process.pid, 'SIGTERM')"); - assert.equal(await result, 143); - }, -); - -test( - "owned descendants retaining pipes cannot hide the first command failure", - { - skip: process.platform === "win32", - }, - async (t) => { - for (const code of [0, 37]) { - let descendants = []; - const { directory, result } = run( - t, - ` - const { spawn } = require("node:child_process"); - spawn(process.execPath, ["-e", "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)"], - { stdio: ["ignore", 1, 2] }).unref(); - process.exit(${code}); - `, - { - timeoutMs: 1_000, - inspect: collectProcesses, - sample: async (processes) => { - descendants = processes; - }, - }, - ); - assert.equal(await result, code || 124); - assert.equal( - events(directory).find((event) => event.event === "deadline-exceeded") - .phase, - "output-drain", - ); - assert.ok(descendants.length > 0); - // kill(0) can still see a zombie briefly; ps must not see a live descendant. - const remaining = await collectProcesses(descendants[0].group); - assert.ok(remaining.every((process) => process.state.startsWith("Z"))); - } - }, -); - -test( - "termination requests retain diagnostics and a failing status", - { - skip: process.platform === "win32", - }, - async (t) => { - const directory = outputDirectory(t); - const script = ` - import { runWithWatchdog } from ${JSON.stringify(new URL("./test-watchdog.mjs", import.meta.url).href)}; - process.exitCode = await runWithWatchdog({ - command: process.execPath, - args: ["-e", "process.stdout.write('ready\\\\n'); setInterval(() => {}, 1000)"], - directory: ${JSON.stringify(directory)}, timeoutMs: 5000, graceMs: 25, - inspect: async () => [], sample: async () => {}, - }); - `; - const child = spawn( - process.execPath, - ["--input-type=module", "-e", script], - { stdio: ["ignore", "pipe", "inherit"] }, - ); - t.after(() => child.kill("SIGKILL")); - const exited = new Promise((resolve) => child.on("exit", resolve)); - await new Promise((resolve) => child.stdout.once("data", resolve)); - child.kill("SIGTERM"); - assert.equal(await exited, 143); - assert.ok(events(directory).some((event) => event.event === "terminated")); - }, -); - -test( - "macOS sample captures an owned process call graph without raw files", - { - skip: process.platform !== "darwin", - }, - async (t) => { - const directory = outputDirectory(t); - const child = spawn( - process.execPath, - ["-e", "process.stdout.write('ready'); setInterval(() => {}, 1000)"], - { - stdio: ["ignore", "pipe", "inherit"], - }, - ); - t.after(() => child.kill("SIGKILL")); - await new Promise((resolve) => child.stdout.once("data", resolve)); - const records = []; - await collectSamples([{ pid: child.pid }], directory, (record) => - records.push(record), - ); - assert.equal(records[0].event, "sample"); - const stacks = readFileSync( - join(directory, `sample-${child.pid}.txt`), - "utf8", - ); - assert.notEqual(stacks, "No call graph available"); - assert.ok(!stacks.includes("Binary Images:")); - }, -); diff --git a/dotnet/ci/windows-watchdog.mjs b/dotnet/ci/windows-watchdog.mjs deleted file mode 100644 index 45c62ef861..0000000000 --- a/dotnet/ci/windows-watchdog.mjs +++ /dev/null @@ -1,203 +0,0 @@ -import { spawn } from "node:child_process"; -import { - closeSync, - mkdirSync, - openSync, - readSync, - writeFileSync, -} from "node:fs"; -import { join } from "node:path"; -import { fileURLToPath } from "node:url"; - -export const windowsSupervisorPath = fileURLToPath( - new URL("./bin/Release/net10.0/WindowsWatchdog.dll", import.meta.url), -); - -export function startWindowsJob(command, args, directory, cwd) { - const statusPath = join(directory, "windows-job.jsonl"); - writeFileSync(statusPath, ""); - const child = spawn("dotnet", [windowsSupervisorPath, statusPath], { - stdio: ["pipe", "pipe", "pipe"], - cwd, - windowsHide: true, - }); - // Arguments are transmitted in memory, not persisted or shell-interpolated. - child.stdin.on("error", (error) => { - if (error.code !== "EPIPE" && error.code !== "EOF") - child.emit("error", error); - }); - child.stdin.end(`${JSON.stringify({ command, args })}\n`); - let offset = 0; - let pending = ""; - let last = { processes: [] }; - return { - child, - status() { - let fd; - try { - fd = openSync(statusPath, "r"); - } catch (error) { - if (error.code === "ENOENT") return last; - throw error; - } - try { - const buffer = Buffer.alloc(1024 * 1024); - const count = readSync(fd, buffer, 0, buffer.length, offset); - offset += count; - pending += buffer.toString("utf8", 0, count); - const end = pending.lastIndexOf("\n"); - if (end !== -1) { - const start = pending.lastIndexOf("\n", end - 1) + 1; - last = JSON.parse(pending.slice(start, end)); - pending = pending.slice(end + 1); - } - if (pending.length > buffer.length) - throw new Error("Windows status exceeded its size limit"); - return last; - } finally { - closeSync(fd); - } - }, - }; -} - -async function stackReport(pid, directory) { - const started = performance.now(); - const statusDirectory = join(directory, `stack-collector-${pid}`); - mkdirSync(statusDirectory, { recursive: true }); - const { child, status } = startWindowsJob( - "dotnet", - [ - "tool", - "run", - "dotnet-stack", - "--", - "report", - "--process-id", - String(pid), - ], - statusDirectory, - import.meta.dirname, - ); - // The tool launcher can also retain a child's pipes. Use the same owned job - // as the test command instead of execFile's parent-only Windows timeout kill. - return await new Promise((resolve, reject) => { - let output = ""; - let size = 0; - let failure; - let forceClose; - let settled = false; - const finish = (error) => { - if (settled) return; - settled = true; - clearTimeout(deadline); - clearTimeout(forceClose); - if (error) reject(error); - else resolve(output); - }; - const stop = (code) => { - failure ??= Object.assign(new Error(code), { code }); - child.kill("SIGKILL"); - forceClose ??= setTimeout(() => { - child.stdout.destroy(); - child.stderr.destroy(); - child.unref(); - finish(failure); - }, 1_000); - }; - const deadline = setTimeout( - () => stop("STACK_TIMEOUT"), - Math.max(0, 10_000 - (performance.now() - started)), - ); - for (const stream of [child.stdout, child.stderr]) { - stream.setEncoding("utf8"); - stream.on("data", (chunk) => { - size += Buffer.byteLength(chunk); - if (size > 4 * 1024 * 1024) stop("STACK_OUTPUT_LIMIT"); - else if (stream === child.stdout) output += chunk; - }); - } - child.on("error", (error) => { - failure ??= error; - }); - child.on("close", (code) => { - try { - if (failure) finish(failure); - else if (code !== 0 || status().exitCode !== 0) - finish(Object.assign(new Error("Stack collector failed"), { code })); - else finish(); - } catch (error) { - finish(error); - } - }); - }); -} - -export function managedStacks(output) { - // dotnet-stack prints type/method signatures, never values. Retain only - // thread IDs, native boundaries and module!method names, excluding headers, - // paths and even parameter signatures. Do not persist the raw EventPipe data. - const lines = output.split(/\r?\n/).flatMap((line) => { - if (/^Thread \(0x[0-9a-f]+\):$/i.test(line)) return [line]; - if (/^\s+\[Native Frames\]$/.test(line)) return [line.trim()]; - const frame = - /^\s+([A-Za-z0-9_.$+`<>,[\]:-]+![A-Za-z0-9_.$+`<>,[\]:-]+)(?:\(|$)/.exec( - line, - ); - return frame ? [` ${frame[1]}`] : []; - }); - let result = lines.slice(0, 4096).join("\n").slice(0, 65_536); - if (lines.length > 4096 || lines.join("\n").length > 65_536) - result = `${result.slice(0, 65_500)}\n[Stack output truncated]`; - for (const name of [ - "COPILOT_HMAC_KEY", - "GH_TOKEN", - "GITHUB_TOKEN", - "COPILOT_GITHUB_TOKEN", - ]) { - if (process.env[name]) - result = result.replaceAll(process.env[name], "[REDACTED]"); - } - return result; -} - -export async function collectWindowsStacks(processes, directory, record) { - const managed = processes.filter(({ runtime }) => runtime === "core"); - for (const { pid, runtime } of processes) { - if (runtime === "framework") - record({ event: "managed-stack-unsupported", pid, runtime }); - } - // Testhosts first, then the build/test orchestrators. Four ten-second caps - // leave most of the four-minute job reserve for cleanup and artifact upload. - managed.sort( - (a, b) => - Number(b.role.startsWith("testhost")) - - Number(a.role.startsWith("testhost")), - ); - if (managed.length > 4) - record({ - event: "managed-stack-target-limit", - available: managed.length, - limit: 4, - }); - for (const { pid } of managed.slice(0, 4)) { - try { - const stdout = await stackReport(pid, directory); - const stacks = managedStacks(stdout); - if (!stacks) { - record({ event: "managed-stack-empty", pid }); - continue; - } - writeFileSync(join(directory, `managed-stack-${pid}.txt`), stacks); - record({ event: "managed-stack", pid }); - } catch (error) { - record({ - event: "managed-stack-unavailable", - pid, - code: error.code, - signal: error.signal, - killed: error.killed, - }); - } - } -} diff --git a/go/.gitignore b/go/.gitignore index 8b305cce00..266339f383 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -22,6 +22,3 @@ go.work # env file .env - -# CI diagnostic artifacts -TestResults/ diff --git a/go/README.md b/go/README.md index b4e1cc0cbc..2eb2c720fc 100644 --- a/go/README.md +++ b/go/README.md @@ -1075,19 +1075,6 @@ cd go ./test.sh ``` -The macOS in-process CI job wraps this same command with the shared test -watchdog. It captures process metadata and native call stacks before the -20-minute job deadline, then fails and terminates its owned process group if the -command or its output pipes remain stuck. The Go test selection, race detector, -and per-package timeout are unchanged. - -The E2E test process also records startup/completion and all goroutine stacks -one minute before the watchdog deadline, independently of `go test`'s buffered -package output. CI uploads these files from `go/TestResults/`. They contain call -frames, not heap dumps, RPC payloads, or arbitrary test logs. Diagnostics are -opt-in through `GO_TEST_DIAGNOSTIC_DIRECTORY` and the epoch-millisecond -`GO_TEST_DIAGNOSTIC_CAPTURE_AT`; ordinary local tests are unaffected. - ## License MIT diff --git a/go/ci/test-watchdog.mjs b/go/ci/test-watchdog.mjs deleted file mode 100644 index 65308746d2..0000000000 --- a/go/ci/test-watchdog.mjs +++ /dev/null @@ -1,69 +0,0 @@ -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { - collectSamples, - runWithWatchdog, -} from "../../dotnet/ci/test-watchdog.mjs"; - -export function goProgressMarker(line) { - if (line === "=== Running Go SDK E2E Tests ===") - return { phase: "go-build-and-test" }; - // Go buffers a package's verbose output until it exits. In-process TestMain - // writes its own phase and goroutine artifacts without relying on this pipe. - const result = - /^(ok|FAIL|\?)\s+(github\.com\/github\/copilot-sdk\/go(?:\/[A-Za-z0-9_.-]+)*)(?=\s|$)/.exec( - line, - ); - return result ? { outcome: result[1], package: result[2] } : null; -} - -export function diagnosticBudget(deadline, now = Date.now()) { - if (!Number.isFinite(deadline) || deadline <= 0) - throw new Error("The Go CI watchdog requires GO_TEST_DEADLINE"); - const timeoutMs = Math.min(15 * 60_000, deadline - now); - return { timeoutMs, captureAt: now + timeoutMs - 60_000 }; -} - -export function prioritizeGoProcesses(processes) { - const rank = ({ role }) => (role === "e2e.test" ? 0 : role === "go" ? 1 : 2); - return [...processes].sort((a, b) => rank(a) - rank(b)); -} - -if ( - process.argv[1] && - resolve(process.argv[1]) === fileURLToPath(import.meta.url) -) { - if (process.platform !== "darwin") - throw new Error("The Go CI watchdog requires macOS"); - const directory = resolve("TestResults"); - const { timeoutMs, captureAt } = diagnosticBudget( - Number(process.env.GO_TEST_DEADLINE), - ); - mkdirSync(directory, { recursive: true }); - const { copilotCliVersion } = JSON.parse( - readFileSync(new URL("../../nodejs/package.json", import.meta.url)), - ); - writeFileSync( - join(directory, "watchdog-runtime.json"), - JSON.stringify({ - copilotCliVersion, - platform: process.platform, - arch: process.arch, - node: process.version, - captureAt, - }), - ); - process.env.GO_TEST_DIAGNOSTIC_DIRECTORY = directory; - process.env.GO_TEST_DIAGNOSTIC_CAPTURE_AT = String(captureAt); - process.exitCode = await runWithWatchdog({ - command: "/bin/bash", - args: ["test.sh"], - directory, - timeoutMs, - marker: goProgressMarker, - label: "Go", - sample: (processes, directory, record) => - collectSamples(prioritizeGoProcesses(processes), directory, record), - }); -} diff --git a/go/ci/test-watchdog.test.mjs b/go/ci/test-watchdog.test.mjs deleted file mode 100644 index aa974cb40d..0000000000 --- a/go/ci/test-watchdog.test.mjs +++ /dev/null @@ -1,87 +0,0 @@ -import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import test from "node:test"; -import { runWithWatchdog } from "../../dotnet/ci/test-watchdog.mjs"; -import { - diagnosticBudget, - goProgressMarker, - prioritizeGoProcesses, -} from "./test-watchdog.mjs"; - -test("only package progress is retained, without arbitrary output or arguments", () => { - assert.deepEqual(goProgressMarker("=== Running Go SDK E2E Tests ==="), { - phase: "go-build-and-test", - }); - for (const outcome of ["ok", "FAIL", "?"]) { - assert.deepEqual( - goProgressMarker( - `${outcome}\tgithub.com/github/copilot-sdk/go/internal/e2e\tsecret-value`, - ), - { outcome, package: "github.com/github/copilot-sdk/go/internal/e2e" }, - ); - } - for (const line of [ - "secret-value", - "=== RUN TestWithSecret/secret-value", - "ok github.com/github/copilot-sdk/go/internal/e2e?secret-value", - ]) { - assert.equal(goProgressMarker(line), null); - } -}); - -test("capture precedes the earlier of the command and absolute job deadlines", () => { - const now = 2_000_000; - assert.deepEqual(diagnosticBudget(now + 18 * 60_000, now), { - timeoutMs: 15 * 60_000, - captureAt: now + 14 * 60_000, - }); - - assert.deepEqual(diagnosticBudget(now + 5 * 60_000, now), { - timeoutMs: 5 * 60_000, - captureAt: now + 4 * 60_000, - }); - assert.equal(diagnosticBudget(now - 1, now).timeoutMs, -1); - for (const deadline of [NaN, Infinity, 0, -1]) - assert.throws(() => diagnosticBudget(deadline, now), /GO_TEST_DEADLINE/); -}); - -test("the shared watchdog uses Go markers and preserves the command failure", async () => { - const directory = mkdtempSync(join(tmpdir(), "go-watchdog-")); - try { - const code = await runWithWatchdog({ - command: process.execPath, - args: [ - "-e", - "console.log('=== Running Go SDK E2E Tests ==='); console.log('secret-value'); process.exitCode = 37;", - ], - directory, - timeoutMs: 10_000, - forwardOutput: false, - inspect: async () => [], - marker: goProgressMarker, - label: "Go", - }); - - assert.equal(code, 37); - const timeline = readFileSync(join(directory, "watchdog.jsonl"), "utf8"); - assert.match(timeline, /"phase":"go-build-and-test"/); - assert.doesNotMatch(timeline, /secret-value/); - } finally { - rmSync(directory, { recursive: true, force: true }); - } -}); - -test("native sampling prioritizes the in-process E2E host over launcher processes", () => { - const processes = [ - ...Array.from({ length: 8 }, (_, pid) => ({ pid, role: "other" })), - { pid: 100, role: "go" }, - { pid: 101, role: "e2e.test" }, - ]; - assert.deepEqual(prioritizeGoProcesses(processes).slice(0, 2), [ - { pid: 101, role: "e2e.test" }, - { pid: 100, role: "go" }, - ]); - assert.equal(processes[0].pid, 0); -}); diff --git a/go/internal/e2e/main_test.go b/go/internal/e2e/main_test.go deleted file mode 100644 index 45702355de..0000000000 --- a/go/internal/e2e/main_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package e2e - -import ( - "os" - "testing" - - "github.com/github/copilot-sdk/go/internal/testdiagnostics" -) - -func TestMain(m *testing.M) { - os.Exit(testdiagnostics.Run(m.Run)) -} diff --git a/go/internal/testdiagnostics/diagnostics.go b/go/internal/testdiagnostics/diagnostics.go deleted file mode 100644 index 68202af4d4..0000000000 --- a/go/internal/testdiagnostics/diagnostics.go +++ /dev/null @@ -1,100 +0,0 @@ -// Package testdiagnostics captures test-process state before the CI job deadline. -package testdiagnostics - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "time" -) - -// Run preserves the test runner's result and optionally records pre-timeout -// goroutine stacks. The external CI watchdog owns termination and native samples. -func Run(run func() int) int { - directory := os.Getenv("GO_TEST_DIAGNOSTIC_DIRECTORY") - if directory == "" { - return run() - } - captureAt, err := strconv.ParseInt(os.Getenv("GO_TEST_DIAGNOSTIC_CAPTURE_AT"), 10, 64) - if err != nil || captureAt <= 0 { - fmt.Fprintln(os.Stderr, "Go diagnostics require a positive GO_TEST_DIAGNOSTIC_CAPTURE_AT timestamp") - return 1 - } - directory, err = filepath.Abs(directory) - if err != nil { - fmt.Fprintln(os.Stderr, "Go diagnostic directory:", err) - return 1 - } - return runMonitored(run, directory, time.UnixMilli(captureAt), func() error { - return captureStacks(directory) - }) -} - -func runMonitored(run func() int, directory string, captureAt time.Time, capture func() error) int { - if err := os.MkdirAll(directory, 0700); err != nil { - fmt.Fprintln(os.Stderr, "Creating Go diagnostic directory:", err) - return 1 - } - record := func(phase string, code *int) error { - data, err := json.Marshal(struct { - Phase string `json:"phase"` - At time.Time `json:"at"` - PID int `json:"pid"` - Go string `json:"go"` - ExitCode *int `json:"exitCode,omitempty"` - }{phase, time.Now().UTC(), os.Getpid(), runtime.Version(), code}) - if err != nil { - return err - } - return os.WriteFile(filepath.Join(directory, "go-test-process.json"), data, 0600) - } - if err := record("tests-started", nil); err != nil { - fmt.Fprintln(os.Stderr, "Recording Go test startup:", err) - return 1 - } - captured := make(chan error, 1) - timer := time.AfterFunc(time.Until(captureAt), func() { - err := capture() - if err != nil { - fmt.Fprintln(os.Stderr, "Capturing Go goroutines:", err) - } - captured <- err - }) - code := run() - if !timer.Stop() { - if err := <-captured; err != nil && code == 0 { - code = 1 - } - } - if err := record("tests-finished", &code); err != nil { - fmt.Fprintln(os.Stderr, "Recording Go test completion:", err) - if code == 0 { - code = 1 - } - } - return code -} - -func captureStacks(directory string) error { - // runtime.Stack reports call frames and numeric arguments, not heap contents, - // RPC payloads, environment variables, or arbitrary test output. - for size := 64 * 1024; size <= 16*1024*1024; size *= 2 { - buffer := make([]byte, size) - n := runtime.Stack(buffer, true) - if n == len(buffer) { - continue - } - stacks := string(buffer[:n]) - for _, name := range []string{"COPILOT_HMAC_KEY", "GH_TOKEN", "GITHUB_TOKEN", "COPILOT_GITHUB_TOKEN"} { - if value := os.Getenv(name); value != "" { - stacks = strings.ReplaceAll(stacks, value, "[REDACTED]") - } - } - return os.WriteFile(filepath.Join(directory, "go-goroutines.txt"), []byte(stacks), 0600) - } - return fmt.Errorf("goroutine stacks exceeded the 16 MiB diagnostic limit") -} diff --git a/go/internal/testdiagnostics/diagnostics_test.go b/go/internal/testdiagnostics/diagnostics_test.go deleted file mode 100644 index b3cea8a05a..0000000000 --- a/go/internal/testdiagnostics/diagnostics_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package testdiagnostics - -import ( - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -func TestDisabledDiagnosticsPreserveResult(t *testing.T) { - t.Setenv("GO_TEST_DIAGNOSTIC_DIRECTORY", "") - if code := Run(func() int { return 37 }); code != 37 { - t.Fatalf("Run returned %d, want 37", code) - } -} - -func TestInvalidConfigurationFailsBeforeTests(t *testing.T) { - for _, timestamp := range []string{"", "invalid", "0", "-1"} { - t.Run(timestamp, func(t *testing.T) { - t.Setenv("GO_TEST_DIAGNOSTIC_DIRECTORY", t.TempDir()) - t.Setenv("GO_TEST_DIAGNOSTIC_CAPTURE_AT", timestamp) - called := false - code := Run(func() int { called = true; return 0 }) - if code != 1 || called { - t.Fatalf("code=%d, called=%v; invalid configuration must fail", code, called) - } - }) - } -} - -func TestConfiguredDiagnosticsPreserveResult(t *testing.T) { - directory := t.TempDir() - t.Setenv("GO_TEST_DIAGNOSTIC_DIRECTORY", directory) - t.Setenv("GO_TEST_DIAGNOSTIC_CAPTURE_AT", "4102444800000") - if code := Run(func() int { return 37 }); code != 37 { - t.Fatalf("configured Run returned %d, want 37", code) - } - if _, err := os.Stat(filepath.Join(directory, "go-test-process.json")); err != nil { - t.Fatalf("configured Run did not write process state: %v", err) - } -} - -func TestCompletionPreservesExitAndCancelsCapture(t *testing.T) { - for _, expected := range []int{0, 37} { - directory := t.TempDir() - code := runMonitored(func() int { return expected }, directory, time.Now().Add(time.Hour), func() error { - t.Error("capture ran after normal completion") - return nil - }) - data, err := os.ReadFile(filepath.Join(directory, "go-test-process.json")) - if err != nil { - t.Fatal(err) - } - var state struct { - Phase string `json:"phase"` - ExitCode int `json:"exitCode"` - } - if err := json.Unmarshal(data, &state); err != nil { - t.Fatal(err) - } - if code != expected || state.ExitCode != expected || state.Phase != "tests-finished" { - t.Fatalf("code=%d, state=%+v, want exit %d", code, state, expected) - } - if _, err := os.Stat(filepath.Join(directory, "go-goroutines.txt")); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("unexpected stack artifact: %v", err) - } - } -} - -func TestCaptureRunsBeforeTestCleanup(t *testing.T) { - directory := t.TempDir() - captured := make(chan struct{}) - code := runMonitored(func() int { - <-captured - return 37 - }, directory, time.Now(), func() error { - defer close(captured) - if err := captureStacks(directory); err != nil { - return err - } - data, err := os.ReadFile(filepath.Join(directory, "go-test-process.json")) - if err == nil && !strings.Contains(string(data), `"phase":"tests-started"`) { - t.Errorf("capture did not precede cleanup: %s", data) - } - return err - }) - if code != 37 { - t.Fatalf("capture changed first failure to %d", code) - } - data, err := os.ReadFile(filepath.Join(directory, "go-goroutines.txt")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(data), ".TestCaptureRunsBeforeTestCleanup.func1(") { - t.Fatalf("timed capture did not include the blocked test runner: %s", data) - } -} - -func TestCaptureFailureIsNotSuccess(t *testing.T) { - for _, expected := range []int{0, 37} { - captured := make(chan struct{}) - code := runMonitored(func() int { <-captured; return expected }, t.TempDir(), time.Now(), func() error { - defer close(captured) - return errors.New("controlled capture failure") - }) - want := expected - if want == 0 { - want = 1 - } - if code != want { - t.Fatalf("code=%d, want %d", code, want) - } - } -} - -func diagnosticBlockedRoutine(ready chan<- struct{}, release <-chan struct{}, done chan<- struct{}) { - close(ready) - <-release - close(done) -} - -func TestCaptureIncludesBlockedGoroutine(t *testing.T) { - directory := t.TempDir() - ready, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) - go diagnosticBlockedRoutine(ready, release, done) - <-ready - defer func() { close(release); <-done }() - if err := captureStacks(directory); err != nil { - t.Fatal(err) - } - data, err := os.ReadFile(filepath.Join(directory, "go-goroutines.txt")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(data), ".diagnosticBlockedRoutine(") || !strings.Contains(string(data), "[chan receive]") { - t.Fatalf("missing blocked goroutine in stack capture: %s", data) - } -} diff --git a/python/README.md b/python/README.md index 1dc1a11627..3b7d937c44 100644 --- a/python/README.md +++ b/python/README.md @@ -1238,23 +1238,8 @@ Signal-based E2E failures from `pytest-timeout` include an **Async timeout diagn section with suspended coroutine await chains, pending JSON-RPC request IDs and methods, session/transport state, and Python thread stacks. The same report is saved under `python/.pytest-diagnostics/`. macOS in-process timeouts also capture a -one-second native thread sample there. CI uploads these files as -`python-timeout--` artifacts. RPC payloads and arbitrary frame locals +one-second native thread sample there. RPC payloads and arbitrary frame locals are not included. After recording the timeout, the harness cancels only the abandoned test coroutine so it does not retain locks needed by later fixture cleanup. The original timeout failure is retained; this does not abort native runtime work or repair a missing RPC response. - -To investigate an intermittent timeout, manually dispatch the **Python SDK Tests** -workflow with `reproduce_timeout=true`. This selects only macOS/inprocess and runs -up to two full suites with the usual xdist ordering. Set -`reproduction_scope=session-config` to instead run the original session-config -module up to ten times with the same xdist options. Both modes stop with a failed -job on the first nonzero exit and retain the 20-minute budget. The conservative -counts leave time for a failing test and its cleanup. - -Diagnostic artifacts include `reproduction-summary.txt`, recording started and -completed invocations; `reproduction_complete=true` appears only after every -planned invocation passes. Artifacts are uploaded after diagnostic cancellation -when the runner can still execute cleanup. An interrupted run is not successful -reproduction. Ordinary manual dispatches and reusable PR checks are unchanged.