From 9e846f8fecbde8d924ea196357f09057860df65a Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:33:00 +0530 Subject: [PATCH 1/3] FIX: fail the SQL Server readiness check when the container never starts the macOS and code-coverage legs poll SQL Server in a loop whose exit status is the final sleep, so the step reported success even when every probe failed. a dead container then surfaced as 30+ minutes of pytest errors instead of an infra failure. both loops now track readiness in a flag, re-probe once after the loop, and exit 1 with container logs when SQL never came up. the six linux legs end their step with CREATE DATABASE, whose exit status already fails them, so they are left alone. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 42 +++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index f0323fb0..59ab1922 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -516,15 +516,32 @@ jobs: -d $(sqlServerImage) # Starting SQL Server container… + sql_ready=false for i in {1..30}; do - docker exec sqlserver \ + if docker exec sqlserver \ /opt/mssql-tools18/bin/sqlcmd \ -S localhost \ -U SA \ -P "$DB_PASSWORD" \ - -C -Q "SELECT 1" && break + -C -Q "SELECT 1"; then + sql_ready=true + break + fi sleep 2 done + if [ "$sql_ready" != true ]; then + # one last probe, in case it came up during the final sleep + docker exec sqlserver \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$DB_PASSWORD" \ + -C -Q "SELECT 1" || { + echo "SQL Server did not become ready after 30 attempts" + docker logs --tail 200 sqlserver || true + exit 1 + } + fi displayName: 'Pull & start SQL Server (Docker)' env: DB_PASSWORD: $(DB_PASSWORD) @@ -2454,15 +2471,32 @@ jobs: -d mcr.microsoft.com/mssql/server:2022-latest # Wait until SQL Server is ready + sql_ready=false for i in {1..30}; do - docker exec sqlserver \ + if docker exec sqlserver \ /opt/mssql-tools18/bin/sqlcmd \ -S localhost \ -U SA \ -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" && break + -C -Q "SELECT 1"; then + sql_ready=true + break + fi sleep 2 done + if [ "$sql_ready" != true ]; then + # one last probe, in case it came up during the final sleep + docker exec sqlserver \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1" || { + echo "SQL Server did not become ready after 30 attempts" + docker logs --tail 200 sqlserver || true + exit 1 + } + fi displayName: 'Start SQL Server container' env: DB_PASSWORD: $(DB_PASSWORD) From 455c057c99817fad8296b135819cb2fc5690bd9f Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:01:56 +0530 Subject: [PATCH 2/3] FIX: surface container logs and re-probe once when macOS SQL setup fails the macOS setup_sql readiness loop reports that SQL never came up but not why, because sqlcmd only tells you the client could not connect. when the container itself exited (password policy, EULA, OOM, port clash) the reason is only in the container's own log, so dump docker logs on the failure path. also probe once more after the loop: the loop sleeps after its final attempt, so SQL can become reachable in that window and would otherwise be reported as a hard failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 37 ++++++++++++------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5c4d0c6a..846ee50d 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -559,7 +559,23 @@ jobs: fi sleep 2 done + + # One last probe: the loop sleeps after its final attempt, so SQL can + # become reachable in that window and would otherwise be missed. + if docker exec sqlserver \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$DB_PASSWORD" \ + -C -Q "SELECT 1"; then + echo "[sql] SQL Server is ready." + return 0 + fi + + # sqlcmd only reports that the client could not connect. When the + # container itself exited (bad password policy, EULA, OOM, port clash) + # the reason is in the container's own log, so surface it here. echo "[sql] SQL Server did not become ready in time." >&2 + echo "[sql] ---- docker logs sqlserver (last 200 lines) ----" >&2 + docker logs --tail 200 sqlserver >&2 || true return 1 } @@ -2510,32 +2526,15 @@ jobs: -d mcr.microsoft.com/mssql/server:2022-latest # Wait until SQL Server is ready - sql_ready=false for i in {1..30}; do - if docker exec sqlserver \ + docker exec sqlserver \ /opt/mssql-tools18/bin/sqlcmd \ -S localhost \ -U SA \ -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1"; then - sql_ready=true - break - fi + -C -Q "SELECT 1" && break sleep 2 done - if [ "$sql_ready" != true ]; then - # one last probe, in case it came up during the final sleep - docker exec sqlserver \ - /opt/mssql-tools18/bin/sqlcmd \ - -S localhost \ - -U SA \ - -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" || { - echo "SQL Server did not become ready after 30 attempts" - docker logs --tail 200 sqlserver || true - exit 1 - } - fi displayName: 'Start SQL Server container' env: DB_PASSWORD: $(DB_PASSWORD) From ab504bf706a8d818c28d2bd150857b8d6cb153fc Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:19:26 +0530 Subject: [PATCH 3/3] FIX: fail SQL Server setup loudly and dump container logs on every leg extends the macOS change to the remaining seven readiness loops. the CodeCoverageReport loop had the same silent-success shape as macOS did: its exit status is the final sleep, so a dead container let the step pass. the six linux distro legs already failed through CREATE DATABASE, but only as an opaque sqlcmd error with no container log. all eight sites now share one shape: track readiness in a flag, probe once more after the loop, and on failure print the attempt count plus docker logs before exiting non-zero. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 130 ++++++++++++++++++++++- 1 file changed, 128 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 846ee50d..100af715 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -784,6 +784,7 @@ jobs: # Wait for SQL Server to be ready echo "Waiting for SQL Server to start..." + sql_ready=false for i in {1..60}; do if docker exec sqlserver-$(distroName) \ /opt/mssql-tools18/bin/sqlcmd \ @@ -792,11 +793,28 @@ jobs: -P "$(DB_PASSWORD)" \ -C -Q "SELECT 1" >/dev/null 2>&1; then echo "SQL Server is ready!" + sql_ready=true break fi echo "Waiting... ($i/60)" sleep 2 done + if [ "$sql_ready" != true ]; then + # the loop sleeps after its final attempt, so probe once more + if docker exec sqlserver-$(distroName) \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + else + echo "SQL Server did not become ready after 60 attempts" + echo "---- docker logs sqlserver-$(distroName) (last 200 lines) ----" + docker logs --tail 200 sqlserver-$(distroName) || true + exit 1 + fi + fi # Create test database docker exec sqlserver-$(distroName) \ @@ -1113,6 +1131,7 @@ jobs: # Wait for SQL Server to be ready echo "Waiting for SQL Server to start..." + sql_ready=false for i in {1..60}; do if docker exec sqlserver-$(distroName)-$(archName) \ /opt/mssql-tools18/bin/sqlcmd \ @@ -1121,11 +1140,28 @@ jobs: -P "$(DB_PASSWORD)" \ -C -Q "SELECT 1" >/dev/null 2>&1; then echo "SQL Server is ready!" + sql_ready=true break fi echo "Waiting... ($i/60)" sleep 2 done + if [ "$sql_ready" != true ]; then + # the loop sleeps after its final attempt, so probe once more + if docker exec sqlserver-$(distroName)-$(archName) \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + else + echo "SQL Server did not become ready after 60 attempts" + echo "---- docker logs sqlserver-$(distroName)-$(archName) (last 200 lines) ----" + docker logs --tail 200 sqlserver-$(distroName)-$(archName) || true + exit 1 + fi + fi # Create test database docker exec sqlserver-$(distroName)-$(archName) \ @@ -1343,6 +1379,7 @@ jobs: # Wait for SQL Server to be ready echo "Waiting for SQL Server to start..." + sql_ready=false for i in {1..60}; do if docker exec sqlserver-rhel9 \ /opt/mssql-tools18/bin/sqlcmd \ @@ -1351,11 +1388,28 @@ jobs: -P "$(DB_PASSWORD)" \ -C -Q "SELECT 1" >/dev/null 2>&1; then echo "SQL Server is ready!" + sql_ready=true break fi echo "Waiting... ($i/60)" sleep 2 done + if [ "$sql_ready" != true ]; then + # the loop sleeps after its final attempt, so probe once more + if docker exec sqlserver-rhel9 \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + else + echo "SQL Server did not become ready after 60 attempts" + echo "---- docker logs sqlserver-rhel9 (last 200 lines) ----" + docker logs --tail 200 sqlserver-rhel9 || true + exit 1 + fi + fi # Create test database docker exec sqlserver-rhel9 \ @@ -1570,6 +1624,7 @@ jobs: # Wait for SQL Server to be ready echo "Waiting for SQL Server to start..." + sql_ready=false for i in {1..60}; do if docker exec sqlserver-rhel9-arm64 \ /opt/mssql-tools18/bin/sqlcmd \ @@ -1578,11 +1633,28 @@ jobs: -P "$(DB_PASSWORD)" \ -C -Q "SELECT 1" >/dev/null 2>&1; then echo "SQL Server is ready!" + sql_ready=true break fi echo "Waiting... ($i/60)" sleep 2 done + if [ "$sql_ready" != true ]; then + # the loop sleeps after its final attempt, so probe once more + if docker exec sqlserver-rhel9-arm64 \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + else + echo "SQL Server did not become ready after 60 attempts" + echo "---- docker logs sqlserver-rhel9-arm64 (last 200 lines) ----" + docker logs --tail 200 sqlserver-rhel9-arm64 || true + exit 1 + fi + fi # Create test database docker exec sqlserver-rhel9-arm64 \ @@ -1801,6 +1873,7 @@ jobs: # Wait for SQL Server to be ready echo "Waiting for SQL Server to start..." + sql_ready=false for i in {1..60}; do if docker exec sqlserver-alpine \ /opt/mssql-tools18/bin/sqlcmd \ @@ -1809,11 +1882,28 @@ jobs: -P "$(DB_PASSWORD)" \ -C -Q "SELECT 1" >/dev/null 2>&1; then echo "SQL Server is ready!" + sql_ready=true break fi echo "Waiting... ($i/60)" sleep 2 done + if [ "$sql_ready" != true ]; then + # the loop sleeps after its final attempt, so probe once more + if docker exec sqlserver-alpine \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + else + echo "SQL Server did not become ready after 60 attempts" + echo "---- docker logs sqlserver-alpine (last 200 lines) ----" + docker logs --tail 200 sqlserver-alpine || true + exit 1 + fi + fi # Create test database docker exec sqlserver-alpine \ @@ -2057,6 +2147,7 @@ jobs: # Wait for SQL Server to be ready echo "Waiting for SQL Server to start..." + sql_ready=false for i in {1..60}; do if docker exec sqlserver-alpine-arm64 \ /opt/mssql-tools18/bin/sqlcmd \ @@ -2065,11 +2156,28 @@ jobs: -P "$(DB_PASSWORD)" \ -C -Q "SELECT 1" >/dev/null 2>&1; then echo "SQL Server is ready!" + sql_ready=true break fi echo "Waiting... ($i/60)" sleep 2 done + if [ "$sql_ready" != true ]; then + # the loop sleeps after its final attempt, so probe once more + if docker exec sqlserver-alpine-arm64 \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready!" + else + echo "SQL Server did not become ready after 60 attempts" + echo "---- docker logs sqlserver-alpine-arm64 (last 200 lines) ----" + docker logs --tail 200 sqlserver-alpine-arm64 || true + exit 1 + fi + fi # Create test database docker exec sqlserver-alpine-arm64 \ @@ -2526,15 +2634,33 @@ jobs: -d mcr.microsoft.com/mssql/server:2022-latest # Wait until SQL Server is ready + sql_ready=false for i in {1..30}; do - docker exec sqlserver \ + if docker exec sqlserver \ /opt/mssql-tools18/bin/sqlcmd \ -S localhost \ -U SA \ -P "$(DB_PASSWORD)" \ - -C -Q "SELECT 1" && break + -C -Q "SELECT 1"; then + sql_ready=true + break + fi sleep 2 done + if [ "$sql_ready" != true ]; then + # the loop sleeps after its final attempt, so probe once more + if ! docker exec sqlserver \ + /opt/mssql-tools18/bin/sqlcmd \ + -S localhost \ + -U SA \ + -P "$(DB_PASSWORD)" \ + -C -Q "SELECT 1"; then + echo "SQL Server did not become ready after 30 attempts" + echo "---- docker logs sqlserver (last 200 lines) ----" + docker logs --tail 200 sqlserver || true + exit 1 + fi + fi displayName: 'Start SQL Server container' env: DB_PASSWORD: $(DB_PASSWORD)