Try to diagnose memory access violation in Windows tests (#35926)

Cole Miller created

Release Notes:

- N/A

Change summary

.github/actions/run_tests_windows/action.yml | 163 +++++++++++++++++++++
1 file changed, 162 insertions(+), 1 deletion(-)

Detailed changes

.github/actions/run_tests_windows/action.yml šŸ”—

@@ -20,7 +20,168 @@ runs:
       with:
         node-version: "18"
 
+    - name: Configure crash dumps
+      shell: powershell
+      run: |
+        # Record the start time for this CI run
+        $runStartTime = Get-Date
+        $runStartTimeStr = $runStartTime.ToString("yyyy-MM-dd HH:mm:ss")
+        Write-Host "CI run started at: $runStartTimeStr"
+
+        # Save the timestamp for later use
+        echo "CI_RUN_START_TIME=$($runStartTime.Ticks)" >> $env:GITHUB_ENV
+
+        # Create crash dump directory in workspace (non-persistent)
+        $dumpPath = "$env:GITHUB_WORKSPACE\crash_dumps"
+        New-Item -ItemType Directory -Force -Path $dumpPath | Out-Null
+
+        Write-Host "Setting up crash dump detection..."
+        Write-Host "Workspace dump path: $dumpPath"
+
+        # Note: We're NOT modifying registry on stateful runners
+        # Instead, we'll check default Windows crash locations after tests
+
     - name: Run tests
       shell: powershell
       working-directory: ${{ inputs.working-directory }}
-      run: cargo nextest run --workspace --no-fail-fast
+      run: |
+        $env:RUST_BACKTRACE = "full"
+
+        # Enable Windows debugging features
+        $env:_NT_SYMBOL_PATH = "srv*https://msdl.microsoft.com/download/symbols"
+
+        # .NET crash dump environment variables (ephemeral)
+        $env:COMPlus_DbgEnableMiniDump = "1"
+        $env:COMPlus_DbgMiniDumpType = "4"
+        $env:COMPlus_CreateDumpDiagnostics = "1"
+
+        cargo nextest run --workspace --no-fail-fast
+      continue-on-error: true
+
+    - name: Analyze crash dumps
+      if: always()
+      shell: powershell
+      run: |
+        Write-Host "Checking for crash dumps..."
+
+        # Get the CI run start time from the environment
+        $runStartTime = [DateTime]::new([long]$env:CI_RUN_START_TIME)
+        Write-Host "Only analyzing dumps created after: $($runStartTime.ToString('yyyy-MM-dd HH:mm:ss'))"
+
+        # Check all possible crash dump locations
+        $searchPaths = @(
+            "$env:GITHUB_WORKSPACE\crash_dumps",
+            "$env:LOCALAPPDATA\CrashDumps",
+            "$env:TEMP",
+            "$env:GITHUB_WORKSPACE",
+            "$env:USERPROFILE\AppData\Local\CrashDumps",
+            "C:\Windows\System32\config\systemprofile\AppData\Local\CrashDumps"
+        )
+
+        $dumps = @()
+        foreach ($path in $searchPaths) {
+            if (Test-Path $path) {
+                Write-Host "Searching in: $path"
+                $found = Get-ChildItem "$path\*.dmp" -ErrorAction SilentlyContinue | Where-Object {
+                    $_.CreationTime -gt $runStartTime
+                }
+                if ($found) {
+                    $dumps += $found
+                    Write-Host "  Found $($found.Count) dump(s) from this CI run"
+                }
+            }
+        }
+
+        if ($dumps) {
+          Write-Host "Found $($dumps.Count) crash dump(s)"
+
+          # Install debugging tools if not present
+          $cdbPath = "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
+          if (-not (Test-Path $cdbPath)) {
+            Write-Host "Installing Windows Debugging Tools..."
+            $url = "https://go.microsoft.com/fwlink/?linkid=2237387"
+            Invoke-WebRequest -Uri $url -OutFile winsdksetup.exe
+            Start-Process -Wait winsdksetup.exe -ArgumentList "/features OptionId.WindowsDesktopDebuggers /quiet"
+          }
+
+          foreach ($dump in $dumps) {
+            Write-Host "`n=================================="
+            Write-Host "Analyzing crash dump: $($dump.Name)"
+            Write-Host "Size: $([math]::Round($dump.Length / 1MB, 2)) MB"
+            Write-Host "Time: $($dump.CreationTime)"
+            Write-Host "=================================="
+
+            # Set symbol path
+            $env:_NT_SYMBOL_PATH = "srv*C:\symbols*https://msdl.microsoft.com/download/symbols"
+
+            # Run analysis
+            $analysisOutput = & $cdbPath -z $dump.FullName -c "!analyze -v; ~*k; lm; q" 2>&1 | Out-String
+
+            # Extract key information
+            if ($analysisOutput -match "ExceptionCode:\s*([\w]+)") {
+              Write-Host "Exception Code: $($Matches[1])"
+              if ($Matches[1] -eq "c0000005") {
+                Write-Host "Exception Type: ACCESS VIOLATION"
+              }
+            }
+
+            if ($analysisOutput -match "EXCEPTION_RECORD:\s*(.+)") {
+              Write-Host "Exception Record: $($Matches[1])"
+            }
+
+            if ($analysisOutput -match "FAULTING_IP:\s*\n(.+)") {
+              Write-Host "Faulting Instruction: $($Matches[1])"
+            }
+
+            # Save full analysis
+            $analysisFile = "$($dump.FullName).analysis.txt"
+            $analysisOutput | Out-File -FilePath $analysisFile
+            Write-Host "`nFull analysis saved to: $analysisFile"
+
+            # Print stack trace section
+            Write-Host "`n--- Stack Trace Preview ---"
+            $stackSection = $analysisOutput -split "STACK_TEXT:" | Select-Object -Last 1
+            $stackLines = $stackSection -split "`n" | Select-Object -First 20
+            $stackLines | ForEach-Object { Write-Host $_ }
+            Write-Host "--- End Stack Trace Preview ---"
+          }
+
+          Write-Host "`nāš ļø  Crash dumps detected! Download the 'crash-dumps' artifact for detailed analysis."
+
+          # Copy dumps to workspace for artifact upload
+          $artifactPath = "$env:GITHUB_WORKSPACE\crash_dumps_collected"
+          New-Item -ItemType Directory -Force -Path $artifactPath | Out-Null
+
+          foreach ($dump in $dumps) {
+            $destName = "$($dump.Directory.Name)_$($dump.Name)"
+            Copy-Item $dump.FullName -Destination "$artifactPath\$destName"
+            if (Test-Path "$($dump.FullName).analysis.txt") {
+              Copy-Item "$($dump.FullName).analysis.txt" -Destination "$artifactPath\$destName.analysis.txt"
+            }
+          }
+
+          Write-Host "Copied $($dumps.Count) dump(s) to artifact directory"
+        } else {
+          Write-Host "No crash dumps from this CI run found"
+        }
+
+    - name: Upload crash dumps
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: crash-dumps-${{ github.run_id }}-${{ github.run_attempt }}
+        path: |
+          crash_dumps_collected/*.dmp
+          crash_dumps_collected/*.txt
+        if-no-files-found: ignore
+        retention-days: 7
+
+    - name: Check test results
+      shell: powershell
+      working-directory: ${{ inputs.working-directory }}
+      run: |
+        # Re-check test results to fail the job if tests failed
+        if ($LASTEXITCODE -ne 0) {
+          Write-Host "Tests failed with exit code: $LASTEXITCODE"
+          exit $LASTEXITCODE
+        }