Retry sentry uploads (#43267)

Conrad Irwin created 1 month ago

We see internal server errors occasionally; and it's very annoying to
have to re-run the entire step

Release Notes:

- N/A

Change summary

script/bundle-linux       | 17 ++++++++++++++---
script/bundle-mac         | 17 +++++++++++++++--
script/bundle-windows.ps1 | 15 ++++++++++++++-
3 files changed, 43 insertions(+), 6 deletions(-)

Detailed changes

script/bundle-linux

@@ -92,9 +92,20 @@ else
         echo "Uploading zed debug symbols to sentry..."
         # note: this uploads the unstripped binary which is needed because it contains
         # .eh_frame data for stack unwinding. see https://github.com/getsentry/symbolic/issues/783
-        sentry-cli debug-files upload --include-sources --wait -p zed -o zed-dev \
-            "${target_dir}/${target_triple}"/release/zed \
-            "${target_dir}/${remote_server_triple}"/release/remote_server
+        for attempt in 1 2 3; do
+            echo "Attempting sentry upload (attempt $attempt/3)..."
+            if sentry-cli debug-files upload --include-sources --wait -p zed -o zed-dev \
+                "${target_dir}/${target_triple}"/release/zed \
+                "${target_dir}/${remote_server_triple}"/release/remote_server; then
+                echo "Sentry upload successful on attempt $attempt"
+                break
+            else
+                echo "Sentry upload failed on attempt $attempt"
+                if [ $attempt -eq 3 ]; then
+                    echo "All sentry upload attempts failed"
+                fi
+            fi
+        done
     else
         echo "missing SENTRY_AUTH_TOKEN. skipping sentry upload."
     fi

script/bundle-mac

@@ -300,8 +300,21 @@ function upload_debug_symbols() {
         # note: this uploads the unstripped binary which is needed because it contains
         # .eh_frame data for stack unwinding. see https://github.com/getsentry/symbolic/issues/783
         sentry-cli debug-files upload --include-sources --wait -p zed -o zed-dev \
-            "target/${target_triple}/${target_dir}/zed.dwarf" \
-            "target/${target_triple}/${target_dir}/remote_server.dwarf"
+        # Try uploading up to 3 times
+        for attempt in 1 2 3; do
+            echo "Sentry upload attempt $attempt..."
+            if sentry-cli debug-files upload --include-sources --wait -p zed -o zed-dev \
+                "target/${target_triple}/${target_dir}/zed.dwarf" \
+                "target/${target_triple}/${target_dir}/remote_server.dwarf"; then
+                break
+            else
+                echo "Sentry upload failed on attempt $attempt"
+                if [ $attempt -eq 3 ]; then
+                    echo "All sentry upload attempts failed"
+                    exit 1
+                fi
+            fi
+        done
     else
         echo "missing SENTRY_AUTH_TOKEN. skipping sentry upload."
     fi

script/bundle-windows.ps1

@@ -147,7 +147,20 @@ function UploadToSentry {
         return
     }
     Write-Output "Uploading zed debug symbols to sentry..."
-    sentry-cli debug-files upload --include-sources --wait -p zed -o zed-dev $CargoOutDir
+    for ($i = 1; $i -le 3; $i++) {
+        try {
+            sentry-cli debug-files upload --include-sources --wait -p zed -o zed-dev $CargoOutDir
+            break
+        }
+        catch {
+            Write-Output "Sentry upload attempt $i failed: $_"
+            if ($i -eq 3) {
+                Write-Output "All sentry upload attempts failed"
+                throw
+            }
+            Start-Sleep -Seconds 2
+        }
+    }
 }
 
 function MakeAppx {