[ WIP ] test(bridge): fix flaky test: TestGithubPushPull

sudoforge created 5 months ago

This test periodically fails with the bug_label_change variant, caused
by an unexpected number of operations existing on the bug (typically in
the range of 7-9).

This tree exists to debug it.

NOTE: This tree is built on top of
de7def3f81745c0c24a482a1e9a5dcb4fbbd69dd, and should not be merged in
directly. Its parent effectively enables this test to run on CI, which
is used periodically during the iteration of this adventure.

Change-Id: I738207f8cb254b66f3ef18aa525fce39c71060e2

Change summary

.github/workflows/build-and-test.yml | 51 +----------------------------
.github/workflows/presubmit.yml      |  3 +
bridge/github/export_test.go         | 33 +++++++++++++++++--
3 files changed, 35 insertions(+), 52 deletions(-)

Detailed changes

.github/workflows/build-and-test.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         go-version: [1.24.2]
-        platform: [ubuntu-latest, macos-latest, windows-latest]
+        platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Check out code
@@ -27,55 +27,10 @@ jobs:
         run: make
 
       - name: Test
-        run: make test
+        run: |
+          go test -v -run TestGithubPushPull ./bridge/github
         env:
           GITHUB_USER: ${{ vars.TEST_USER_GITHUB }}
           GITHUB_TOKEN: ${{ secrets.TEST_TOKEN_GITHUB }}
           GITHUB_TOKEN_PRIVATE: ${{ secrets._GITHUB_TOKEN_PRIVATE }}
           GITHUB_TOKEN_PUBLIC: ${{ secrets._GITHUB_TOKEN_PUBLIC }}
-          GITLAB_API_TOKEN: ${{ secrets.GITLAB_API_TOKEN }}
-          GITLAB_PROJECT_ID: ${{ secrets.GITLAB_PROJECT_ID }}
-
-  with-node:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [16.x, 18.x, 20.x]
-    defaults:
-      run:
-        working-directory: webui
-    steps:
-      - name: Setup Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: Check out code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - uses: DeterminateSystems/nix-installer-action@21a544727d0c62386e78b4befe52d19ad12692e3 # v17
-
-      - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1
-
-      - name: Install
-        run: make install
-
-      - name: Build
-        run: make build
-
-      - name: Test
-        run: make test
-
-  with-nix:
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-      - uses: DeterminateSystems/nix-installer-action@21a544727d0c62386e78b4befe52d19ad12692e3 # v17
-
-      - uses: nicknovitski/nix-develop@9be7cfb4b10451d3390a75dc18ad0465bed4932a # v1.2.1
-
-      - run: nix flake check

.github/workflows/presubmit.yml

@@ -12,6 +12,9 @@ on:
   merge_group:
     types:
       - checks_requested
+  push:
+    branches:
+      - I738207f8cb254b66f3ef18aa525fce39c71060e2
   pull_request:
     branches:
       - master

bridge/github/export_test.go

@@ -191,10 +191,21 @@ func TestGithubPushPull(t *testing.T) {
 
 	// Make sure to remove the Github repository when the test end
 	defer func(t *testing.T) {
-		if err := deleteRepository(projectName, envUser, envToken); err != nil {
-			t.Fatal(err)
+		ci := os.Getenv("CI") == "true"
+		if !t.Failed() || ci {
+			if err := deleteRepository(projectName, envUser, envToken); err != nil {
+				t.Fatal(err)
+			}
+
+			reason := "test success"
+			if ci {
+				reason = "CI"
+			}
+
+			slog.Info("deleted repository", "reason", reason, "name", projectName)
+		} else {
+			slog.Info("persisted repository", "reason", "test failure", "name", projectName)
 		}
-		fmt.Println("deleted repository:", projectName)
 	}(t)
 
 	interrupt.RegisterCleaner(func() error {
@@ -275,8 +286,22 @@ func TestGithubPushPull(t *testing.T) {
 			importedBug, err := backendTwo.Bugs().ResolveBugCreateMetadata(metaKeyGithubId, bugGithubID)
 			require.NoError(t, err)
 
+			importedOpCount := len(importedBug.Snapshot().Operations)
+
+			if tt.numOrOp != len(importedBug.Snapshot().Operations) {
+				slog.Info("invalid number of ops for imported bug", "github-id", bugGithubID, "title", importedBug.Snapshot().Title, "opCount", importedOpCount, "labels", importedBug.Snapshot().Labels)
+				for _, op := range importedBug.Snapshot().Operations {
+					slog.Info("  operation", "type", op.Type(), "time", op.Time(), "author", op.Author().Name())
+					for k, v := range op.AllMetadata() {
+						slog.Info("    metadata", "key", k, "val", v)
+					}
+				}
+
+				slog.Info("final snapshot", "status", importedBug.Snapshot().Status.String(), "labels", importedBug.Snapshot().Labels)
+			}
+
 			// verify bug have same number of original operations
-			require.Len(t, importedBug.Snapshot().Operations, tt.numOrOp)
+			require.Equal(t, importedOpCount, tt.numOrOp)
 
 			// verify bugs are tagged with origin=github
 			issueOrigin, ok := importedBug.Snapshot().GetCreateMetadata(core.MetaKeyOrigin)