Second attempt at fixing agents flakes (#1281)

For #1280

Co-authored-by: Trajan0x <trajan0x@users.noreply.github.com>

.github/workflows/go.yml

@@ -163,9 +163,9 @@ jobs:
       - name: Test
         uses: nick-fields/retry@v2
         with:
-          command: cd ${{matrix.package}} && $PYROSCOPE_PATH exec --apiKey=${{ secrets.PYROSCOPE_CLOUD_TOKEN }} -- go test -coverpkg=./... ./... -coverprofile=profile.cov
-          max_attempts: 6
-          timeout_minutes: 15
+          command: cd ${{matrix.package}} && $PYROSCOPE_PATH exec --apiKey=${{ secrets.PYROSCOPE_CLOUD_TOKEN }} -- go test -parallel 1 -coverpkg=./... ./... -coverprofile=profile.cov
+          max_attempts: 2
+          timeout_minutes: 20
         env:
           PYROSCOPE_PATH: '${{ steps.pyroscope-path.outputs.PYROSCOPE_PATH }}'
           ENABLE_MYSQL_TEST: true
@@ -175,10 +175,11 @@ jobs:
           MYSQL_DATABASE: test
           MYSQL_ROOT_PASSWORD: password
           MYSQL_PORT: ${{ job.services.mariadb.ports[3306] }}
-          GOMAXPROCS: 18
-          GOMEMLIMIT: 6GiB
+          GOMEMLIMIT: 3GiB
           GOGC: -1
           ETHEREUM_RPC_URI: ${{ secrets.ETHEREUM_RPC_URI }}
+          GOPROXY: https://proxy.golang.org
+
 
       - name: Send Coverage (Codecov)
         uses: Wandalen/wretry.action@v1.0.36

.github/workflows/sonar.yml

@@ -1,14 +1,25 @@
-name: Build
+name: SonarCloud
 on:
   push:
     branches:
       - master
   pull_request:
     types: [opened, synchronize, reopened]
 jobs:
+  cancel-outdated:
+    name: Cancel Outdated Jobs
+    runs-on: ubuntu-latest
+    steps:
+      - id: skip_check
+        if: ${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref && !contains(github.event.head_commit.message, '[no_skip]') }}
+        uses: fkirc/skip-duplicate-actions@v5
+        with:
+          cancel_others: 'true'
+
   sonarcloud:
     name: SonarCloud
     runs-on: ubuntu-latest
+    needs: cancel-outdated # not strictly true, but this can prevent others from building up too quickly/cancel-outdated is fast
     steps:
       - uses: actions/checkout@v3
         with:

agents/testutil/simulated_backends_suite.go

@@ -1,6 +1,7 @@
 package testutil
 
 import (
+	"github.com/synapsecns/sanguine/core"
 	"math/big"
 	"sync"
 	"testing"
@@ -142,23 +143,31 @@ func NewSimulatedBackendsTestSuite(tb testing.TB) *SimulatedBackendsTestSuite {
 func (a *SimulatedBackendsTestSuite) SetupSuite() {
 	a.TestSuite.SetupSuite()
 	a.TestSuite.LogDir = filet.TmpDir(a.T(), "")
-	localmetrics.SetupTestJaeger(a.GetSuiteContext(), a.T())
+
+	// don't use metrics on ci for integration tests
+	useMetrics := core.GetEnvBool("CI", true)
+	metricsHandler := metrics.Null
+
+	if useMetrics {
+		localmetrics.SetupTestJaeger(a.GetSuiteContext(), a.T())
+		metricsHandler = metrics.Jaeger
+	}
 
 	var err error
-	a.ScribeMetrics, err = metrics.NewByType(a.GetSuiteContext(), scribeMetadata.BuildInfo(), metrics.Jaeger)
+	a.ScribeMetrics, err = metrics.NewByType(a.GetSuiteContext(), scribeMetadata.BuildInfo(), metricsHandler)
 	a.Require().Nil(err)
-	a.ExecutorMetrics, err = metrics.NewByType(a.GetSuiteContext(), executorMetadata.BuildInfo(), metrics.Jaeger)
+	a.ExecutorMetrics, err = metrics.NewByType(a.GetSuiteContext(), executorMetadata.BuildInfo(), metricsHandler)
 	a.Require().Nil(err)
-	a.NotaryMetrics, err = metrics.NewByType(a.GetSuiteContext(), notaryMetadata.BuildInfo(), metrics.Jaeger)
+	a.NotaryMetrics, err = metrics.NewByType(a.GetSuiteContext(), notaryMetadata.BuildInfo(), metricsHandler)
 	a.Require().Nil(err)
-	a.GuardMetrics, err = metrics.NewByType(a.GetSuiteContext(), guardMetadata.BuildInfo(), metrics.Jaeger)
+	a.GuardMetrics, err = metrics.NewByType(a.GetSuiteContext(), guardMetadata.BuildInfo(), metricsHandler)
 	a.Require().Nil(err)
 	a.ContractMetrics, err = metrics.NewByType(a.GetSuiteContext(), coreConfig.NewBuildInfo(
 		coreConfig.DefaultVersion,
 		coreConfig.DefaultCommit,
 		"contract",
 		coreConfig.DefaultDate,
-	), metrics.Jaeger)
+	), metricsHandler)
 	a.Require().Nil(err)
 }
 

core/README.md

@@ -31,3 +31,4 @@ root
 ├── <a href="./tunnel">tunnel</a>: Reverse tunneling service for debugging services in ci.
 </pre>
 
+

core/metrics/localmetrics/local.go

@@ -88,8 +88,8 @@ func startServer(parentCtx context.Context, tb testing.TB, options ...Option) *t
 
 	tb.Cleanup(func() {
 		defer cancel()
-		// TODO: move me
-		if tb.Failed() {
+		// Do not keep containers on ci.
+		if tb.Failed() && os.Getenv("CI") == "" {
 			logger.Warn("Test failed, will temporarily continue serving \n" + tj.buildLogMessage(false))
 		} else if !tj.cfg.keepContainers {
 			tj.purgeResources()
@@ -171,8 +171,8 @@ func (j *testJaeger) purgeResources() {
 	wg.Add(len(resources))
 	for _, resource := range resources {
 		go func(resource *dockertest.Resource) {
-			defer wg.Done()
 			_ = j.pool.Purge(resource)
+			wg.Done()
 		}(resource)
 	}
 	wg.Wait()

core/metrics/metrics_test.go

@@ -2,7 +2,6 @@ package metrics_test
 
 import (
 	"context"
-	"github.com/brianvoe/gofakeit/v6"
 	"github.com/stretchr/testify/assert"
 	. "github.com/stretchr/testify/assert"
 	"github.com/synapsecns/sanguine/core/config"
@@ -24,8 +23,6 @@ func TestSetupFromEnv(t *testing.T) {
 			defer cancel()
 
 			t.Setenv(metrics.HandlerEnv, handler.String())
-			// used for testing
-			t.Setenv("NEW_RELIC_LICENSE_KEY", gofakeit.Sentence(40)[0:40])
 
 			_, err := metrics.NewFromEnv(ctx, config.NewBuildInfo(config.DefaultVersion, config.DefaultCommit, config.AppName, config.DefaultDate))
 			Nil(t, err)

ethergo/backends/anvil/anvil.go

@@ -174,10 +174,14 @@ func NewAnvilBackend(ctx context.Context, t *testing.T, args *OptionBuilder) *Ba
 	err = backend.storeWallets(args)
 	require.Nilf(t, err, "failed to store wallets on chain id %s: %v", chainID, err)
 
-	go func() {
-		<-ctx.Done()
-		_ = pool.Purge(resource)
-	}()
+	t.Cleanup(func() {
+		select {
+		case <-ctx.Done():
+			_ = pool.Purge(resource)
+		default:
+			// do nothing, we don't want to purge the container if this is just a subtest
+		}
+	})
 
 	return &backend
 }

ethergo/backends/anvil/options.go

@@ -58,9 +58,6 @@ func NewAnvilOptionBuilder() *OptionBuilder {
 		panic(err)
 	}
 
-	// TODO: we'll enable this after the sourcify integration is complete for verifying contracts locally.
-	optionsBuilder.OtterscanEnabled(true)
-
 	return optionsBuilder
 }