fix: unlock mutex for experiment ResourcePool() [RM-152] (#9119)

docs/release-notes/experiment-deadlock.rst

@@ -0,0 +1,6 @@
+:orphan:
+
+**Bug Fixes**
+
+-  API: Fix a bug where calling ``det job update`` could prevent jobs from being scheduled and ``det
+   job ls`` to hang.

master/.golangci.yml

@@ -81,6 +81,7 @@ linters-settings:
       - 'bundebug.WithVerbose'
       - 'http.Client' # Use cleanhttp instead.
       - 'http.Transport' # Use cleanhttp instead.
+      - 'defer .*\.Lock\(\)'
   staticcheck:
     go: "1.20"
 

master/internal/api_project_intg_test.go

@@ -30,10 +30,12 @@ import (
 	"github.com/determined-ai/determined/proto/pkg/userv1"
 )
 
+const mockType = "mock"
+
 var pAuthZ *mocks.ProjectAuthZ
 
 func isMockAuthZ() bool {
-	return config.GetMasterConfig().Security.AuthZ.Type == "mock"
+	return config.GetMasterConfig().Security.AuthZ.Type == mockType
 }
 
 // pgdb can be nil to use the singleton database for testing.
@@ -45,7 +47,7 @@ func setupProjectAuthZTest(
 
 	if pAuthZ == nil {
 		pAuthZ = &mocks.ProjectAuthZ{}
-		project.AuthZProvider.Register("mock", pAuthZ)
+		project.AuthZProvider.Register(mockType, pAuthZ)
 	}
 	return api, pAuthZ, workspaceAuthZ, curUser, ctx
 }

master/internal/experiment_job_service.go

@@ -82,7 +82,7 @@ func (e *internalExperiment) SetResourcePool(resourcePool string) error {
 // ResourcePool gets the experiment's resource pool.
 func (e *internalExperiment) ResourcePool() string {
 	e.mu.Lock()
-	defer e.mu.Lock()
+	defer e.mu.Unlock()
 
 	return e.activeConfig.Resources().ResourcePool()
 }

master/internal/experiment_job_service_test.go

@@ -0,0 +1,24 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/determined-ai/determined/master/pkg/schemas/expconf"
+)
+
+func TestResourcePool(t *testing.T) {
+	rp := "mock"
+	//nolint:exhaustruct
+	e := &internalExperiment{
+		activeConfig: expconf.ExperimentConfigV0{
+			RawResources: &expconf.ResourcesConfigV0{
+				RawResourcePool: &rp,
+			},
+		},
+	}
+
+	rpName := e.ResourcePool()
+	require.Equal(t, rp, rpName)
+}