fix: skip webhook regex matching for exp config (#9511)

determined-ai · Jun 14, 2024 · f9ba7f4 · f9ba7f4
1 parent b51bc93
commit f9ba7f4
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 4 deletions.
diff --git a/e2e_tests/tests/cluster/test_webhooks.py b/e2e_tests/tests/cluster/test_webhooks.py
@@ -89,7 +89,7 @@ def test_log_pattern_send_webhook(should_match: bool) -> None:
 
     regex = r"assert 0 <= self\.metrics_sigma"
     if not should_match:
-        regex = r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"
+        regex = r"(.*)cuda(.*)"
 
     webhook_trigger = bindings.v1Trigger(
         triggerType=bindings.v1TriggerType.TASK_LOG,

diff --git a/master/internal/logpattern/logpattern.go b/master/internal/logpattern/logpattern.go
@@ -18,8 +18,9 @@ import (
 const regexCacheSize = 256
 
 var (
-	defaultSingleton       *LogPatternPolicies
-	expconfigCompiledRegex = regexp.MustCompile("(.*)(\\\"log_policies\\\":)(.*)")
+	defaultSingleton *LogPatternPolicies
+	// ExpconfigCompiledRegex is used to identify user submitted config.
+	ExpconfigCompiledRegex = regexp.MustCompile("(.*)(\\\"log_policies\\\":)(.*)")
 )
 
 // LogPatternPolicies performs log pattern checks.
@@ -70,7 +71,7 @@ func (l *LogPatternPolicies) monitor(ctx context.Context,
 
 			// One of the trial logs prints expconf which has the regex pattern.
 			// We skip monitoring this line.
-			if expconfigCompiledRegex.MatchString(log.Log) {
+			if ExpconfigCompiledRegex.MatchString(log.Log) {
 				continue
 			}
 

diff --git a/master/internal/webhooks/postgres_webhook.go b/master/internal/webhooks/postgres_webhook.go
@@ -14,6 +14,7 @@ import (
 
 	conf "github.com/determined-ai/determined/master/internal/config"
 	"github.com/determined-ai/determined/master/internal/db"
+	"github.com/determined-ai/determined/master/internal/logpattern"
 	"github.com/determined-ai/determined/master/internal/project"
 	"github.com/determined-ai/determined/master/internal/workspace"
 	"github.com/determined-ai/determined/master/pkg/model"
@@ -121,6 +122,11 @@ func (l *WebhookManager) scanLogs(ctx context.Context, logs []*model.TaskLog) er
 		}
 
 		for _, cacheItem := range l.regexToTriggers {
+			// One of the trial logs prints expconf which has the regex pattern.
+			// We skip monitoring this line.
+			if logpattern.ExpconfigCompiledRegex.MatchString(log.Log) {
+				continue
+			}
 			if cacheItem.re.MatchString(log.Log) {
 				for _, t := range cacheItem.triggerIDToTrigger {
 					if err := addTaskLogEvent(ctx,