fix: adding logic to apply trustyai prometheus

components/trustyai/trustyai.go

@@ -3,6 +3,7 @@ package trustyai
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
@@ -13,6 +14,7 @@ import (
 	dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1"
 	"github.com/opendatahub-io/opendatahub-operator/v2/components"
 	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy"
+	"github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring"
 )
 
 var (
@@ -50,12 +52,13 @@ func (t *TrustyAI) GetComponentName() string {
 	return ComponentName
 }
 
-func (t *TrustyAI) ReconcileComponent(_ context.Context, cli client.Client, _ *rest.Config, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error {
+func (t *TrustyAI) ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error {
 	var imageParamMap = map[string]string{
 		"trustyaiServiceImage":  "RELATED_IMAGE_ODH_TRUSTYAI_SERVICE_IMAGE",
 		"trustyaiOperatorImage": "RELATED_IMAGE_ODH_TRUSTYAI_SERVICE_OPERATOR_IMAGE",
 	}
 	enabled := t.GetManagementState() == operatorv1.Managed
+	monitoringEnabled := dscispec.Monitoring.ManagementState == operatorv1.Managed
 
 	platform, err := deploy.GetPlatform(cli)
 	if err != nil {
@@ -76,6 +79,27 @@ func (t *TrustyAI) ReconcileComponent(_ context.Context, cli client.Client, _ *r
 		}
 	}
 	// Deploy TrustyAI Operator
-	err = deploy.DeployManifestsFromPath(cli, owner, Path, dscispec.ApplicationsNamespace, t.GetComponentName(), enabled)
-	return err
+	if err := deploy.DeployManifestsFromPath(cli, owner, Path, dscispec.ApplicationsNamespace, t.GetComponentName(), enabled); err != nil {
+		return err
+	}
+
+	// CloudService Monitoring handling
+	if platform == deploy.ManagedRhods {
+		if enabled {
+			if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 10, 1); err != nil {
+				return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err)
+			}
+			fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentName)
+		}
+		if err := t.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil {
+			return err
+		}
+		if err = deploy.DeployManifestsFromPath(cli, owner,
+			filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"),
+			dscispec.Monitoring.Namespace,
+			"prometheus", true); err != nil {
+			return err
+		}
+	}
+	return nil
 }

config/monitoring/prometheus/apps/prometheus-configs.yaml

@@ -314,19 +314,22 @@ data:
       metrics_path: /metrics
       scheme: http
       kubernetes_sd_configs:
-        - role: endpoints
+        - role: pod
           namespaces:
             names:
               - <odh_application_namespace>
+          selectors:
+            - role: pod
+              label: 'app.opendatahub.io/trustyai=true'
       relabel_configs:
-        - source_labels: [__meta_kubernetes_service_name]
-          regex: ^(trustyai-service-operator-controller-manager)$
+        - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_part_of]
+          regex: ^(trustyai)$
           target_label: kubernetes_name
           action: keep
         - source_labels: [__address__]
           regex: (.+):(\d+)
           target_label: __address__
-          replacement: ${1}:8080          
+          replacement: ${1}:8080
 
     - job_name: 'RHODS Metrics'
       honor_labels: true
@@ -1184,7 +1187,7 @@ data:
 
   kserve-alerting.rules: |
     groups:
-      - name: SLOs-probe_success
+      - name: SLOs-probe_success_kserve
         rules:
         - alert: Kserve Controller Probe Success Burn Rate
           annotations:
@@ -1464,7 +1467,7 @@ data:
           record: probe_success:burnrate6h
   trustyai-alerting.rules: |
     groups:
-      - name: SLOs-probe_success
+      - name: SLOs-probe_success_trustyai
         rules:
         - alert: TrustyAI Controller Probe Success Burn Rate
           annotations:
@@ -1478,6 +1481,7 @@ data:
           for: 2m
           labels:
             severity: critical
+            instance: trustyai-service-operator-controller-manager
         - alert: TrustyAI Controller Probe Success Burn Rate
           annotations:
             message: 'High error budget burn for {{ $labels.instance }} (current value: {{ $value }}).'
@@ -1490,6 +1494,7 @@ data:
           for: 15m
           labels:
             severity: critical
+            instance: trustyai-service-operator-controller-manager
         - alert: TrustyAI Controller Probe Success Burn Rate
           annotations:
             message: 'High error budget burn for {{ $labels.instance }} (current value: {{ $value }}).'
@@ -1502,3 +1507,4 @@ data:
           for: 1h
           labels:
             severity: warning
+            instance: trustyai-service-operator-controller-manager