wip

api/types/autodiscover.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2024 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package types
+
+// Auto Discover EC2 issues identifiers which are used to provide better error messages to the user.
+const (
+	// AutoDiscoverEC2IssueEICEFailedToCreateNode is used when the EICE flow fails to auto enroll an EC2 instance
+	// as an EICE node.
+	AutoDiscoverEC2IssueEICEFailedToCreateNode = "ec2-eice-creation"
+	// AutoDiscoverEC2IssueScriptSSMAgentNotRunning is used when the SSM Agent is not present in the instance.
+	// This can also happen when the SSM was not able to connect to AWS Systems Manager.
+	AutoDiscoverEC2IssueScriptSSMAgentNotRunning = "ec2-ssm-agent-not-running"
+)

api/types/usertasks/object.go

@@ -19,6 +19,8 @@
 package usertasks
 
 import (
+	"slices"
+
 	"github.com/gravitational/trace"
 
 	headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1"
@@ -45,6 +47,15 @@ func NewUserTask(name string, spec *usertasksv1.UserTaskSpec) (*usertasksv1.User
 	return cj, nil
 }
 
+const (
+	// TaskStateOpen identifies an issue with an instance that is not yet resolved.
+	TaskStateOpen = "OPEN"
+	// TaskStateResolved identifies an issue with an instance that is resolved.
+	TaskStateResolved = "RESOLVED"
+)
+
+var validTaskStates = []string{TaskStateOpen, TaskStateResolved}
+
 const (
 	// TaskTypeDiscoverEC2 identifies a User Tasks that is created
 	// when an auto-enrollment of an EC2 instance fails.
@@ -84,12 +95,32 @@ func ValidateUserTask(uit *usertasksv1.UserTask) error {
 }
 
 func validateDiscoverEC2TaskType(uit *usertasksv1.UserTask) error {
-	if uit.Spec.DiscoverEc2 == nil {
+	if uit.GetSpec().DiscoverEc2 == nil {
 		return trace.BadParameter("%s requires the discover_ec2 field", TaskTypeDiscoverEC2)
 	}
-	if uit.Spec.IssueType == "" {
+	if uit.GetSpec().IssueType == "" {
 		return trace.BadParameter("issue type is required")
 	}
+	if !slices.Contains(validTaskStates, uit.GetSpec().State) {
+		return trace.BadParameter("invalid task state, allowed values: %v", validTaskStates)
+	}
+	for instanceName, instanceIssue := range uit.Spec.DiscoverEc2.Instances {
+		if instanceName == "" {
+			return trace.BadParameter("instance name in discover_ec2.instances field is required")
+		}
+		if instanceIssue.AccountId == "" {
+			return trace.BadParameter("account id in discover_ec2.instances field is required")
+		}
+		if instanceIssue.Region == "" {
+			return trace.BadParameter("region in discover_ec2.instances field is required")
+		}
+		if instanceIssue.DiscoveryConfig == "" {
+			return trace.BadParameter("discovery config in discover_ec2.instances field is required")
+		}
+		if instanceIssue.DiscoveryGroup == "" {
+			return trace.BadParameter("discovery group in discover_ec2.instances field is required")
+		}
+	}
 
 	return nil
 }

lib/srv/discovery/discovery.go

@@ -36,11 +36,13 @@ import (
 	"github.com/gravitational/trace"
 	"github.com/jonboulle/clockwork"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/types/known/timestamppb"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 
 	"github.com/gravitational/teleport"
 	"github.com/gravitational/teleport/api/client/proto"
+	usertasksv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/usertasks/v1"
 	usageeventsv1 "github.com/gravitational/teleport/api/gen/proto/go/usageevents/v1"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/types/discoveryconfig"
@@ -329,6 +331,7 @@ type Server struct {
 
 	awsSyncStatus         awsSyncStatus
 	awsEC2ResourcesStatus awsResourcesStatus
+	awsEC2Tasks           awsEC2Tasks
 
 	// caRotationCh receives nodes that need to have their CAs rotated.
 	caRotationCh chan []types.Server
@@ -460,6 +463,7 @@ func (s *Server) initAWSWatchers(matchers []types.AWSMatcher) error {
 		server.WithTriggerFetchC(s.newDiscoveryConfigChangedSub()),
 		server.WithPreFetchHookFn(func() {
 			s.awsEC2ResourcesStatus.iterationStarted()
+			s.awsEC2Tasks.iterationStarted()
 		}),
 	)
 	if err != nil {
@@ -883,6 +887,22 @@ func (s *Server) heartbeatEICEInstance(instances *server.EC2Instances) {
 				discoveryConfig: instances.DiscoveryConfig,
 				integration:     instances.Integration,
 			}, 1)
+
+			s.awsEC2Tasks.addFailedEnrollment(
+				awsEC2FailedEnrollmentGroup{
+					integration: instances.Integration,
+					issueType:   types.AutoDiscoverEC2IssueEICEFailedToCreateNode,
+				},
+				ec2Instance.InstanceID,
+				&usertasksv1.DiscoverEC2Instance{
+					// TODO(marco): add instance name
+					AccountId:       instances.AccountID,
+					Region:          instances.Region,
+					DiscoveryConfig: instances.DiscoveryConfig,
+					DiscoveryGroup:  s.DiscoveryGroup,
+					SyncTime:        timestamppb.New(s.clock.Now()),
+				},
+			)
 			continue
 		}
 
@@ -925,6 +945,22 @@ func (s *Server) heartbeatEICEInstance(instances *server.EC2Instances) {
 				discoveryConfig: instances.DiscoveryConfig,
 				integration:     instances.Integration,
 			}, 1)
+
+			s.awsEC2Tasks.addFailedEnrollment(
+				awsEC2FailedEnrollmentGroup{
+					integration: instances.Integration,
+					issueType:   types.AutoDiscoverEC2IssueEICEFailedToCreateNode,
+				},
+				instanceID,
+				&usertasksv1.DiscoverEC2Instance{
+					// TODO(marco): add instance name
+					AccountId:       instances.AccountID,
+					Region:          instances.Region,
+					DiscoveryConfig: instances.DiscoveryConfig,
+					DiscoveryGroup:  s.DiscoveryGroup,
+					SyncTime:        timestamppb.New(s.clock.Now()),
+				},
+			)
 		}
 	})
 	if err != nil {
@@ -960,6 +996,24 @@ func (s *Server) handleEC2RemoteInstallation(instances *server.EC2Instances) err
 			discoveryConfig: instances.DiscoveryConfig,
 			integration:     instances.Integration,
 		}, len(req.Instances))
+
+		for _, instance := range req.Instances {
+			s.awsEC2Tasks.addFailedEnrollment(
+				awsEC2FailedEnrollmentGroup{
+					integration: instances.Integration,
+					issueType:   types.AutoDiscoverEC2IssueScriptSSMAgentNotRunning,
+				},
+				instance.InstanceID,
+				&usertasksv1.DiscoverEC2Instance{
+					// TODO(marco): add instance name
+					AccountId:       instances.AccountID,
+					Region:          instances.Region,
+					DiscoveryConfig: instances.DiscoveryConfig,
+					DiscoveryGroup:  s.DiscoveryGroup,
+					SyncTime:        timestamppb.New(s.clock.Now()),
+				},
+			)
+		}
 		return trace.Wrap(err)
 	}
 	return nil
@@ -1072,6 +1126,7 @@ func (s *Server) handleEC2Discovery() {
 			}
 
 			s.updateDiscoveryConfigStatus(instances.EC2.DiscoveryConfig)
+			s.upsertTasksForAWSEC2FailedEnrollments()
 		case <-s.ctx.Done():
 			s.ec2Watcher.Stop()
 			return

lib/srv/discovery/status.go

@@ -25,9 +25,12 @@ import (
 	"time"
 
 	"github.com/gravitational/trace"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	discoveryconfigv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/discoveryconfig/v1"
+	usertasksv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/usertasks/v1"
 	"github.com/gravitational/teleport/api/types/discoveryconfig"
+	"github.com/gravitational/teleport/api/types/usertasks"
 	libevents "github.com/gravitational/teleport/lib/events"
 	aws_sync "github.com/gravitational/teleport/lib/srv/discovery/fetchers/aws-sync"
 	"github.com/gravitational/teleport/lib/srv/server"
@@ -293,5 +296,94 @@ func (s *Server) ReportEC2SSMInstallationResult(ctx context.Context, result *ser
 
 	s.updateDiscoveryConfigStatus(result.DiscoveryConfig)
 
+	s.awsEC2Tasks.addFailedEnrollment(
+		awsEC2FailedEnrollmentGroup{
+			integration: result.IntegrationName,
+			// TODO(marco): create and use more consts like
+			// AutoDiscoverEC2IssueScriptSSMAgentNotRunning
+			issueType: result.SSMRunEvent.Code,
+		},
+		result.SSMRunEvent.InstanceID,
+		&usertasksv1.DiscoverEC2Instance{
+			// TODO(marco): add instance name
+			Region:          result.SSMRunEvent.Region,
+			InvocationUrl:   result.SSMRunEvent.InvocationURL,
+			DiscoveryConfig: result.DiscoveryConfig,
+			DiscoveryGroup:  s.DiscoveryGroup,
+			SyncTime:        timestamppb.New(result.SSMRunEvent.Time),
+		},
+	)
+
 	return nil
 }
+
+// awsEC2FailedEnrollments ...
+type awsEC2Tasks struct {
+	mu sync.RWMutex
+	// instancesIssue maps the DiscoveryConfig name and integration to a summary of discovered/enrolled resources.
+	instancesIssue map[awsEC2FailedEnrollmentGroup]map[string]*usertasksv1.DiscoverEC2Instance
+	groupPending   map[awsEC2FailedEnrollmentGroup]struct{}
+}
+
+// awsEC2FailedEnrollmentGroup ...
+type awsEC2FailedEnrollmentGroup struct {
+	integration string
+	issueType   string
+}
+
+func (d *awsEC2Tasks) iterationStarted() {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	d.instancesIssue = make(map[awsEC2FailedEnrollmentGroup]map[string]*usertasksv1.DiscoverEC2Instance)
+	d.groupPending = make(map[awsEC2FailedEnrollmentGroup]struct{})
+}
+
+func (d *awsEC2Tasks) addFailedEnrollment(g awsEC2FailedEnrollmentGroup, instanceID string, instance *usertasksv1.DiscoverEC2Instance) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.instancesIssue == nil {
+		d.instancesIssue = make(map[awsEC2FailedEnrollmentGroup]map[string]*usertasksv1.DiscoverEC2Instance)
+	}
+	if _, ok := d.instancesIssue[g]; !ok {
+		d.instancesIssue[g] = make(map[string]*usertasksv1.DiscoverEC2Instance)
+	}
+	d.instancesIssue[g][instanceID] = instance
+
+	if d.groupPending == nil {
+		d.groupPending = make(map[awsEC2FailedEnrollmentGroup]struct{})
+	}
+	d.groupPending[g] = struct{}{}
+}
+
+func (s *Server) upsertTasksForAWSEC2FailedEnrollments() {
+	s.awsEC2Tasks.mu.Lock()
+	defer s.awsEC2Tasks.mu.Unlock()
+	for g, instances := range s.awsEC2Tasks.instancesIssue {
+		if _, pending := s.awsEC2Tasks.groupPending[g]; !pending {
+			continue
+		}
+
+		task, err := usertasks.NewUserTask(
+			"name", // TODO(marco): ensure names are uniq so that task gets replaced
+			&usertasksv1.UserTaskSpec{
+				Integration: g.integration,
+				TaskType:    usertasks.TaskTypeDiscoverEC2,
+				IssueType:   g.issueType,
+				State:       usertasks.TaskStateOpen,
+				DiscoverEc2: &usertasksv1.DiscoverEC2{
+					Instances: instances,
+				},
+			},
+		)
+		if err != nil {
+			s.Log.WithError(err).Warn("failed to create user task for failed to enroll instance")
+			continue
+		}
+		if _, err := s.AccessPoint.UpsertUserTask(s.ctx, task); err != nil {
+			s.Log.WithError(err).Warn("failed to create user task for failed to enroll instances")
+			continue
+		}
+		delete(s.awsEC2Tasks.groupPending, g)
+	}
+}