diff --git a/api/types/autodiscover.go b/api/types/autodiscover.go new file mode 100644 index 0000000000000..733f266290250 --- /dev/null +++ b/api/types/autodiscover.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +// Auto Discover EC2 issues identifiers which are used to provide better error messages to the user. +const ( + // AutoDiscoverEC2IssueEICEFailedToCreateNode is used when the EICE flow fails to auto enroll an EC2 instance + // as an EICE node. + AutoDiscoverEC2IssueEICEFailedToCreateNode = "ec2-eice-creation" + // AutoDiscoverEC2IssueScriptSSMAgentNotRunning is used when the SSM Agent is not present in the instance. + // This can also happen when the SSM was not able to connect to AWS Systems Manager. + AutoDiscoverEC2IssueScriptSSMAgentNotRunning = "ec2-ssm-agent-not-running" +) diff --git a/api/types/usertasks/object.go b/api/types/usertasks/object.go index 978d10c3d1121..15c7ac0772c20 100644 --- a/api/types/usertasks/object.go +++ b/api/types/usertasks/object.go @@ -19,6 +19,8 @@ package usertasks import ( + "slices" + "github.com/gravitational/trace" headerv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/header/v1" @@ -45,6 +47,15 @@ func NewUserTask(name string, spec *usertasksv1.UserTaskSpec) (*usertasksv1.User return cj, nil } +const ( + // TaskStateOpen identifies an issue with an instance that is not yet resolved. + TaskStateOpen = "OPEN" + // TaskStateResolved identifies an issue with an instance that is resolved. + TaskStateResolved = "RESOLVED" +) + +var validTaskStates = []string{TaskStateOpen, TaskStateResolved} + const ( // TaskTypeDiscoverEC2 identifies a User Tasks that is created // when an auto-enrollment of an EC2 instance fails. @@ -84,12 +95,32 @@ func ValidateUserTask(uit *usertasksv1.UserTask) error { } func validateDiscoverEC2TaskType(uit *usertasksv1.UserTask) error { - if uit.Spec.DiscoverEc2 == nil { + if uit.GetSpec().DiscoverEc2 == nil { return trace.BadParameter("%s requires the discover_ec2 field", TaskTypeDiscoverEC2) } - if uit.Spec.IssueType == "" { + if uit.GetSpec().IssueType == "" { return trace.BadParameter("issue type is required") } + if !slices.Contains(validTaskStates, uit.GetSpec().State) { + return trace.BadParameter("invalid task state, allowed values: %v", validTaskStates) + } + for instanceName, instanceIssue := range uit.Spec.DiscoverEc2.Instances { + if instanceName == "" { + return trace.BadParameter("instance name in discover_ec2.instances field is required") + } + if instanceIssue.AccountId == "" { + return trace.BadParameter("account id in discover_ec2.instances field is required") + } + if instanceIssue.Region == "" { + return trace.BadParameter("region in discover_ec2.instances field is required") + } + if instanceIssue.DiscoveryConfig == "" { + return trace.BadParameter("discovery config in discover_ec2.instances field is required") + } + if instanceIssue.DiscoveryGroup == "" { + return trace.BadParameter("discovery group in discover_ec2.instances field is required") + } + } return nil } diff --git a/lib/srv/discovery/discovery.go b/lib/srv/discovery/discovery.go index 92e46ad3f3abe..5502ccca359ab 100644 --- a/lib/srv/discovery/discovery.go +++ b/lib/srv/discovery/discovery.go @@ -36,11 +36,13 @@ import ( "github.com/gravitational/trace" "github.com/jonboulle/clockwork" "github.com/sirupsen/logrus" + "google.golang.org/protobuf/types/known/timestamppb" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "github.com/gravitational/teleport" "github.com/gravitational/teleport/api/client/proto" + usertasksv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/usertasks/v1" usageeventsv1 "github.com/gravitational/teleport/api/gen/proto/go/usageevents/v1" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/api/types/discoveryconfig" @@ -329,6 +331,7 @@ type Server struct { awsSyncStatus awsSyncStatus awsEC2ResourcesStatus awsResourcesStatus + awsEC2Tasks awsEC2Tasks // caRotationCh receives nodes that need to have their CAs rotated. caRotationCh chan []types.Server @@ -460,6 +463,7 @@ func (s *Server) initAWSWatchers(matchers []types.AWSMatcher) error { server.WithTriggerFetchC(s.newDiscoveryConfigChangedSub()), server.WithPreFetchHookFn(func() { s.awsEC2ResourcesStatus.iterationStarted() + s.awsEC2Tasks.iterationStarted() }), ) if err != nil { @@ -883,6 +887,22 @@ func (s *Server) heartbeatEICEInstance(instances *server.EC2Instances) { discoveryConfig: instances.DiscoveryConfig, integration: instances.Integration, }, 1) + + s.awsEC2Tasks.addFailedEnrollment( + awsEC2FailedEnrollmentGroup{ + integration: instances.Integration, + issueType: types.AutoDiscoverEC2IssueEICEFailedToCreateNode, + }, + ec2Instance.InstanceID, + &usertasksv1.DiscoverEC2Instance{ + // TODO(marco): add instance name + AccountId: instances.AccountID, + Region: instances.Region, + DiscoveryConfig: instances.DiscoveryConfig, + DiscoveryGroup: s.DiscoveryGroup, + SyncTime: timestamppb.New(s.clock.Now()), + }, + ) continue } @@ -925,6 +945,22 @@ func (s *Server) heartbeatEICEInstance(instances *server.EC2Instances) { discoveryConfig: instances.DiscoveryConfig, integration: instances.Integration, }, 1) + + s.awsEC2Tasks.addFailedEnrollment( + awsEC2FailedEnrollmentGroup{ + integration: instances.Integration, + issueType: types.AutoDiscoverEC2IssueEICEFailedToCreateNode, + }, + instanceID, + &usertasksv1.DiscoverEC2Instance{ + // TODO(marco): add instance name + AccountId: instances.AccountID, + Region: instances.Region, + DiscoveryConfig: instances.DiscoveryConfig, + DiscoveryGroup: s.DiscoveryGroup, + SyncTime: timestamppb.New(s.clock.Now()), + }, + ) } }) if err != nil { @@ -960,6 +996,24 @@ func (s *Server) handleEC2RemoteInstallation(instances *server.EC2Instances) err discoveryConfig: instances.DiscoveryConfig, integration: instances.Integration, }, len(req.Instances)) + + for _, instance := range req.Instances { + s.awsEC2Tasks.addFailedEnrollment( + awsEC2FailedEnrollmentGroup{ + integration: instances.Integration, + issueType: types.AutoDiscoverEC2IssueScriptSSMAgentNotRunning, + }, + instance.InstanceID, + &usertasksv1.DiscoverEC2Instance{ + // TODO(marco): add instance name + AccountId: instances.AccountID, + Region: instances.Region, + DiscoveryConfig: instances.DiscoveryConfig, + DiscoveryGroup: s.DiscoveryGroup, + SyncTime: timestamppb.New(s.clock.Now()), + }, + ) + } return trace.Wrap(err) } return nil @@ -1072,6 +1126,7 @@ func (s *Server) handleEC2Discovery() { } s.updateDiscoveryConfigStatus(instances.EC2.DiscoveryConfig) + s.upsertTasksForAWSEC2FailedEnrollments() case <-s.ctx.Done(): s.ec2Watcher.Stop() return diff --git a/lib/srv/discovery/status.go b/lib/srv/discovery/status.go index a7194d87372b5..cedab58f539ee 100644 --- a/lib/srv/discovery/status.go +++ b/lib/srv/discovery/status.go @@ -25,9 +25,12 @@ import ( "time" "github.com/gravitational/trace" + "google.golang.org/protobuf/types/known/timestamppb" discoveryconfigv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/discoveryconfig/v1" + usertasksv1 "github.com/gravitational/teleport/api/gen/proto/go/teleport/usertasks/v1" "github.com/gravitational/teleport/api/types/discoveryconfig" + "github.com/gravitational/teleport/api/types/usertasks" libevents "github.com/gravitational/teleport/lib/events" aws_sync "github.com/gravitational/teleport/lib/srv/discovery/fetchers/aws-sync" "github.com/gravitational/teleport/lib/srv/server" @@ -293,5 +296,94 @@ func (s *Server) ReportEC2SSMInstallationResult(ctx context.Context, result *ser s.updateDiscoveryConfigStatus(result.DiscoveryConfig) + s.awsEC2Tasks.addFailedEnrollment( + awsEC2FailedEnrollmentGroup{ + integration: result.IntegrationName, + // TODO(marco): create and use more consts like + // AutoDiscoverEC2IssueScriptSSMAgentNotRunning + issueType: result.SSMRunEvent.Code, + }, + result.SSMRunEvent.InstanceID, + &usertasksv1.DiscoverEC2Instance{ + // TODO(marco): add instance name + Region: result.SSMRunEvent.Region, + InvocationUrl: result.SSMRunEvent.InvocationURL, + DiscoveryConfig: result.DiscoveryConfig, + DiscoveryGroup: s.DiscoveryGroup, + SyncTime: timestamppb.New(result.SSMRunEvent.Time), + }, + ) + return nil } + +// awsEC2FailedEnrollments ... +type awsEC2Tasks struct { + mu sync.RWMutex + // instancesIssue maps the DiscoveryConfig name and integration to a summary of discovered/enrolled resources. + instancesIssue map[awsEC2FailedEnrollmentGroup]map[string]*usertasksv1.DiscoverEC2Instance + groupPending map[awsEC2FailedEnrollmentGroup]struct{} +} + +// awsEC2FailedEnrollmentGroup ... +type awsEC2FailedEnrollmentGroup struct { + integration string + issueType string +} + +func (d *awsEC2Tasks) iterationStarted() { + d.mu.Lock() + defer d.mu.Unlock() + + d.instancesIssue = make(map[awsEC2FailedEnrollmentGroup]map[string]*usertasksv1.DiscoverEC2Instance) + d.groupPending = make(map[awsEC2FailedEnrollmentGroup]struct{}) +} + +func (d *awsEC2Tasks) addFailedEnrollment(g awsEC2FailedEnrollmentGroup, instanceID string, instance *usertasksv1.DiscoverEC2Instance) { + d.mu.Lock() + defer d.mu.Unlock() + if d.instancesIssue == nil { + d.instancesIssue = make(map[awsEC2FailedEnrollmentGroup]map[string]*usertasksv1.DiscoverEC2Instance) + } + if _, ok := d.instancesIssue[g]; !ok { + d.instancesIssue[g] = make(map[string]*usertasksv1.DiscoverEC2Instance) + } + d.instancesIssue[g][instanceID] = instance + + if d.groupPending == nil { + d.groupPending = make(map[awsEC2FailedEnrollmentGroup]struct{}) + } + d.groupPending[g] = struct{}{} +} + +func (s *Server) upsertTasksForAWSEC2FailedEnrollments() { + s.awsEC2Tasks.mu.Lock() + defer s.awsEC2Tasks.mu.Unlock() + for g, instances := range s.awsEC2Tasks.instancesIssue { + if _, pending := s.awsEC2Tasks.groupPending[g]; !pending { + continue + } + + task, err := usertasks.NewUserTask( + "name", // TODO(marco): ensure names are uniq so that task gets replaced + &usertasksv1.UserTaskSpec{ + Integration: g.integration, + TaskType: usertasks.TaskTypeDiscoverEC2, + IssueType: g.issueType, + State: usertasks.TaskStateOpen, + DiscoverEc2: &usertasksv1.DiscoverEC2{ + Instances: instances, + }, + }, + ) + if err != nil { + s.Log.WithError(err).Warn("failed to create user task for failed to enroll instance") + continue + } + if _, err := s.AccessPoint.UpsertUserTask(s.ctx, task); err != nil { + s.Log.WithError(err).Warn("failed to create user task for failed to enroll instances") + continue + } + delete(s.awsEC2Tasks.groupPending, g) + } +}