From 1afa13cf357d6d47d3389b299e0cc4517803c735 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Tue, 15 Dec 2020 01:29:41 +0100 Subject: [PATCH] Launch Template cleanup retries and add tests into CI (#326) * Mark ec2_launch_template tests as supported * Update launch template test to use permitted harmless managed policy * Make sure we delete the instance profile * Cleanup imports * use ansible_dict_to_boto3_tag_list * Make sure retries are enabled * Update role names in line with CI policy * Lookup AMI name rather than hardcoding IDs * Add changelog fragment --- .../fragments/326-launch_template_retry.yml | 2 ++ plugins/modules/ec2_launch_template.py | 29 +++++++++-------- .../targets/ec2_launch_template/aliases | 2 +- .../ec2_launch_template/defaults/main.yml | 19 ++--------- .../ec2_launch_template/tasks/cpu_options.yml | 2 +- .../tasks/iam_instance_role.yml | 32 +++++++++---------- .../roles/ec2_launch_template/tasks/main.yml | 10 ++++++ .../tasks/tags_and_vpc_settings.yml | 10 +++--- .../ec2_launch_template/tasks/versions.yml | 6 ++-- 9 files changed, 55 insertions(+), 57 deletions(-) create mode 100644 changelogs/fragments/326-launch_template_retry.yml diff --git a/changelogs/fragments/326-launch_template_retry.yml b/changelogs/fragments/326-launch_template_retry.yml new file mode 100644 index 00000000000..ab7506d8847 --- /dev/null +++ b/changelogs/fragments/326-launch_template_retry.yml @@ -0,0 +1,2 @@ +minor_changes: +- ec2_launch_template - Add retries on common AWS failures (https://github.com/ansible-collections/community.aws/pull/326). diff --git a/plugins/modules/ec2_launch_template.py b/plugins/modules/ec2_launch_template.py index 4553a8e794d..4a35812cfb4 100644 --- a/plugins/modules/ec2_launch_template.py +++ b/plugins/modules/ec2_launch_template.py @@ -367,13 +367,14 @@ from uuid import uuid4 from ansible.module_utils._text import to_text -from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule, is_boto3_error_code, get_boto3_client_method_parameters -from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict, snake_dict_to_camel_dict -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (ansible_dict_to_boto3_tag_list, - AWSRetry, - boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_tag_list, - ) +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict +from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict + +from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict try: from botocore.exceptions import ClientError, BotoCoreError, WaiterError @@ -399,9 +400,9 @@ def existing_templates(module): matches = None try: if module.params.get('template_id'): - matches = ec2.describe_launch_templates(LaunchTemplateIds=[module.params.get('template_id')]) + matches = ec2.describe_launch_templates(LaunchTemplateIds=[module.params.get('template_id')], aws_retry=True) elif module.params.get('template_name'): - matches = ec2.describe_launch_templates(LaunchTemplateNames=[module.params.get('template_name')]) + matches = ec2.describe_launch_templates(LaunchTemplateNames=[module.params.get('template_name')], aws_retry=True) except is_boto3_error_code('InvalidLaunchTemplateName.NotFoundException') as e: # no named template was found, return nothing/empty versions return None, [] @@ -418,20 +419,18 @@ def existing_templates(module): template = matches['LaunchTemplates'][0] template_id, template_version, template_default = template['LaunchTemplateId'], template['LatestVersionNumber'], template['DefaultVersionNumber'] try: - return template, ec2.describe_launch_template_versions(LaunchTemplateId=template_id)['LaunchTemplateVersions'] + return template, ec2.describe_launch_template_versions(LaunchTemplateId=template_id, aws_retry=True)['LaunchTemplateVersions'] except (ClientError, BotoCoreError, WaiterError) as e: module.fail_json_aws(e, msg='Could not find launch template versions for {0} (ID: {1}).'.format(template['LaunchTemplateName'], template_id)) def params_to_launch_data(module, template_params): if template_params.get('tags'): + tag_list = ansible_dict_to_boto3_tag_list(template_params.get('tags')) template_params['tag_specifications'] = [ { 'resource_type': r_type, - 'tags': [ - {'Key': k, 'Value': v} for k, v - in template_params['tags'].items() - ] + 'tags': tag_list } for r_type in ('instance', 'volume') ] @@ -456,6 +455,7 @@ def delete_template(module): v_resp = ec2.delete_launch_template_versions( LaunchTemplateId=template['LaunchTemplateId'], Versions=non_default_versions, + aws_retry=True, ) if v_resp['UnsuccessfullyDeletedLaunchTemplateVersions']: module.warn('Failed to delete template versions {0} on launch template {1}'.format( @@ -468,6 +468,7 @@ def delete_template(module): try: resp = ec2.delete_launch_template( LaunchTemplateId=template['LaunchTemplateId'], + aws_retry=True, ) except (ClientError, BotoCoreError) as e: module.fail_json_aws(e, msg="Could not delete launch template {0}".format(template['LaunchTemplateId'])) diff --git a/tests/integration/targets/ec2_launch_template/aliases b/tests/integration/targets/ec2_launch_template/aliases index 56927195182..157ce0c9d4c 100644 --- a/tests/integration/targets/ec2_launch_template/aliases +++ b/tests/integration/targets/ec2_launch_template/aliases @@ -1,2 +1,2 @@ cloud/aws -unsupported +shippable/aws/group3 diff --git a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/defaults/main.yml b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/defaults/main.yml index 9651b91642a..19eb792b91d 100644 --- a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/defaults/main.yml +++ b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/defaults/main.yml @@ -1,18 +1,3 @@ --- -resource_prefix: ansible-test-default-group -ec2_ami_image: - # https://wiki.centos.org/Cloud/AWS collected 2018-01-10 - ap-northeast-1: ami-571e3c30 - ap-northeast-2: ami-97cb19f9 - ap-south-1: ami-11f0837e - ap-southeast-1: ami-30318f53 - ap-southeast-2: ami-24959b47 - ca-central-1: ami-daeb57be - eu-central-1: ami-7cbc6e13 - eu-west-1: ami-0d063c6b - eu-west-2: ami-c22236a6 - sa-east-1: ami-864f2dea - us-east-1: ami-ae7bfdb8 - us-east-2: ami-9cbf9bf9 - us-west-1: ami-7c280d1c - us-west-2: ami-0c2aba6c +ec2_ami_name: amzn2-ami-hvm-2.*-x86_64-gp2 +test_role_name: ansible-test-{{ resource_prefix }} diff --git a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/cpu_options.yml b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/cpu_options.yml index 8d610a2ea75..b98c3dceaa8 100644 --- a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/cpu_options.yml +++ b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/cpu_options.yml @@ -11,7 +11,7 @@ - name: create c4.large instance with cpu_options ec2_launch_template: name: "{{ resource_prefix }}-c4large-1-threads-per-core" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" tags: TestId: "{{ resource_prefix }}" instance_type: c4.large diff --git a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/iam_instance_role.yml b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/iam_instance_role.yml index 5e9b7f563de..a303cc628cc 100644 --- a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/iam_instance_role.yml +++ b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/iam_instance_role.yml @@ -1,30 +1,30 @@ - block: - name: Create IAM role for test iam_role: - name: "{{ resource_prefix }}-test-policy" + name: "{{ test_role_name }}-1" assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" state: present create_instance_profile: yes managed_policy: - - AmazonS3ReadOnlyAccess + - AWSDenyAll register: iam_role - name: Create second IAM role for test iam_role: - name: "{{ resource_prefix }}-test-policy-2" + name: "{{ test_role_name }}-2" assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" state: present create_instance_profile: yes managed_policy: - - AmazonS3ReadOnlyAccess + - AWSDenyAll register: iam_role_2 - name: Make instance with an instance_role ec2_launch_template: name: "{{ resource_prefix }}-test-instance-role" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" instance_type: t2.micro - iam_instance_profile: "{{ resource_prefix }}-test-policy" + iam_instance_profile: "{{ test_role_name }}-1" register: template_with_role - assert: @@ -34,9 +34,9 @@ - name: Create template again, with no change to instance_role ec2_launch_template: name: "{{ resource_prefix }}-test-instance-role" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" instance_type: t2.micro - iam_instance_profile: "{{ resource_prefix }}-test-policy" + iam_instance_profile: "{{ test_role_name }}-1" register: template_with_role - assert: @@ -47,9 +47,9 @@ - name: Update instance with new instance_role ec2_launch_template: name: "{{ resource_prefix }}-test-instance-role" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" instance_type: t2.micro - iam_instance_profile: "{{ resource_prefix }}-test-policy-2" + iam_instance_profile: "{{ test_role_name }}-2" register: template_with_updated_role - assert: @@ -63,9 +63,9 @@ - name: Re-set with same new instance_role ec2_launch_template: name: "{{ resource_prefix }}-test-instance-role" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" instance_type: t2.micro - iam_instance_profile: "{{ resource_prefix }}-test-policy-2" + iam_instance_profile: "{{ test_role_name }}-2" register: template_with_updated_role - assert: @@ -84,20 +84,20 @@ retries: 10 - name: Delete IAM role for test iam_role: - name: "{{ resource_prefix }}-test-policy" + name: "{{ test_role_name }}-1" assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" state: absent - create_instance_profile: yes + delete_instance_profile: yes register: iam_removed until: iam_removed is not failed ignore_errors: yes retries: 10 - name: Delete IAM role for test iam_role: - name: "{{ resource_prefix }}-test-policy-2" + name: "{{ test_role_name }}-2" assume_role_policy_document: "{{ lookup('file','assume-role-policy.json') }}" state: absent - create_instance_profile: yes + delete_instance_profile: yes register: iam_2_removed until: iam_2_removed is not failed ignore_errors: yes diff --git a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/main.yml b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/main.yml index 4976da276e0..d3b24d64181 100644 --- a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/main.yml +++ b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/main.yml @@ -15,6 +15,16 @@ security_token: "{{ security_token | default(omit) }}" region: "{{ aws_region }}" block: + + - name: Find AMI to use + ec2_ami_info: + owners: 'amazon' + filters: + name: '{{ ec2_ami_name }}' + register: ec2_amis + - set_fact: + ec2_ami_image: '{{ ec2_amis.images[0].image_id }}' + - include_tasks: cpu_options.yml - include_tasks: iam_instance_role.yml - include_tasks: versions.yml diff --git a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/tags_and_vpc_settings.yml b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/tags_and_vpc_settings.yml index 7da7f770af6..aab9e61b98f 100644 --- a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/tags_and_vpc_settings.yml +++ b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/tags_and_vpc_settings.yml @@ -49,7 +49,7 @@ - name: Make instance in the testing subnet created in the test VPC ec2_instance: name: "{{ resource_prefix }}-test-basic-vpc-create" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" user_data: | #cloud-config package_upgrade: true @@ -71,7 +71,7 @@ - name: Try to re-make the instance, hopefully this shows changed=False ec2_instance: name: "{{ resource_prefix }}-test-basic-vpc-create" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" user_data: | #cloud-config package_upgrade: true @@ -96,7 +96,7 @@ - name: Alter it by adding tags ec2_instance: name: "{{ resource_prefix }}-test-basic-vpc-create" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" tags: TestId: "{{ resource_prefix }}" Another: thing @@ -113,11 +113,11 @@ that: - check_tags.instances[0].tags.Another == 'thing' - check_tags.instances[0].tags.Something == 'else' - + - name: Purge a tag ec2_instance: name: "{{ resource_prefix }}-test-basic-vpc-create" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" purge_tags: true tags: TestId: "{{ resource_prefix }}" diff --git a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/versions.yml b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/versions.yml index 9035467a60f..d188d9c2e51 100644 --- a/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/versions.yml +++ b/tests/integration/targets/ec2_launch_template/playbooks/roles/ec2_launch_template/tasks/versions.yml @@ -2,7 +2,7 @@ - name: create simple instance template ec2_launch_template: name: "{{ resource_prefix }}-simple" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" tags: TestId: "{{ resource_prefix }}" instance_type: c4.large @@ -20,7 +20,7 @@ ec2_launch_template: name: "{{ resource_prefix }}-simple" default_version: 1 - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" tags: TestId: "{{ resource_prefix }}" instance_type: m5.large @@ -37,7 +37,7 @@ - name: update simple instance template ec2_launch_template: name: "{{ resource_prefix }}-simple" - image_id: "{{ ec2_ami_image[aws_region] }}" + image_id: "{{ ec2_ami_image }}" tags: TestId: "{{ resource_prefix }}" instance_type: t3.medium