Pull request md

.bumpversion.cfg

@@ -1,7 +1,7 @@
 [bumpversion]
 commit = False
 tag = True
-current_version = 1.1.74
+current_version = 1.1.146
 tag_name = v{current_version}
 message = GitHub Actions Build {current_version}
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,29 @@
+## Type of change
+
+- [ ] Refactor
+- [ ] New feature
+- [ ] Bug fix
+- [ ] Optimization
+- [ ] Documentation Update
+
+## Description
+
+<!--- Describe your changes in detail -->
+
+## Related Tickets & Documents
+
+- Related Issue #
+- Closes #
+
+## Checklist before requesting a review
+
+- [ ] I have performed a self-review of my code.
+- [ ] If it is a core feature, I have added thorough tests.
+
+## Testing
+- Does below tests are passed
+  - [ ] UnitTest
+  - [ ] IntegrationTest
+- Please describe the System Under Test.
+- Please provide detailed steps to perform tests related to this code change.
+- How were the fix/results from this change verified? Please provide relevant screenshots or results.

.github/workflows/Build.yml

@@ -7,6 +7,10 @@ on:
   push:
     branches: [ main ]
 
+concurrency:
+  group: merge-queue
+  cancel-in-progress: false
+
 jobs:
   unittest:
     name: unittest
@@ -23,6 +27,7 @@ jobs:
       - name: Install dependencies
         run: |
           # ldap requirements
+          sudo apt update -y
           sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
           python -m pip install --upgrade pip
           pip install flake8 pytest pytest-cov
@@ -131,6 +136,7 @@ jobs:
       - name: Install dependencies
         run: |
           # ldap requirements
+          sudo apt update -y
           sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
           python -m pip install --upgrade pip
           pip install flake8 pytest pytest-cov
@@ -148,6 +154,13 @@ jobs:
           aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.REGION }}
+      - name: Set GCP credentials for pytest
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS_CONTENTS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+          RUNNER_PATH: ${{ secrets.RUNNER_PATH }}
+        run: |
+          echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENTS" > "$RUNNER_PATH/gcp_service.json"
+          echo "GOOGLE_APPLICATION_CREDENTIALS=$RUNNER_PATH/gcp_service.json" >> "$GITHUB_ENV"
       - name: 📃 Integration tests with pytest
         env:
           BUCKET: ${{ secrets.BUCKET }}
@@ -158,6 +171,8 @@ jobs:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          GCP_DATABASE_NAME: ${{ secrets.GCP_DATABASE_NAME }}
+          GCP_DATABASE_TABLE_NAME: ${{ secrets.GCP_DATABASE_TABLE_NAME }}
         run: |
           pytest -v tests/integration --cov=cloud_governqance --cov-report=term-missing
           coverage run -m pytest -v tests/integration
@@ -252,6 +267,7 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
         run: |
           # ldap requirements
+          sudo apt update -y
           sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
           echo '⌛ Wait till package will be updated in PyPI'
           # Verfiy and wait till latest cloud-governance version will be updated in Pypi (timeout 900 seconds)

.github/workflows/PR.yml

@@ -5,15 +5,24 @@ name: PR
 
 on:
   pull_request_target:
+    types: [labeled, synchronize]
     branches: [ main ]
 
 jobs:
+  approve: # First step
+    runs-on: ubuntu-latest
+    steps:
+      - name: Approve
+        run: echo For security reasons, all pull requests need to be approved first before running any automated CI.
   unittest:
     name: unittest
     runs-on: ubuntu-latest
+    needs: [approve]
     strategy:
       matrix:
         python-version: [ '3.8', '3.9', '3.10' ]
+    # minimize potential vulnerabilities
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
     steps:
     - uses: actions/checkout@v3
       with:
@@ -25,6 +34,7 @@ jobs:
     - name: Install dependencies
       run: |
         # ldap requirements
+        sudo apt update -y
         sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
         python -m pip install --upgrade pip
         pip install flake8 pytest pytest-cov
@@ -51,7 +61,7 @@ jobs:
 
   terraform_apply:
     name: terraform_apply
-    needs: [unittest]
+    needs: [unittest, approve]
     runs-on: ubuntu-latest
     outputs:
       INSTANCE_ID: ${{ steps.terraform_instance_id.outputs.INSTANCE_ID }}
@@ -99,7 +109,7 @@ jobs:
 
   integration:
     name: integration
-    needs: [ unittest, terraform_apply ]
+    needs: [ unittest, terraform_apply, approve ]
     runs-on: ubuntu-latest
     strategy:
       max-parallel: 1
@@ -129,6 +139,7 @@ jobs:
       - name: Install dependencies
         run: |
           # ldap requirements
+          sudo apt update -y
           sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
           python -m pip install --upgrade pip
           pip install flake8 pytest pytest-cov
@@ -146,6 +157,13 @@ jobs:
           aws-access-key-id: ${{ secrets.ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.REGION }}
+      - name: Set GCP credentials for pytest
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS_CONTENTS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+          RUNNER_PATH: ${{ secrets.RUNNER_PATH }}
+        run: |
+          echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENTS" > "$RUNNER_PATH/gcp_service.json"
+          echo "GOOGLE_APPLICATION_CREDENTIALS=$RUNNER_PATH/gcp_service.json" >> "$GITHUB_ENV"
       - name: 📃 Integration tests with pytest
         env:
           BUCKET: ${{ secrets.BUCKET }}
@@ -156,12 +174,14 @@ jobs:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          GCP_DATABASE_NAME: ${{ secrets.GCP_DATABASE_NAME }}
+          GCP_DATABASE_TABLE_NAME: ${{ secrets.GCP_DATABASE_TABLE_NAME }}
         run: |
           python -m pytest -v tests/integration
 
   terraform_destroy:
     name: terraform_destroy
-    needs: [unittest, terraform_apply, integration]
+    needs: [unittest, terraform_apply, integration, approve]
     if: success() || failure()
     runs-on: ubuntu-latest
     steps:
@@ -203,7 +223,7 @@ jobs:
 
   e2e:
     name: e2e
-    needs: [ unittest, terraform_apply, integration  ]
+    needs: [ unittest, terraform_apply, integration, approve  ]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -219,6 +239,7 @@ jobs:
       - name: Install dependencies
         run: |
           # ldap requirements
+          sudo apt update -y
           sudo apt-get install build-essential python3-dev libldap2-dev libsasl2-dev vim -y
           python -m pip install --upgrade pip
           pip install flake8 pytest pytest-cov

.gitignore

@@ -216,3 +216,4 @@ empty_test_environment_variables.py
 /cloud_governance/main/.env
 /cloud_governance/main/.test_env
 /cloud_governance/policy/send_mail.py
+cloudsensei/.env.txt

MANIFEST.in

@@ -1,5 +1,3 @@
 include cloud_governance/policy/*.yml
 include iam/cloud/azure/CloudGovernanceCostManagement.json
-
-
-
+include cloud_governance/common/mails/templates/cro_request_for_manager_approval.j2

README.md

@@ -35,7 +35,7 @@ This tool support the following policies:
 * [s3_inactive](cloud_governance/policy/aws/s3_inactive.py): Get the inactive/empty buckets and delete them after 7 days.
 * [empty_roles](cloud_governance/policy/aws/empty_roles.py): Get empty roles and delete it after 7 days.
 * [zombie_snapshots](cloud_governance/policy/aws/zombie_snapshots.py): Get the zombie snapshots and delete it after 7 days.
-* [nat_gateway_unused](cloud_governance/policy/aws/nat_gateway_unused.py): Get the unused nat gateways and deletes it after 7 days.
+* [nat_gateway_unused](cloud_governance/policy/aws/unused_nat_gateway.py): Get the unused nat gateways and deletes it after 7 days.
 * gitleaks: scan Github repository git leak (security scan)  
 * [cost_over_usage](cloud_governance/policy/aws/cost_over_usage.py): send mail to aws user if over usage cost
 

aws_lambda_functions/CloudResourceOrchestration/lambda_function.py

@@ -0,0 +1,101 @@
+import json
+import boto3
+import jira
+
+ssm_client = boto3.client('ssm', region_name='us-east-1')
+APPROVED = 'APPROVED'
+REJECT = 'REJECT'
+REFINEMENT = '61'
+CLOSED = '41'
+JIRA_TOKEN = 'JIRA_TOKEN'
+JIRA_PROJECT = 'JIRA_PROJECT'
+JIRA_API_SERVER = 'JIRA_API_SERVER'
+CRO_ADMINS = ['athiruma@redhat.com', 'natashba@redhat.com', 'ebattat@redhat.com']
+
+
+def get_receive_mail_details(event_data):
+    """
+    This method returns the received mail data
+    :param event_data:
+    :return:
+    # """
+    records = event_data.get('Records')
+    mail_details = []
+    if records:
+        for record in records:
+            if record.get('eventSource') == 'aws:ses':
+                ses_data = record.get('ses')
+                common_headers = ses_data.get('mail', {}).get('commonHeaders', {})
+                if common_headers:
+                    mail_details.append({
+                        'from': ses_data.get('mail', {}).get('source'),
+                        'to': common_headers.get('to'),
+                        'subject': common_headers.get('subject')
+                    })
+    return mail_details
+
+
+def lambda_handler(event, context):
+    """
+    This lambda function is to approve the user budget by Email Receiving
+    :param event:
+    :param context:
+    :return:
+    """
+    try:
+        parameters = ssm_client.get_parameters(Names=[JIRA_TOKEN, JIRA_PROJECT, JIRA_API_SERVER], WithDecryption=True)['Parameters']
+        if parameters:
+            output_parameters = {}
+            for parameter in parameters:
+                output_parameters[parameter.get('Name')] = parameter.get('Value')
+            jira_auth_token = output_parameters.get(JIRA_TOKEN)
+            jira_server_api = output_parameters.get(JIRA_API_SERVER)
+            jira_project = output_parameters.get(JIRA_PROJECT)
+            jira_conn = jira.JIRA(server=jira_server_api, token_auth=jira_auth_token)
+            mail_results = get_receive_mail_details(event)
+            for mail_result in mail_results:
+                action, ticket = mail_result.get('subject').split(';')
+                manager_mail = mail_result.get('from')
+                ticket_id = f'{jira_project}-{ticket}'
+                issue = jira_conn.issue(id=ticket_id)
+                jira_description = jira_conn.issue(ticket_id).fields.description
+                fields = {}
+                for filed_value in jira_description.split('\n'):
+                    if filed_value:
+                        if ':' in filed_value:
+                            key, value = filed_value.strip().split(':', 1)
+                            fields[key.strip()] = value.strip()
+                CRO_ADMINS.append(fields.get('ManagerApprovalAddress'))
+                if manager_mail in CRO_ADMINS:
+                    jira_description += f'\nApprovedManager: {mail_result.get("from")}\n'
+                    if action.upper() == APPROVED:
+                        issue.update(description=jira_description, comment=f'From: {manager_mail}\nApproved\nPlease refer to your manager, in case any issues')
+                        jira_conn.transition_issue(issue=ticket_id, transition=REFINEMENT)
+                        return {
+                            'statusCode': 204,
+                            'body': json.dumps(f'Approved the TicketId: {ticket}, by Manager: {manager_mail}')
+                        }
+                    else:
+                        if action.upper() == REJECT:
+                            jira_conn.transition_issue(issue=ticket_id, transition=CLOSED, comment=f'From: {manager_mail}\nRejected\nPlease refer to your manager, in case any issues')
+                        return {
+                            'statusCode': 204,
+                            'body': json.dumps(f'Rejected the TicketId: {ticket}, by Manager: {manager_mail}')
+                        }
+                else:
+                    issue.update(comment=f'From: {manager_mail}\n{manager_mail.split("@")[0]} is not authorized to perform this action')
+                    return {
+                        'statusCode': 500,
+                        'body': json.dumps(f'{manager_mail} is not authorized to perform this action')
+                    }
+
+        else:
+            return {
+                'statusCode': 400,
+                'body': json.dumps(f'Jira Token not found in the parameter store')
+            }
+    except Exception as err:
+        return {
+            'statusCode': 500,
+            'body': json.dumps(f'Something went wrong {err}')
+        }

aws_lambda_functions/CloudResourceOrchestration/requirements.txt

@@ -0,0 +1,2 @@
+boto3==1.26.1
+jira~=3.4.1

aws_lambda_functions/CloudResourceOrchestration/upload_to_lambda.sh

@@ -0,0 +1,25 @@
+PROJECT_NAME="CloudResourceOrchestration"
+SUCCESS_OUTPUT_PATH="/dev/null"
+AWS_DEFAULT_REGION="us-east-1"
+echo "Clearing if previously created zip file"
+
+PROJECT_PATH="$PWD/$PROJECT_NAME.zip"
+
+if [ -f $PROJECT_PATH ]; then
+    rm -rf  $PROJECT_PATH
+    rm -rf ./package
+    echo "Deleted Previously created zip file"
+fi
+
+pip install --target ./package -r requirements.txt > $SUCCESS_OUTPUT_PATH
+pushd package
+zip -r ../$PROJECT_NAME.zip . > $SUCCESS_OUTPUT_PATH
+popd
+zip -g $PROJECT_NAME.zip lambda_function.py > $SUCCESS_OUTPUT_PATH
+
+echo "#############################"
+# Uploading to AWS Lambda
+echo "Uploading to AWS Lambda install Region: $AWS_DEFAULT_REGION"
+aws lambda update-function-code --function-name CloudResourceOrch --zip-file fileb://$PROJECT_PATH --region $AWS_DEFAULT_REGION > $SUCCESS_OUTPUT_PATH
+echo "Uploaded to AWS Lambda"
+echo "#############################"

cloud_governance/cloud_resource_orchestration/README.md

@@ -0,0 +1,14 @@
+## Cloud Resource Orchestration
+
+This is the process to control costs on public clouds. \
+This process requires the data how many days a project will run and estimated_cost. \
+Details are collected from the front end page @[https://cloud-governance.rdu2.scalelab.redhat.com/](https://cloud-governance.rdu2.scalelab.redhat.com/)
+After filling the form, mail sent to manager for approval after approved your request.
+Tag your instances with TicketId: #ticket_number. \
+Then cloud_governance will start **cloud_resource_orchestration** and monitor your instances.
+
+To start **cloud_resource_orchestration** CI run the below podman command 
+
+```commandline
+podman run --net="host" --rm --name  cloud_resource_orchestration -e AWS_DEFAULT_REGION="ap-south-1" -e CLOUD_RESOURCE_ORCHESTRATION="True" -e account="$account" -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" -e PUBLIC_CLOUD_NAME="$PUBLIC_CLOUD_NAME" -e es_host="$ES_HOST" -e es_port="$ES_PORT" -e CRO_ES_INDEX="$CRO_ES_INDEX" -e log_level="INFO" -e LDAP_HOST_NAME="$LDAP_HOST_NAME" -e JIRA_QUEUE="$JIRA_QUEUE" -e JIRA_TOKEN="$JIRA_TOKEN" -e JIRA_USERNAME="$JIRA_USERNAME" -e JIRA_URL="$JIRA_URL" -e CRO_COST_OVER_USAGE="$CRO_COST_OVER_USAGE" -e CRO_PORTAL="$CRO_PORTAL" -e CRO_DEFAULT_ADMINS="$CRO_DEFAULT_ADMINS" -e CRO_REPLACED_USERNAMES="$CRO_REPLACED_USERNAMES" -e CRO_DURATION_DAYS="30" quay.io/ebattat/cloud-governance:latest
+```