+ Your {cloud_name} cost usage in the last {self.__cro_duration_days} days is $ {user_cost} and {message}
+ You must open the project ticket in the following
Link.
+ After submitting a ticket, you must add Tag (TicketId:#) to every active resource that is related to the project ticket.
+
+ If you have any questions, please let us know in slack channel #perf-dept-public-clouds
+
+ {self.FOOTER}
+"""
+ return subject, body
+
+ def cro_request_for_manager_approval(self, manager: str, request_user: str, cloud_name: str, ticket_id: str, description: dict, **kwargs):
+ """
+ This method returns the message for manager, regarding user approval
+ :param description:
+ :param ticket_id:
+ :param manager:
+ :param request_user:
+ :param cloud_name:
+ :return:
+ """
+ subject = '[Action required]: Cloud Resources Budget Request Approval'
+ manager_full_name = self.get_user_ldap_details(user_name=manager)
+ user_full_name = self.get_user_ldap_details(user_name=request_user)
+ ticket_id = ticket_id.split('-')[-1]
+ context = {'manager': manager, 'manager_full_name': manager_full_name, 'user_full_name': user_full_name,
+ 'ticket_id': ticket_id, 'portal': self.__portal, 'request_user': request_user, 'description': description,
+ 'footer': self.FOOTER}
+ template_loader = self.env_loader.get_template('cro_request_for_manager_approval.j2')
+ context['extra_message'] = kwargs.get('extra_message', '')
+ body = template_loader.render(context)
+ return subject, body
+
+ def cro_send_user_alert_to_add_tags(self, user: str, ticket_ids: list):
+ """
+ This method return the subject, body for adding tags
+ :param user:
+ :param ticket_ids:
+ :return:
+ """
+ subject = '[Action required]: Add TicketId tag'
+ ticket_ids = "\n".join([f"
{val}" for idx, val in enumerate(ticket_ids)])
+ user_display_name = self.get_user_ldap_details(user_name=user)
+ body = f"""
+
Hi {user_display_name},
+
You have the following Approved JIRA Ticket-Ids
+
+ Currently, there are several instances running over budget, kindly review and tag instances with TicketId: #
+
Please find the below attached document.
+
+ {self.FOOTER}
+ """
+ return subject, body
+
+ def cro_send_closed_alert(self, user: str, ticket_id: str):
+ """
+ This method send cro ticket close alert
+ :param user:
+ :param ticket_id:
+ :return:
+ """
+ subject = 'Closing Cloud Budget Request ticket'
+ ticket_id = ticket_id.split('-')[-1]
+ user_full_name = self.get_user_ldap_details(user_name=user)
+ body = f"""
+
Hi {user_full_name},
+
+ Your cloud budget request ( TicketId: {ticket_id} ) duration expired and the ticket auto closed.
+ You can find the summary in
Portal.
+
+ {self.FOOTER}
+ """
+ return subject, body
+
+ def filter_resources_on_days(self, resources: dict):
+ """
+ This method return the resources based on the days
+ :param resources:
+ :param days:
+ :return:
+ """
+ resources_by_days = {}
+ for policy_name, resource_data in resources.items():
+ for region_name, policy_region_data in resource_data.items():
+ for data_item in policy_region_data:
+ resources_by_days.setdefault(data_item.get('Days'), []).append(data_item)
+ return resources_by_days
+
+ def get_data_in_html_table_format(self, resources: dict):
+ """
+ This method returns user policy alerts in HTML table format
+ :param resources:
+ :return:
+ """
+ style = """
+
+ """
+ html_table_format = f"""{style}
"""
+ thead_values = ['Policy', 'Region', 'ResourceId', 'Name', 'Action', 'DeletedDay']
+ th_elements = ''.join([f'| {value} | ' for value in thead_values])
+ html_table_format += f'{th_elements}
'
+ for days, resource_data in resources.items():
+ resource_data = sorted(resource_data, key=lambda item: (item.get('Policy'), item.get('Region')))
+ for resource in resource_data:
+ html_table_format += ''
+ for th_value in thead_values:
+ if 'Deleted' == resource.get(th_value):
+ html_table_format += f"| {resource.get(th_value)} 🗑 | "
+ else:
+ html_table_format += f"""{resource.get(th_value)} | """
+ html_table_format += '
'
+ html_table_format += '
'
+ return html_table_format
+
+ def get_agg_policies_mail_message(self, user: str, user_resources: dict):
+ """
+ This method returns the message for the aggregated alert of all policies
+ :param user:
+ :param user_resources:
+ :return:
+ """
+ display_name = self.get_user_ldap_details(user_name=user)
+ resources_by_days = self.filter_resources_on_days(resources=user_resources)
+ table_data = self.get_data_in_html_table_format(resources=resources_by_days)
+ display_name = display_name if display_name else user
+ subject = f'Cloud Governance: Policy Alerts'
+ body = f"""
+
+
+
You can find below your unused resources in the {self.__public_cloud_name} account ({self.account}).
+
If you want to keep them, please add "Policy=Not_Delete" or "Policy=skip" tag for each resource
+ {table_data}
+
+
{self.RESTRICTION}
+ {self.FOOTER}
+"""
+ return subject, body
+
+ def cro_monitor_budget_remain_alert(self, ticket_id: str, budget: int, user: str, used_budget: int, remain_budget: int):
+ """
+ This method returns subject, body for the budget remain alert
+ :param ticket_id:
+ :param budget:
+ :param user:
+ :param used_budget:
+ :param remain_budget:
+ :return:
+ """
+ ticket_id = ticket_id.split('-')[-1]
+ subject = f'[Action required] Cloud Resources Budget Remain'
+ user_display_name = self.get_user_ldap_details(user_name=user)
+ template_loader = self.env_loader.get_template('cro_monitor_budget_remain_alert.j2')
+ context = {'name': user_display_name, 'ticket_id': ticket_id, 'portal': self.__portal,
+ 'budget': budget, 'used_budget': used_budget, 'remain_budget': remain_budget, 'footer': self.FOOTER}
+ body = template_loader.render(context)
+ return subject, body
+
+ def cro_monitor_budget_remain_high_alert(self, ticket_id: str, budget: int, user: str, used_budget: int, remain_budget: int):
+ """
+ This method returns subject, body for the budget completed high alert
+ :param ticket_id:
+ :param budget:
+ :param user:
+ :param used_budget:
+ :param remain_budget:
+ :return:
+ """
+ ticket_id = ticket_id.split('-')[-1]
+ subject = f'[Action required] Cloud Resources Budget Remain'
+ user_display_name = self.get_user_ldap_details(user_name=user)
+ template_loader = self.env_loader.get_template('cro_monitor_budget_remain_high_alert.j2')
+ context = {'name': user_display_name, 'ticket_id': ticket_id, 'portal': self.__portal,
+ 'budget': budget, 'used_budget': used_budget, 'remain_budget': remain_budget,
+ 'footer': self.FOOTER}
+ body = template_loader.render(context)
return subject, body
diff --git a/cloud_governance/common/mails/postfix.py b/cloud_governance/common/mails/postfix.py
index 266b9d3d..4cc9191c 100644
--- a/cloud_governance/common/mails/postfix.py
+++ b/cloud_governance/common/mails/postfix.py
@@ -34,6 +34,10 @@ def __init__(self):
self.__es_port = self.__environment_variables_dict.get('es_port', '')
self.__account = self.__environment_variables_dict.get('account', '')
self.__policy_output = self.__environment_variables_dict.get('policy_output', '')
+ self.__default_admins = self.__environment_variables_dict.get('DEFAULT_ADMINS')
+ self.__email_alert = self.__environment_variables_dict.get('EMAIL_ALERT')
+ self.__mail_to = self.__environment_variables_dict.get('EMAIL_TO') # testing purposes
+ self.__mail_cc = self.__environment_variables_dict.get('EMAIL_CC')
self.bucket_name, self.key = self.get_bucket_name()
self.__es_index = 'cloud-governance-mail-messages'
if self.__es_host:
@@ -53,62 +57,72 @@ def get_bucket_name(self):
@logger_time_stamp
def send_email_postfix(self, subject: str, to: any, cc: list, content: str, **kwargs):
- msg = MIMEMultipart('alternative')
- msg["Subject"] = subject
- msg["From"] = "%s <%s>" % (
- 'cloud-governance',
- "@".join(["noreply-cloud-governance", 'redhat.com']),
- )
- if isinstance(to, str):
- msg["To"] = "@".join([to, 'redhat.com'])
- elif isinstance(to, list):
- msg["To"] = ", ".join(to)
- msg["Cc"] = ",".join(cc)
- # msg.add_header("Reply-To", self.reply_to)
- # msg.add_header("User-Agent", self.reply_to)
- if kwargs.get('filename'):
- attachment = MIMEText(open(kwargs['filename']).read())
- attachment.add_header('Content-Disposition', 'attachment',
- filename=kwargs['filename'].split('/')[-1])
- msg.attach(attachment)
- if kwargs.get('mime_type'):
- msg.attach(MIMEText(content, kwargs.get('mime_type')))
- else:
- msg.attach(MIMEText(content))
- email_string = msg.as_string()
- email_host = 'localhost'
- try:
- with smtplib.SMTP(email_host) as s:
- try:
- logger.debug(email_string)
- s.send_message(msg)
- if isinstance(to, str):
- logger.info(f'Mail sent successfully to {to}@redhat.com')
- elif isinstance(to, list):
- logger.info(f'Mail sent successfully to {", ".join(to)}@redhat.com')
- if kwargs.get('filename'):
- file_name = kwargs['filename'].split('/')[-1]
- date_key = datetime.datetime.now().strftime("%Y%m%d%H")
- if self.__policy_output:
- self.__s3_operations.upload_file(file_name_path=kwargs['filename'],
- bucket=self.bucket_name, key=f'{self.key}/{self.__policy}/{date_key}',
- upload_file=file_name)
- s3_path = f'{self.__policy_output}/logs/{self.__policy}/{date_key}/{file_name}'
- content += f'\n\nresource_file_path: s3://{s3_path}\n\n'
- data = {'Policy': self.__policy, 'To': to, 'Cc': cc, 'Message': content, 'Account': self.__account.upper(), 'MessageType': kwargs.get('message_type')}
- if kwargs.get('resource_id'):
- data['resource_id'] = kwargs['resource_id']
- if kwargs.get('extra_purse'):
- data['extra_purse'] = round(kwargs['extra_purse'], 3)
- if self.__es_host:
- self.__es_operations.upload_to_elasticsearch(data=data, index=self.__es_index)
- logger.info(f'Uploaded to es index: {self.__es_index}')
- else:
- logger.info('Error missing the es_host')
- except smtplib.SMTPException as ex:
- logger.info(f'Error while sending mail, {ex}')
- return False
- return True
- except Exception as err:
- logger.info(f'Some error occurred, {err}')
- return False
+ if self.__email_alert:
+ if self.__mail_to:
+ to = self.__mail_to
+ if self.__mail_cc:
+ cc = self.__mail_cc
+ cc = [cc_user for cc_user in cc if to and to not in cc_user]
+ cc = [cc_user if '@redhat.com' in cc_user else f'{cc_user}@redhat.com' for cc_user in cc]
+ msg = MIMEMultipart('alternative')
+ msg["Subject"] = subject
+ msg["From"] = "%s <%s>" % (
+ 'cloud-governance',
+ "@".join(["noreply-cloud-governance", 'redhat.com']),
+ )
+ if isinstance(to, str):
+ msg["To"] = "@".join([to, 'redhat.com'])
+ elif isinstance(to, list):
+ msg["To"] = ", ".join(to)
+ msg["Cc"] = ",".join(cc)
+ # msg.add_header("Reply-To", self.reply_to)
+ # msg.add_header("User-Agent", self.reply_to)
+ if kwargs.get('filename'):
+ attachment = MIMEText(open(kwargs['filename']).read())
+ attachment.add_header('Content-Disposition', 'attachment',
+ filename=kwargs['filename'].split('/')[-1])
+ msg.attach(attachment)
+ if kwargs.get('mime_type'):
+ msg.attach(MIMEText(content, kwargs.get('mime_type')))
+ else:
+ msg.attach(MIMEText(content))
+ email_string = msg.as_string()
+ email_host = 'localhost'
+ try:
+ with smtplib.SMTP(email_host) as s:
+ try:
+ logger.debug(email_string)
+ s.send_message(msg)
+ if isinstance(to, str):
+ logger.warn(f'Mail sent successfully to {to}@redhat.com')
+ elif isinstance(to, list):
+ logger.warn(f'Mail sent successfully to {", ".join(to)}@redhat.com')
+ if kwargs.get('filename'):
+ file_name = kwargs['filename'].split('/')[-1]
+ date_key = datetime.datetime.now().strftime("%Y%m%d%H")
+ if self.__policy_output:
+ self.__s3_operations.upload_file(file_name_path=kwargs['filename'],
+ bucket=self.bucket_name, key=f'{self.key}/{self.__policy}/{date_key}',
+ upload_file=file_name)
+ s3_path = f'{self.__policy_output}/logs/{self.__policy}/{date_key}/{file_name}'
+ content += f'\n\nresource_file_path: s3://{s3_path}\n\n'
+ es_data = kwargs.get('es_data')
+ data = {'Policy': self.__policy, 'To': to, 'Cc': cc, 'Message': content, 'Account': self.__account.upper(), 'MessageType': kwargs.get('message_type', 'alert')}
+ if es_data:
+ data.update(es_data)
+ if kwargs.get('resource_id'):
+ data['resource_id'] = kwargs['resource_id']
+ if kwargs.get('extra_purse'):
+ data['extra_purse'] = round(kwargs['extra_purse'], 3)
+ if self.__es_host:
+ self.__es_operations.upload_to_elasticsearch(data=data, index=self.__es_index)
+ logger.warn(f'Uploaded to es index: {self.__es_index}')
+ else:
+ logger.warn('Error missing the es_host')
+ except smtplib.SMTPException as ex:
+ logger.error(f'Error while sending mail, {ex}')
+ return False
+ return True
+ except Exception as err:
+ logger.error(f'Some error occurred, {err}')
+ return False
diff --git a/cloud_governance/common/mails/templates/cro_monitor_budget_remain_alert.j2 b/cloud_governance/common/mails/templates/cro_monitor_budget_remain_alert.j2
new file mode 100644
index 00000000..6967b1a5
--- /dev/null
+++ b/cloud_governance/common/mails/templates/cro_monitor_budget_remain_alert.j2
@@ -0,0 +1,12 @@
+
+
+
You completed {{used_budget}} / {{budget}} of budget till today of the TicketId: {{ticket_id}}.
+ Remaining budget will be {{remain_budget}}.
+
Open the budget extension if further needed.
+
This ticket will be closed, if the budget will exceeds 110%.
+
You can extend the budget here.
+
Visit the wiki page to get more information
+
+{{footer}}
diff --git a/cloud_governance/common/mails/templates/cro_monitor_budget_remain_high_alert.j2 b/cloud_governance/common/mails/templates/cro_monitor_budget_remain_high_alert.j2
new file mode 100644
index 00000000..06cf2176
--- /dev/null
+++ b/cloud_governance/common/mails/templates/cro_monitor_budget_remain_high_alert.j2
@@ -0,0 +1,11 @@
+
+
+
You completed {{used_budget}} / {{budget}} of budget till today of the TicketId: {{ticket_id}}.
+ You exceed budget {{-1*remain_budget}} more.
+
Open the budget extension if further needed.
+
You can extend the budget here.
+
Visit the wiki page to get more information
+
+{{footer}}
diff --git a/cloud_governance/common/mails/templates/cro_request_for_manager_approval.j2 b/cloud_governance/common/mails/templates/cro_request_for_manager_approval.j2
new file mode 100644
index 00000000..bfc54109
--- /dev/null
+++ b/cloud_governance/common/mails/templates/cro_request_for_manager_approval.j2
@@ -0,0 +1,18 @@
+
Hi {{manager_full_name}},
+{%if extra_message %}
{{extra_message}}
{%endif%}
+
+ {{user_full_name}} is waiting for your project cloud budget approval
+ Please approve the request in the following url
{{portal}}
+
+
+ Description of the New Request
+
+ {% for key, value in description.items() %}
+ {{key}}: {{value}}
+ {% endfor %}
+
+ If you are not able to approve from the site.
+ Please click here to send mail approve,
Approve request or
+
Reject Request.
+ Note: Highly recommend to use site for approving requests
+{{footer}}
diff --git a/cloud_governance/common/pandas/__init__.py b/cloud_governance/common/pandas/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloud_governance/common/pandas/pandas_operations.py b/cloud_governance/common/pandas/pandas_operations.py
new file mode 100644
index 00000000..1559b267
--- /dev/null
+++ b/cloud_governance/common/pandas/pandas_operations.py
@@ -0,0 +1,55 @@
+import tempfile
+
+import pandas as pd
+import typeguard
+
+from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+
+
+class PandasOperations:
+ """
+ This class performs the pandas operations
+ """
+ CHUNK_SIZE = 5000
+
+ def __init__(self, region_name: str = 'us-east-1'):
+ self.__s3_operations = S3Operations(region_name=region_name)
+
+ @typeguard.typechecked
+ @logger_time_stamp
+ def get_dataframe_from_csv_file(self, file_path: str):
+ """
+ This method returns the pandas dataframe from the csv file
+ :param file_path:
+ :return:
+ """
+ dataframes = []
+ for data_chunk in pd.read_csv(filepath_or_buffer=file_path, chunksize=self.CHUNK_SIZE):
+ dataframes.append(data_chunk)
+ dataframe = pd.concat(dataframes, ignore_index=True)
+ return dataframe
+
+ @typeguard.typechecked
+ @logger_time_stamp
+ def get_dataframe_from_s3_file(self, bucket: str, key: str, download_file: str):
+ """
+ This method returns the pandas dataframe from the s3 file
+ :return:
+ """
+ if not self.__s3_operations.file_exist(bucket=bucket, key=key, file_name=download_file):
+ raise FileNotFoundError(f"{key}/{download_file} path is not exists else verify your credentials")
+ with tempfile.NamedTemporaryFile(suffix='.csv', mode='w') as file_name:
+ self.__s3_operations.download_file(bucket=bucket, key=key, download_file=download_file,
+ file_name_path=file_name.name)
+ return self.get_dataframe_from_csv_file(file_path=file_name.name)
+
+ @typeguard.typechecked
+ @logger_time_stamp
+ def get_data_dictonary_from_dataframe(self, dataframe: pd.DataFrame):
+ """
+ This method returns the dataframe format to dictonary order
+ :param dataframe:
+ :return:
+ """
+ return dataframe.to_dict(orient='records')
diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py
index 6564f5e8..a50d8ef6 100644
--- a/cloud_governance/main/environment_variables.py
+++ b/cloud_governance/main/environment_variables.py
@@ -1,6 +1,12 @@
+import argparse
import os
-from cloud_governance.common.clouds.aws.iam.iam_operations import IAMOperations
+import tempfile
+from ast import literal_eval
+
+import boto3
+
+from cloud_governance.main.environment_variables_exceptions import ParseFailed
class EnvironmentVariables:
@@ -31,24 +37,30 @@ def __init__(self):
##################################################################################################
# dynamic parameters - configure for local run
# parameters for running policies
- self._environment_variables_dict['account'] = EnvironmentVariables.get_env('account', '').upper()
+ self._environment_variables_dict['account'] = EnvironmentVariables.get_env('account', '').upper().strip()
self._environment_variables_dict['AWS_DEFAULT_REGION'] = EnvironmentVariables.get_env('AWS_DEFAULT_REGION', '')
-
+ self._environment_variables_dict['log_level'] = EnvironmentVariables.get_env('log_level', 'INFO')
+ self._environment_variables_dict['PRINT_LOGS'] = EnvironmentVariables.get_boolean_from_environment('PRINT_LOGS', True)
+ if not self._environment_variables_dict['AWS_DEFAULT_REGION']:
+ self._environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2'
+ self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = EnvironmentVariables.get_env('PUBLIC_CLOUD_NAME', 'AWS')
if EnvironmentVariables.get_env('AWS_ACCESS_KEY_ID', '') and EnvironmentVariables.get_env('AWS_SECRET_ACCESS_KEY', ''):
- self.iam_operations = IAMOperations()
- self._environment_variables_dict['account'] = self.iam_operations.get_account_alias_cloud_name()[0].upper()
-
+ self._environment_variables_dict['account'] = self.get_aws_account_alias_name().upper().replace('OPENSHIFT-', '')
self._environment_variables_dict['policy'] = EnvironmentVariables.get_env('policy', '')
self._environment_variables_dict['aws_non_cluster_policies'] = ['ec2_idle', 'ec2_stop', 'ec2_run', 'ebs_in_use',
'ebs_unattached', 's3_inactive',
'empty_roles', 'ip_unattached',
- 'nat_gateway_unused',
+ 'unused_nat_gateway',
'zombie_snapshots', 'skipped_resources',
'monthly_report']
- self._environment_variables_dict['cost_policies'] = ['cost_explorer', 'cost_over_usage', 'cost_billing_reports', 'cost_explorer_payer_billings']
+ es_index = 'cloud-governance-policy-es-index'
+ self._environment_variables_dict['cost_policies'] = ['cost_explorer', 'cost_over_usage', 'cost_billing_reports',
+ 'cost_explorer_payer_billings', 'spot_savings_analysis']
self._environment_variables_dict['ibm_policies'] = ['tag_baremetal', 'tag_vm', 'ibm_cost_report',
'ibm_cost_over_usage']
+ if self._environment_variables_dict['policy'] in self._environment_variables_dict['cost_policies']:
+ es_index = 'cloud-governance-global-cost-billing-index'
# AWS env vars
self._environment_variables_dict['resource_name'] = EnvironmentVariables.get_env('resource_name', '')
@@ -64,6 +76,7 @@ def __init__(self):
self._environment_variables_dict['service_type'] = EnvironmentVariables.get_env('service_type', '')
self._environment_variables_dict['TABLE_NAME'] = EnvironmentVariables.get_env('TABLE_NAME', '')
self._environment_variables_dict['REPLACE_ACCOUNT_NAME'] = EnvironmentVariables.get_env('REPLACE_ACCOUNT_NAME', '{}')
+ self._environment_variables_dict['DAYS_TO_DELETE_RESOURCE'] = int(EnvironmentVariables.get_env('DAYS_TO_DELETE_RESOURCE', '7'))
# AWS Cost Explorer tags
self._environment_variables_dict['cost_metric'] = EnvironmentVariables.get_env('cost_metric', 'UnblendedCost')
@@ -71,6 +84,7 @@ def __init__(self):
self._environment_variables_dict['end_date'] = EnvironmentVariables.get_env('end_date', '')
self._environment_variables_dict['granularity'] = EnvironmentVariables.get_env('granularity', 'DAILY')
self._environment_variables_dict['cost_explorer_tags'] = EnvironmentVariables.get_env('cost_explorer_tags', '{}')
+ self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = EnvironmentVariables.get_env('PUBLIC_CLOUD_NAME', 'AWS')
# AZURE Credentials
self._environment_variables_dict['AZURE_ACCOUNT_ID'] = EnvironmentVariables.get_env('AZURE_ACCOUNT_ID', '')
@@ -80,7 +94,7 @@ def __init__(self):
if self._environment_variables_dict['AZURE_CLIENT_ID'] and self._environment_variables_dict['AZURE_TENANT_ID']\
and self._environment_variables_dict['AZURE_CLIENT_SECRET']:
self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'AZURE'
- self._environment_variables_dict['TOTAL_ACCOUNTS'] = bool(EnvironmentVariables.get_env('TOTAL_ACCOUNTS', ''))
+ self._environment_variables_dict['TOTAL_ACCOUNTS'] = EnvironmentVariables.get_boolean_from_environment('TOTAL_ACCOUNTS', False)
# IBM env vars
self._environment_variables_dict['IBM_ACCOUNT_ID'] = EnvironmentVariables.get_env('IBM_ACCOUNT_ID', '')
@@ -98,7 +112,7 @@ def __init__(self):
# Common env vars
self._environment_variables_dict['dry_run'] = EnvironmentVariables.get_env('dry_run', 'yes')
- self._environment_variables_dict['FORCE_DELETE'] = EnvironmentVariables.get_env('FORCE_DELETE', False)
+ self._environment_variables_dict['FORCE_DELETE'] = EnvironmentVariables.get_boolean_from_environment('FORCE_DELETE', False)
self._environment_variables_dict['policy_output'] = EnvironmentVariables.get_env('policy_output', '')
self._environment_variables_dict['bucket'] = EnvironmentVariables.get_env('bucket', '')
self._environment_variables_dict['file_path'] = EnvironmentVariables.get_env('file_path', '')
@@ -109,7 +123,7 @@ def __init__(self):
self._environment_variables_dict['upload_data_es'] = EnvironmentVariables.get_env('upload_data_es', '')
self._environment_variables_dict['es_host'] = EnvironmentVariables.get_env('es_host', '')
self._environment_variables_dict['es_port'] = EnvironmentVariables.get_env('es_port', '')
- self._environment_variables_dict['es_index'] = EnvironmentVariables.get_env('es_index', '')
+ self._environment_variables_dict['es_index'] = EnvironmentVariables.get_env('es_index', es_index)
self._environment_variables_dict['es_doc_type'] = EnvironmentVariables.get_env('es_doc_type', '')
self._environment_variables_dict['ES_TIMEOUT'] = EnvironmentVariables.get_env('ES_TIMEOUT', 2000)
@@ -137,6 +151,8 @@ def __init__(self):
# AWS Top Acconut
self._environment_variables_dict['AWS_ACCOUNT_ROLE'] = EnvironmentVariables.get_env('AWS_ACCOUNT_ROLE', '')
+ self._environment_variables_dict['PAYER_SUPPORT_FEE_CREDIT'] = EnvironmentVariables.get_env('PAYER_SUPPORT_FEE_CREDIT', 0)
+ self._environment_variables_dict['TEMPORARY_DIR'] = EnvironmentVariables.get_env('TEMPORARY_DIR', '/tmp')
self._environment_variables_dict['COST_CENTER_OWNER'] = EnvironmentVariables.get_env('COST_CENTER_OWNER', '{}')
# Jira env parameters
@@ -147,13 +163,112 @@ def __init__(self):
self._environment_variables_dict['JIRA_PASSWORD'] = EnvironmentVariables.get_env('JIRA_PASSWORD', '')
# Cloud Resource Orchestration
+ self._environment_variables_dict['CRO_PORTAL'] = EnvironmentVariables.get_env('CRO_PORTAL', '')
self._environment_variables_dict['CLOUD_NAME'] = EnvironmentVariables.get_env('CLOUD_NAME', '')
self._environment_variables_dict['MONITOR'] = EnvironmentVariables.get_env('MONITOR', '')
- self._environment_variables_dict['MANAGEMENT'] = bool(EnvironmentVariables.get_env('MANAGEMENT', False))
+ self._environment_variables_dict['MANAGEMENT'] = EnvironmentVariables.get_boolean_from_environment('MANAGEMENT', False)
+
+ # GCP Account
+ self._environment_variables_dict['GCP_DATABASE_NAME'] = EnvironmentVariables.get_env('GCP_DATABASE_NAME')
+ self._environment_variables_dict['GCP_DATABASE_TABLE_NAME'] = EnvironmentVariables.get_env('GCP_DATABASE_TABLE_NAME')
+ if self._environment_variables_dict.get('GCP_DATABASE_TABLE_NAME'):
+ self._environment_variables_dict['PUBLIC_CLOUD_NAME'] = 'GCP'
+
+ self._environment_variables_dict['EMAIL_ALERT'] = EnvironmentVariables.get_boolean_from_environment('EMAIL_ALERT', True)
+ self._environment_variables_dict['MANAGER_EMAIL_ALERT'] = EnvironmentVariables.get_boolean_from_environment('MANAGER_EMAIL_ALERT', True)
+ self._environment_variables_dict['UPDATE_TAG_BULKS'] = int(EnvironmentVariables.get_env('UPDATE_TAG_BULKS', '20'))
+
+ # policies aggregate alert
+ self._environment_variables_dict['BUCKET_NAME'] = EnvironmentVariables.get_env('BUCKET_NAME')
+ self._environment_variables_dict['BUCKET_KEY'] = EnvironmentVariables.get_env('BUCKET_KEY')
+ self._environment_variables_dict['MAIL_ALERT_DAYS'] = literal_eval(EnvironmentVariables.get_env('MAIL_ALERT_DAYS', '[]'))
+ self._environment_variables_dict['POLICY_ACTIONS_DAYS'] = literal_eval(EnvironmentVariables.get_env('POLICY_ACTIONS_DAYS', '[]'))
+ self._environment_variables_dict['DEFAULT_ADMINS'] = literal_eval(EnvironmentVariables.get_env('DEFAULT_ADMINS', '[]'))
+ self._environment_variables_dict['KERBEROS_USERS'] = literal_eval(EnvironmentVariables.get_env('KERBEROS_USERS', '[]'))
+ self._environment_variables_dict['POLICIES_TO_ALERT'] = literal_eval(EnvironmentVariables.get_env('POLICIES_TO_ALERT', '[]'))
+ if self._environment_variables_dict.get('policy') in ['send_aggregated_alerts']:
+ self._environment_variables_dict['COMMON_POLICIES'] = True
+ # CRO -- Cloud Resource Orch
+ self._environment_variables_dict['CLOUD_RESOURCE_ORCHESTRATION'] = EnvironmentVariables.get_boolean_from_environment('CLOUD_RESOURCE_ORCHESTRATION', False)
+ self._environment_variables_dict['USER_COST_INDEX'] = EnvironmentVariables.get_env('USER_COST_INDEX', '')
+ self._environment_variables_dict['CRO_ES_INDEX'] = EnvironmentVariables.get_env('CRO_ES_INDEX', 'cloud-governance-resource-orchestration')
+ self._environment_variables_dict['CRO_COST_OVER_USAGE'] = int(EnvironmentVariables.get_env('CRO_COST_OVER_USAGE', '500'))
+ self._environment_variables_dict['CRO_DEFAULT_ADMINS'] = literal_eval(EnvironmentVariables.get_env('CRO_DEFAULT_ADMINS', "[]"))
+ self._environment_variables_dict['CRO_DURATION_DAYS'] = int(EnvironmentVariables.get_env('CRO_DURATION_DAYS', '30'))
+ self._environment_variables_dict['RUN_ACTIVE_REGIONS'] = EnvironmentVariables.get_boolean_from_environment('RUN_ACTIVE_REGIONS', False)
+ self._environment_variables_dict['CRO_RESOURCE_TAG_NAME'] = EnvironmentVariables.get_env('CRO_RESOURCE_TAG_NAME', 'TicketId')
+ self._environment_variables_dict['CRO_REPLACED_USERNAMES'] = literal_eval(EnvironmentVariables.get_env('CRO_REPLACED_USERNAMES', "['osdCcsAdmin']"))
+ self._environment_variables_dict['CE_PAYER_INDEX'] = EnvironmentVariables.get_env('CE_PAYER_INDEX', '')
+ self._environment_variables_dict['EMAIL_TO'] = EnvironmentVariables.get_env('EMAIL_TO', '')
+ self._environment_variables_dict['EMAIL_CC'] = literal_eval(EnvironmentVariables.get_env('EMAIL_CC', "[]"))
+ self._environment_variables_dict['MANAGER_ESCALATION_DAYS'] = int(EnvironmentVariables.get_env('MANAGER_ESCALATION_DAYS', '3'))
+ self._environment_variables_dict['GLOBAL_CLOUD_ADMIN'] = EnvironmentVariables.get_env('GLOBAL_CLOUD_ADMIN', 'natashba')
+ self._environment_variables_dict['TICKET_OVER_USAGE_LIMIT'] = int(EnvironmentVariables.get_env('TICKET_OVER_USAGE_LIMIT', '80'))
+
+ # AWS Athena
+ self._environment_variables_dict['S3_RESULTS_PATH'] = EnvironmentVariables.get_env('S3_RESULTS_PATH', '')
+ self._environment_variables_dict['DEFAULT_ROUND_DIGITS'] = \
+ int(EnvironmentVariables.get_env('DEFAULT_ROUND_DIGITS', '3'))
+ self._environment_variables_dict['ATHENA_DATABASE_NAME'] = EnvironmentVariables.get_env('ATHENA_DATABASE_NAME', '')
+ self._environment_variables_dict['ATHENA_TABLE_NAME'] = EnvironmentVariables.get_env('ATHENA_TABLE_NAME', '')
+
+
+
@staticmethod
- def get_env(var: str, defval: any = ''):
- return os.environ.get(var, defval)
+ def to_bool(arg, def_val: bool = None):
+ if isinstance(arg, bool):
+ return arg
+ if isinstance(arg, (int, float)):
+ return arg != 0
+ if isinstance(arg, str):
+ arg = arg.lower()
+ if arg == 'true' or arg == 'yes':
+ return True
+ elif arg == 'false' or arg == 'no':
+ return False
+ try:
+ arg1 = int(arg)
+ return arg1 != 0
+ except Exception:
+ pass
+ if def_val is not None:
+ return def_val
+ raise ParseFailed(f'Cannot parse {arg} as a boolean value')
+
+ def get_aws_account_alias_name(self):
+ """
+ This method return the aws account alias name
+ :return:
+ """
+ iam_client = boto3.client('iam')
+ try:
+ account_alias = iam_client.list_account_aliases()['AccountAliases']
+ if account_alias:
+ return account_alias[0].upper()
+ except:
+ return os.environ.get('account', '').upper()
+
+
+ @staticmethod
+ def get_env(var: str, defval=''):
+ lcvar = var.lower()
+ dashvar = lcvar.replace('_', '-')
+ parser = argparse.ArgumentParser(description='Run CloudGovernance', allow_abbrev=False)
+ if lcvar == dashvar:
+ parser.add_argument(f"--{lcvar}", default=os.environ.get(var, defval), type=str, metavar='String', help=var)
+ else:
+ parser.add_argument(f"--{lcvar}", f"--{dashvar}", default=os.environ.get(var, defval), type=str,
+ metavar='String', help=var)
+ args, ignore = parser.parse_known_args()
+ if hasattr(args, lcvar):
+ return getattr(args, lcvar)
+ else:
+ return os.environ.get(var, defval)
+
+ @staticmethod
+ def get_boolean_from_environment(var: str, defval: bool):
+ return EnvironmentVariables.to_bool(EnvironmentVariables.get_env(var), defval)
@property
def environment_variables_dict(self):
@@ -164,7 +279,6 @@ def environment_variables_dict(self):
environment_variables = EnvironmentVariables()
-
# env vars examples
# os.environ['AWS_DEFAULT_REGION'] = 'us-east-2'
# os.environ['AWS_DEFAULT_REGION'] = 'all'
diff --git a/cloud_governance/main/main.py b/cloud_governance/main/main.py
index 61b92c2d..ca5bd962 100644
--- a/cloud_governance/main/main.py
+++ b/cloud_governance/main/main.py
@@ -4,6 +4,8 @@
import boto3 # regions
from cloud_governance.cloud_resource_orchestration.monitor.cloud_monitor import CloudMonitor
+from cloud_governance.main.main_common_operations import run_common_policies
+from cloud_governance.main.run_cloud_resource_orchestration import run_cloud_resource_orchestration
from cloud_governance.policy.policy_operations.aws.cost_expenditure.cost_report_policies import CostReportPolicies
from cloud_governance.policy.policy_operations.azure.azure_policy_runner import AzurePolicyRunner
from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp, logger
@@ -13,6 +15,7 @@
remove_tag_non_cluster_resource, tag_na_resources
from cloud_governance.policy.policy_operations.aws.tag_user.run_tag_iam_user import tag_iam_user, run_validate_iam_user_tags
from cloud_governance.policy.policy_operations.aws.zombie_cluster.run_zombie_cluster_resources import zombie_cluster_resource
+from cloud_governance.policy.policy_operations.gcp.gcp_policy_runner import GcpPolicyRunner
from cloud_governance.policy.policy_operations.gitleaks.gitleaks import GitLeaks
from cloud_governance.policy.policy_operations.ibm.ibm_operations.ibm_policy_runner import IBMPolicyRunner
from cloud_governance.main.environment_variables import environment_variables
@@ -167,12 +170,6 @@ def run_policy(account: str, policy: str, region: str, dry_run: str):
raise Exception(f'Missing Policy name: {policy}')
-@logger_time_stamp
-def run_cloud_management():
- """This method run the cloud management"""
- return CloudMonitor().run()
-
-
@logger_time_stamp
def main():
"""
@@ -194,110 +191,124 @@ def main():
es_doc_type = environment_variables_dict.get('es_doc_type', '')
bucket = environment_variables_dict.get('bucket', '')
- non_cluster_polices_runner = None
- is_non_cluster_polices_runner = policy in environment_variables_dict.get('aws_non_cluster_policies')
- if is_non_cluster_polices_runner:
- non_cluster_polices_runner = ZombieNonClusterPolicies()
+ if environment_variables_dict.get('COMMON_POLICIES'):
+ run_common_policies()
+ elif environment_variables_dict.get('CLOUD_RESOURCE_ORCHESTRATION'):
+ run_cloud_resource_orchestration()
+ else:
+ non_cluster_polices_runner = None
+ is_non_cluster_polices_runner = policy in environment_variables_dict.get('aws_non_cluster_policies')
+ if is_non_cluster_polices_runner:
+ non_cluster_polices_runner = ZombieNonClusterPolicies()
- ibm_classic_infrastructure_policy_runner = None
- is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('ibm_policies')
- if not is_tag_ibm_classic_infrastructure_runner:
- if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'IBM':
- is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('cost_policies')
- if is_tag_ibm_classic_infrastructure_runner:
- ibm_classic_infrastructure_policy_runner = IBMPolicyRunner()
+ ibm_classic_infrastructure_policy_runner = None
+ is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('ibm_policies')
+ if not is_tag_ibm_classic_infrastructure_runner:
+ if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'IBM':
+ is_tag_ibm_classic_infrastructure_runner = policy in environment_variables_dict.get('cost_policies')
+ if is_tag_ibm_classic_infrastructure_runner:
+ ibm_classic_infrastructure_policy_runner = IBMPolicyRunner()
- is_cost_explorer_policies_runner = ''
- if not environment_variables_dict.get('PUBLIC_CLOUD_NAME'):
- cost_explorer_policies_runner = None
- is_cost_explorer_policies_runner = policy in environment_variables_dict.get('cost_policies')
- if is_cost_explorer_policies_runner:
- cost_explorer_policies_runner = CostReportPolicies()
+ is_cost_explorer_policies_runner = ''
+ if environment_variables_dict.get('PUBLIC_CLOUD_NAME') == 'AWS':
+ cost_explorer_policies_runner = None
+ is_cost_explorer_policies_runner = policy in environment_variables_dict.get('cost_policies')
+ if is_cost_explorer_policies_runner:
+ cost_explorer_policies_runner = CostReportPolicies()
- is_azure_policy_runner = ''
- if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AZURE':
- azure_cost_policy_runner = None
- is_azure_policy_runner = policy in environment_variables_dict.get('cost_policies')
- if is_azure_policy_runner:
- azure_cost_policy_runner = AzurePolicyRunner()
+ is_azure_policy_runner = ''
+ if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AZURE':
+ azure_cost_policy_runner = None
+ is_azure_policy_runner = policy in environment_variables_dict.get('cost_policies')
+ if is_azure_policy_runner:
+ azure_cost_policy_runner = AzurePolicyRunner()
- # cloud_resource_orchestration lon_run/short_run
- is_cloud_management = False
- if environment_variables_dict.get('MANAGEMENT'):
- is_cloud_management = True
+ is_gcp_policy_runner = ''
+ if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'GCP':
+ gcp_cost_policy_runner = None
+ is_gcp_policy_runner = policy in environment_variables_dict.get('cost_policies')
+ if is_gcp_policy_runner:
+ gcp_cost_policy_runner = GcpPolicyRunner()
- @logger_time_stamp
- def run_non_cluster_polices_runner():
- """
- This method run the aws non-cluster policies
- @return:
- """
- non_cluster_polices_runner.run()
+ @logger_time_stamp
+ def run_non_cluster_polices_runner():
+ """
+ This method run the aws non-cluster policies
+ @return:
+ """
+ non_cluster_polices_runner.run()
- def run_tag_ibm_classic_infrastructure_runner():
- """
- This method run the IBM policies
- @return:
- """
- ibm_classic_infrastructure_policy_runner.run()
+ def run_tag_ibm_classic_infrastructure_runner():
+ """
+ This method run the IBM policies
+ @return:
+ """
+ ibm_classic_infrastructure_policy_runner.run()
- @logger_time_stamp
- def run_cost_explorer_policies_runner():
- """
- This method run the aws cost_explorer policies
- @return:
- """
- cost_explorer_policies_runner.run()
+ @logger_time_stamp
+ def run_cost_explorer_policies_runner():
+ """
+ This method run the aws cost_explorer policies
+ @return:
+ """
+ cost_explorer_policies_runner.run()
- @logger_time_stamp
- def run_azure_policy_runner():
- """
- This method run the azure policies
- @return:
- """
- azure_cost_policy_runner.run()
+ @logger_time_stamp
+ def run_azure_policy_runner():
+ """
+ This method run the azure policies
+ @return:
+ """
+ azure_cost_policy_runner.run()
- # 1. ELK Uploader
- if upload_data_es:
- input_data = {'es_host': es_host,
- 'es_port': int(es_port),
- 'es_index': es_index,
- 'es_doc_type': es_doc_type,
- 'es_add_items': {'account': account},
- 'bucket': bucket,
- 'logs_bucket_key': 'logs',
- 's3_file_name': 'resources.json',
- 'region': region_env,
- 'policy': policy,
- }
- elk_uploader = ESUploader(**input_data)
- elk_uploader.upload_to_es(account=account)
- # 2. POLICY
- elif is_non_cluster_polices_runner:
- run_non_cluster_polices_runner()
- elif is_tag_ibm_classic_infrastructure_runner:
- run_tag_ibm_classic_infrastructure_runner()
- elif is_cost_explorer_policies_runner:
- run_cost_explorer_policies_runner()
- elif is_azure_policy_runner:
- run_azure_policy_runner()
- elif is_cloud_management:
- run_cloud_management()
- else:
- if not policy:
- logger.exception(f'Missing Policy name: "{policy}"')
- raise Exception(f'Missing Policy name: "{policy}"')
- if region_env == 'all':
- # must be set for boto3 client default region
- # environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2'
- ec2 = boto3.client('ec2')
- regions_data = ec2.describe_regions()
- for region in regions_data['Regions']:
- # logger.info(f"region: {region['RegionName']}")
- environment_variables_dict['AWS_DEFAULT_REGION'] = region['RegionName']
- run_policy(account=account, policy=policy, region=region['RegionName'], dry_run=dry_run)
+ @logger_time_stamp
+ def run_gcp_policy_runner():
+ """
+ This method run the gcp policies
+ """
+ gcp_cost_policy_runner.run()
+
+ # 1. ELK Uploader
+ if upload_data_es:
+ input_data = {'es_host': es_host,
+ 'es_port': int(es_port),
+ 'es_index': es_index,
+ 'es_doc_type': es_doc_type,
+ 'es_add_items': {'account': account},
+ 'bucket': bucket,
+ 'logs_bucket_key': 'logs',
+ 's3_file_name': 'resources.json',
+ 'region': region_env,
+ 'policy': policy,
+ }
+ elk_uploader = ESUploader(**input_data)
+ elk_uploader.upload_to_es(account=account)
+ # 2. POLICY
+ elif is_non_cluster_polices_runner:
+ run_non_cluster_polices_runner()
+ elif is_tag_ibm_classic_infrastructure_runner:
+ run_tag_ibm_classic_infrastructure_runner()
+ elif is_cost_explorer_policies_runner:
+ run_cost_explorer_policies_runner()
+ elif is_azure_policy_runner:
+ run_azure_policy_runner()
+ elif is_gcp_policy_runner:
+ run_gcp_policy_runner()
else:
- run_policy(account=account, policy=policy, region=region_env, dry_run=dry_run)
+ if not policy:
+ logger.exception(f'Missing Policy name: "{policy}"')
+ raise Exception(f'Missing Policy name: "{policy}"')
+ if region_env == 'all':
+ # must be set for boto3 client default region
+ # environment_variables_dict['AWS_DEFAULT_REGION'] = 'us-east-2'
+ ec2 = boto3.client('ec2')
+ regions_data = ec2.describe_regions()
+ for region in regions_data['Regions']:
+ # logger.info(f"region: {region['RegionName']}")
+ environment_variables_dict['AWS_DEFAULT_REGION'] = region['RegionName']
+ run_policy(account=account, policy=policy, region=region['RegionName'], dry_run=dry_run)
+ else:
+ run_policy(account=account, policy=policy, region=region_env, dry_run=dry_run)
main()
diff --git a/cloud_governance/main/main_common_operations.py b/cloud_governance/main/main_common_operations.py
new file mode 100644
index 00000000..eb6a2437
--- /dev/null
+++ b/cloud_governance/main/main_common_operations.py
@@ -0,0 +1,12 @@
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+from cloud_governance.policy.policy_runners.common_policy_runner import CommonPolicyRunner
+
+
+@logger_time_stamp
+def run_common_policies():
+ """
+ This method run the common policies
+ :return:
+ """
+ common_policy_runner = CommonPolicyRunner()
+ common_policy_runner.run()
diff --git a/cloud_governance/main/run_cloud_resource_orchestration.py b/cloud_governance/main/run_cloud_resource_orchestration.py
new file mode 100644
index 00000000..268b0479
--- /dev/null
+++ b/cloud_governance/main/run_cloud_resource_orchestration.py
@@ -0,0 +1,19 @@
+from tempfile import TemporaryDirectory
+
+from cloud_governance.cloud_resource_orchestration.monitor.cloud_monitor import CloudMonitor
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+from cloud_governance.main.environment_variables import environment_variables
+
+
+@logger_time_stamp
+def run_cloud_management():
+ """This method run the cloud management"""
+ environment_variables_dict = environment_variables.environment_variables_dict
+ with TemporaryDirectory() as cache_temp_dir:
+ environment_variables_dict['TEMPORARY_DIRECTORY'] = cache_temp_dir
+ environment_variables_dict['policy'] = 'cloud_resource_orchestration'
+ return CloudMonitor().run()
+
+
+def run_cloud_resource_orchestration():
+ run_cloud_management()
diff --git a/cloud_governance/policy/aws/cost_billing_reports.py b/cloud_governance/policy/aws/cost_billing_reports.py
index ee15ae6b..a2acb789 100644
--- a/cloud_governance/policy/aws/cost_billing_reports.py
+++ b/cloud_governance/policy/aws/cost_billing_reports.py
@@ -35,13 +35,13 @@ def __init__(self):
self.__gsheet_id = self._environment_variables_dict.get('SPREADSHEET_ID', '')
self.gdrive_operations = GoogleDriveOperations()
self.update_to_gsheet = UploadToGsheet()
- self.cost_center, self.__account_budget, self.__years = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.__account_id)
+ self.cost_center, self.__account_budget, self.__years, self.__owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.__account_id)
except:
pass
def __get_start_date(self, end_date: datetime, days: int, operation: operator) -> datetime:
"""
- This method return start_date
+ This method returns start_date
@param operation:
@param end_date:
@param days:
diff --git a/cloud_governance/policy/aws/cost_explorer.py b/cloud_governance/policy/aws/cost_explorer.py
index 21426c2a..f1313dd4 100644
--- a/cloud_governance/policy/aws/cost_explorer.py
+++ b/cloud_governance/policy/aws/cost_explorer.py
@@ -1,4 +1,3 @@
-
from ast import literal_eval
from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
@@ -16,6 +15,18 @@ class CostExplorer:
"""
BULK_UPLOAD_THREADS = 8
+ SAVINGS_PLAN_FILTER = { # savings plan usage for ce filter
+ 'Dimensions': {
+ 'Key': 'RECORD_TYPE',
+ 'Values': ['SavingsPlanRecurringFee', 'SavingsPlanNegation', 'SavingsPlanCoveredUsage']
+ }
+ }
+ EXCLUDE = 'exclude'
+ INCLUDE = 'include'
+ CE_KEY_RESULTS_BY_TIME = 'ResultsByTime'
+ INDEX_ID = 'index_id'
+ APPEND = 'a'
+ CE_OPERATION = 'Not'
def __init__(self):
super().__init__()
@@ -32,11 +43,12 @@ def __init__(self):
self._elastic_upload = ElasticUpload()
self.__account = self.__environment_variables_dict.get('account').upper().replace('OPENSHIFT-', "").strip()
- def filter_data_by_tag(self, groups: dict, tag: str):
+ def filter_data_by_tag(self, groups: dict, tag: str, savings_plan: str):
"""
This method extract data by tag
@param tag:
@param groups: Data from the cloud explorer
+ @param savings_plan:
@return: converted into dict format
"""
data = {}
@@ -58,13 +70,14 @@ def filter_data_by_tag(self, groups: dict, tag: str):
else:
if 'vm_import_image' in name:
name = 'vm_import_image'
- index_id = f'{start_time.lower()}-{account.lower()}-{tag.lower()}-{name.lower()}'
+ index_id = f'{start_time}-{account}-{tag}-savings-{savings_plan}-{name}'.lower()
if index_id not in data:
- upload_data = {tag: name if tag.upper() == 'ChargeType'.upper() else name.upper(),
- 'Cost': round(float(amount), 3), 'index_id': index_id, 'timestamp': start_time}
+ upload_data = {tag: name if tag.upper() in list(self.__cost_explorer.CE_COST_TYPES) else name.upper(),
+ 'Cost': round(float(amount), 3), self.INDEX_ID: index_id, 'timestamp': start_time, 'savings_plan': savings_plan}
if 'global' in self._elastic_upload.es_index:
if 'Budget' not in upload_data:
upload_data['Budget'] = self._elastic_upload.account
+ upload_data['tag'] = tag.lower()
data[index_id] = upload_data
else:
data[index_id]['Cost'] += round(float(amount), 3)
@@ -77,15 +90,19 @@ def __get_daily_cost_by_tags(self):
"""
data_house = {}
for tag in self.cost_tags:
- if self.start_date and self.end_date:
- response = self.__cost_explorer.get_cost_by_tags(tag=tag, start_date=self.start_date, end_date=self.end_date, granularity=self.granularity, cost_metric=self.cost_metric)
- else:
- response = self.__cost_explorer.get_cost_by_tags(tag=tag, granularity=self.granularity, cost_metric=self.cost_metric)
- results_by_time = response.get('ResultsByTime')
- if results_by_time:
- data_house[tag] = []
- for result in results_by_time:
- data_house[tag].extend(self.filter_data_by_tag(result, tag))
+ for savings_plan in [self.EXCLUDE, self.INCLUDE]:
+ filters = {}
+ if savings_plan == self.EXCLUDE:
+ filters = {self.CE_OPERATION: self.SAVINGS_PLAN_FILTER}
+ if self.start_date and self.end_date:
+ response = self.__cost_explorer.get_cost_by_tags(tag=tag, start_date=self.start_date, end_date=self.end_date, granularity=self.granularity, cost_metric=self.cost_metric, Filter=filters)
+ else:
+ response = self.__cost_explorer.get_cost_by_tags(tag=tag, granularity=self.granularity, cost_metric=self.cost_metric, Filter=filters)
+ results_by_time = response.get(self.CE_KEY_RESULTS_BY_TIME)
+ if results_by_time:
+ data_house[f'{tag}-{savings_plan}'] = []
+ for result in results_by_time:
+ data_house[f'{tag}-{savings_plan}'].extend(self.filter_data_by_tag(result, tag, savings_plan))
return data_house
@logger_time_stamp
@@ -97,12 +114,12 @@ def __upload_data(self, data: list, index: str):
@return:
"""
if self.file_name:
- with open(f'/tmp/{self.file_name}', 'a') as file:
+ with open(f'/tmp/{self.file_name}', self.APPEND) as file:
for value in data:
file.write(f'{value}\n')
else:
for value in data:
- self.upload_item_to_es(index=index, item=value, index_id=value['index_id'])
+ self.upload_item_to_es(index=index, item=value, index_id=value[self.INDEX_ID])
logger.info(f'Data uploaded to {index}, Total Data: {len(data)}')
def upload_item_to_es(self, item: dict, index: str, index_id: str = ''):
@@ -126,8 +143,8 @@ def upload_tags_cost_to_elastic_search(self):
logger.info(f'Get {self.granularity} Cost usage by metric: {self.cost_metric}')
cost_data = self.__get_daily_cost_by_tags()
for key, values in cost_data.items():
- index = f'{self._elastic_upload.es_index}-{key.lower()}'
- self.__upload_data(values, index)
+ logger.info(f"Uploading the data of {key} tag")
+ self.__upload_data(values, self._elastic_upload.es_index)
def run(self):
"""
diff --git a/cloud_governance/policy/aws/cost_explorer_payer_billings.py b/cloud_governance/policy/aws/cost_explorer_payer_billings.py
index 2c28739f..be159456 100644
--- a/cloud_governance/policy/aws/cost_explorer_payer_billings.py
+++ b/cloud_governance/policy/aws/cost_explorer_payer_billings.py
@@ -1,26 +1,43 @@
import copy
import datetime
+import logging
from ast import literal_eval
import boto3
+
+from cloud_governance.common.clouds.aws.iam.iam_operations import IAMOperations
+from cloud_governance.common.clouds.aws.savingsplan.savings_plans_operations import SavingsPlansOperations
from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
-from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.common.logger.init_logger import logger, handler
from cloud_governance.common.clouds.aws.cost_explorer.cost_explorer_operations import CostExplorerOperations
from cloud_governance.policy.aws.cost_billing_reports import CostBillingReports
class CostExplorerPayerBillings(CostBillingReports):
- """This class is responsible for generation cost billing report for Budget, Actual, Forecast from the Org level"""
+ """
+ This class is responsible for generation cost billing report for Budget, Actual, Forecast from the Org level
+ Monthly savings Plan Amortization: (linked_account_total_cost/payer_account_total_cost) * monthly_savings_plan_cost
+ Monthly_support_fee: (monthly_support_fee - (monthly_support_fee * discount ) ) * (linked_account_total_cost/payer_account_total_cost)
+ """
+
+ DEFAULT_ROUND_DIGITS = 3
def __init__(self):
super().__init__()
self.__aws_role = self._environment_variables_dict.get("AWS_ACCOUNT_ROLE")
self.__access_key, self.__secret_key, self.__session = self.__get_sts_credentials()
self.__ce_client = boto3.client('ce', aws_access_key_id=self.__access_key, aws_secret_access_key=self.__secret_key, aws_session_token=self.__session)
+ self.__savings_plan_client = boto3.client('savingsplans', aws_access_key_id=self.__access_key, aws_secret_access_key=self.__secret_key, aws_session_token=self.__session)
+ self.__iam_client = boto3.client('iam', aws_access_key_id=self.__access_key, aws_secret_access_key=self.__secret_key, aws_session_token=self.__session)
+ self.__assumed_role_account_name = IAMOperations(iam_client=self.__iam_client).get_account_alias_cloud_name()
self.__cost_explorer_operations = CostExplorerOperations(ce_client=self.__ce_client)
- self.__cost_center_owner = literal_eval(self._environment_variables_dict.get('COST_CENTER_OWNER'))
+ self.__savings_plan_operations = SavingsPlansOperations(savings_plan_client=self.__savings_plan_client)
self.__replacement_account = literal_eval(self._environment_variables_dict.get('REPLACE_ACCOUNT_NAME'))
+ self.__savings_discounts = float(self._environment_variables_dict.get('PAYER_SUPPORT_FEE_CREDIT', 0))
+ self.__monthly_cost_for_spa_calc = {}
+ self.__monthly_cost_for_support_fee = {}
+ self.__temporary_dir = self._environment_variables_dict.get('TEMPORARY_DIR', '')
def __get_sts_credentials(self):
"""This method returns the temporary credentials from the sts service"""
@@ -50,23 +67,27 @@ def filter_data_by_tag(self, cost_data: dict, tag: str, cost_center: int = ''):
if name and amount:
if name not in data:
if cost_center:
- acc_cost_center, account_budget, years = self.update_to_gsheet.get_cost_center_budget_details(account_id=name, dir_path='/tmp')
+ acc_cost_center, account_budget, years, owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=name, dir_path=self.__temporary_dir)
timestamp = datetime.datetime.strptime(start_time, '%Y-%m-%d')
month = datetime.datetime.strftime(timestamp, "%Y %b")
- owner = self.__cost_center_owner.get(str(acc_cost_center)) if self.__cost_center_owner.get(str(acc_cost_center)) else 'Others'
+ month_full_name = datetime.datetime.strftime(timestamp, "%B")
+ payer_monthly_savings_plan = self.update_to_gsheet.get_monthly_spa(month_name=month_full_name, dir_path=self.__temporary_dir)
budget = account_budget if start_time.split('-')[0] in years else 0
index_id = f'{start_time}-{name}'
- upload_data = {tag: name, 'Actual': round(float(amount), 3), 'start_date': start_time,
+ upload_data = {tag: name, 'Actual': round(float(amount), self.DEFAULT_ROUND_DIGITS), 'start_date': start_time,
'timestamp': timestamp, 'CloudName': 'AWS', 'Month': month,
'Forecast': 0,
'filter_date': f'{start_time}-{month.split()[-1]}',
- 'Budget': round(budget / self.MONTHS, 3), 'CostCenter': cost_center,
+ 'Budget': round(budget / self.MONTHS, self.DEFAULT_ROUND_DIGITS), 'CostCenter': cost_center,
'AllocatedBudget': budget,
- "Owner": owner
+ "Owner": owner,
+ 'SavingsPlanCost': (float(amount) / float(self.__monthly_cost_for_spa_calc.get(start_time))) * payer_monthly_savings_plan,
+ 'TotalPercentage': (float(amount) / float(self.__monthly_cost_for_spa_calc.get(start_time)))
}
+ upload_data['PremiumSupportFee'] = (float(self.__monthly_cost_for_support_fee.get(start_time)) - (float(self.__monthly_cost_for_support_fee.get(start_time)) * self.__savings_discounts)) * upload_data['TotalPercentage'],
else:
index_id = f'{start_time}-{name}'
- upload_data = {tag: name, 'Actual': round(float(amount), 3)}
+ upload_data = {tag: name, 'Actual': round(float(amount), self.DEFAULT_ROUND_DIGITS)}
if index_id:
data[index_id] = upload_data
if cost_data.get('DimensionValueAttributes'):
@@ -82,20 +103,28 @@ def filter_data_by_tag(self, cost_data: dict, tag: str, cost_center: int = ''):
return data
def filter_forecast_data(self, cost_forecast_data: list, cost_usage_data: dict, account_id: str, cost_center: int, account: str):
- acc_cost_center, account_budget, years = self.update_to_gsheet.get_cost_center_budget_details(account_id=account_id, dir_path='/tmp')
- owner = self.__cost_center_owner.get(str(acc_cost_center)) if self.__cost_center_owner.get(str(acc_cost_center)) else 'Others'
+ acc_cost_center, account_budget, years, owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=account_id, dir_path=self.__temporary_dir)
for cost_forecast in cost_forecast_data:
start_date = str((cost_forecast.get('TimePeriod').get('Start')))
start_year = start_date.split('-')[0]
- cost = round(float(cost_forecast.get('MeanValue')), 3)
+ cost = round(float(cost_forecast.get('MeanValue')), self.DEFAULT_ROUND_DIGITS)
index = f'{start_date}-{account_id}'
+ month_full_name = datetime.datetime.strftime(datetime.datetime.strptime(start_date, '%Y-%m-%d'), "%B")
+ total_percentage = (cost / float(self.__monthly_cost_for_spa_calc.get(start_date)))
+ payer_monthly_savings_plan = self.update_to_gsheet.get_monthly_spa(month_name=month_full_name, dir_path=self.__temporary_dir)
if index in cost_usage_data[account]:
cost_usage_data[account][index]['Forecast'] = cost
+ cost_usage_data[account][index]['TotalPercentage'] = total_percentage
+ cost_usage_data[account][index]['SavingsPlanCost'] = total_percentage * payer_monthly_savings_plan
+ cost_usage_data[account][index]['PremiumSupportFee'] = total_percentage * (float(self.__monthly_cost_for_support_fee.get(start_date)) - (float(self.__monthly_cost_for_support_fee.get(start_date)) * self.__savings_discounts))
else:
data = {}
data['AccountId'] = account_id
data['Actual'] = 0
data['Forecast'] = cost
+ data['TotalPercentage'] = total_percentage
+ data['SavingsPlanCost'] = total_percentage * payer_monthly_savings_plan
+ data['PremiumSupportFee'] = total_percentage * (float(self.__monthly_cost_for_support_fee.get(start_date)) - (float(self.__monthly_cost_for_support_fee.get(start_date)) * self.__savings_discounts))
data['Account'] = account
data['start_date'] = str((cost_forecast.get('TimePeriod').get('Start')))
data['index_id'] = f"""{data['start_date']}-{data['Account'].lower()}"""
@@ -103,7 +132,7 @@ def filter_forecast_data(self, cost_forecast_data: list, cost_usage_data: dict,
data['Month'] = datetime.datetime.strftime(data['timestamp'], '%Y %b')
data['Owner'] = owner
if start_year in years:
- data['Budget'] = round(account_budget / self.MONTHS, 3)
+ data['Budget'] = round(account_budget / self.MONTHS, self.DEFAULT_ROUND_DIGITS)
data['AllocatedBudget'] = account_budget
else:
data['Budget'] = 0
@@ -113,7 +142,6 @@ def filter_forecast_data(self, cost_forecast_data: list, cost_usage_data: dict,
data['filter_date'] = f'{data["start_date"]}-{data["Month"].split()[-1]}'
cost_usage_data[account][index] = data
- @logger_time_stamp
def get_linked_accounts_forecast(self, linked_account_usage: dict):
"""
This method append the forecast to the linked accounts
@@ -125,8 +153,8 @@ def get_linked_accounts_forecast(self, linked_account_usage: dict):
try:
cost_forecast_data = self.__cost_explorer_operations.get_cost_forecast(start_date=start_date, end_date=end_date, granularity=self.GRANULARITY, cost_metric=self.COST_METRIC, Filter={'Dimensions': {'Key': 'LINKED_ACCOUNT', 'Values': [account_id]}})
self.filter_forecast_data(cost_forecast_data=cost_forecast_data['ForecastResultsByTime'], cost_center=cost_center, account=account, account_id=account_id, cost_usage_data=linked_account_usage)
- except:
- logger.info(f'No Data to get forecast: {account_id}: {account}')
+ except Exception as err:
+ logger.info(f'No Data to get forecast: {account_id}: {account}, {err}')
@logger_time_stamp
def get_cost_centers(self):
@@ -151,9 +179,38 @@ def get_linked_accounts_usage(self):
account = usage['Account']
cost_usage_data.setdefault(account, {}).update({idx: usage})
self.get_linked_accounts_forecast(linked_account_usage=cost_usage_data)
+ self.__get_ce_cost_usage_by_filter_tag(tag_name='spot', cost_centers=cost_centers, cost_usage_data=cost_usage_data)
+ handler.setLevel(logging.WARN)
+ # To prevent printing the **kwargs of the function when using the logger_time_stamp decorator.
self.upload_data_elastic_search(linked_account_usage=cost_usage_data)
+ handler.setLevel(logging.INFO)
return cost_usage_data
+ def __get_ce_cost_usage_by_filter_tag(self, cost_centers: list, tag_name: str, cost_usage_data: dict):
+ """
+ This method returns the cost by filter tag_name
+ :param cost_centers:
+ :param tag_name:
+ :return:
+ """
+ start_date, end_date = self.get_date_ranges()
+ for cost_center in cost_centers:
+ cost_center_number = cost_center.get('CostCenter')
+ filter_cost_center = {'CostCategories': {'Key': 'CostCenter', 'Values': [cost_center_number]}}
+ values = self.__cost_explorer_operations.CE_COST_FILTERS[tag_name.upper()]['Values']
+ filter_tag_value = {'Dimensions': {'Key': 'PURCHASE_TYPE', 'Values': values}}
+ group_by = {'Type': 'DIMENSION', 'Key': 'LINKED_ACCOUNT'}
+ cost_data = self.__cost_explorer_operations.get_cost_and_usage_from_aws(start_date=start_date, end_date=end_date, granularity="MONTHLY",
+ GroupBy=[group_by], Filter={'And': [filter_cost_center, filter_tag_value]})
+ filtered_data = self.__cost_explorer_operations.get_ce_report_filter_data(ce_response=cost_data, tag_name=tag_name)
+ if filtered_data:
+ for index_id, row in filtered_data.items():
+ account = row.get('Account')
+ if account in self.__replacement_account:
+ account = self.__replacement_account[account]
+ if account in cost_usage_data:
+ cost_usage_data[account][index_id][f'{tag_name.title()}Usage'] = round(float(row.get(tag_name)), self.DEFAULT_ROUND_DIGITS)
+
@logger_time_stamp
def upload_data_elastic_search(self, linked_account_usage: dict):
"""This method uploads the data to elastic search"""
@@ -163,8 +220,43 @@ def upload_data_elastic_search(self, linked_account_usage: dict):
monthly_account_cost.append(cost)
self.elastic_upload.es_upload_data(items=monthly_account_cost, set_index='index_id')
+ def filter_cost_details_for_sp(self, total_cost: list):
+ """"This method filter the account total cost"""
+ results = {}
+ for row in total_cost:
+ start_time = row.get('TimePeriod').get('Start')
+ if row.get('MeanValue'):
+ cost = round(float(row.get('MeanValue')), self.DEFAULT_ROUND_DIGITS)
+ else:
+ cost = round(float(row.get('Total').get('UnblendedCost').get('Amount')), self.DEFAULT_ROUND_DIGITS)
+ results[start_time] = cost
+ return results
+
+ def get_monthly_cost_details(self, start_date: datetime = None, end_date: datetime = None):
+ """This method list the savings plan details"""
+ current_date = datetime.datetime.utcnow()
+ if not start_date and not end_date:
+ end_date = (current_date.replace(day=1) - datetime.timedelta(days=1)).date()
+ start_date = end_date.replace(day=1)
+ end_date = end_date + datetime.timedelta(days=1)
+ payer_cost_response = self.__cost_explorer_operations.get_cost_and_usage_from_aws(start_date=str(start_date), end_date=str(end_date), granularity='MONTHLY', cost_metric=self.COST_METRIC, Filter={'Not': {'Dimensions': {'Key': 'RECORD_TYPE', 'Values': ['Support', 'Refund', 'Credit']}}})
+ payer_support_fee = self.__cost_explorer_operations.get_cost_and_usage_from_aws(start_date=str(start_date), end_date=str(end_date), granularity='MONTHLY', cost_metric=self.COST_METRIC, Filter={'Dimensions': {'Key': 'RECORD_TYPE', 'Values': ['Support']}})
+ filtered_payer_cost = self.filter_cost_details_for_sp(payer_cost_response.get('ResultsByTime'))
+ filtered_support_fee = self.filter_cost_details_for_sp(payer_support_fee.get('ResultsByTime'))
+ self.__monthly_cost_for_spa_calc = filtered_payer_cost
+ self.__monthly_cost_for_support_fee.update(filtered_support_fee)
+ start_date = current_date.date()
+ end_date = start_date + datetime.timedelta(days=360)
+ forecast_response = self.__cost_explorer_operations.get_cost_forecast(start_date=str(start_date), end_date=str(end_date), granularity=self.GRANULARITY, cost_metric=self.COST_METRIC, Filter={'Not': {'Dimensions': {'Key': 'RECORD_TYPE', 'Values': ['Support', 'Refund', 'Credit']}}})
+ payer_forecast_support_fee = self.__cost_explorer_operations.get_cost_forecast(start_date=str(start_date), end_date=str(end_date), granularity=self.GRANULARITY, cost_metric=self.COST_METRIC, Filter={'Dimensions': {'Key': 'RECORD_TYPE', 'Values': ['Support']}})
+ filtered_payer_forecast = self.filter_cost_details_for_sp(forecast_response.get('ForecastResultsByTime'))
+ filtered_payer_support_forecast = self.filter_cost_details_for_sp(payer_forecast_support_fee.get('ForecastResultsByTime'))
+ self.__monthly_cost_for_spa_calc.update(filtered_payer_forecast)
+ self.__monthly_cost_for_support_fee.update(filtered_payer_support_forecast)
+
def run(self):
"""
This method run the methods
"""
+ self.get_monthly_cost_details()
self.get_linked_accounts_usage()
diff --git a/cloud_governance/policy/aws/cost_over_usage.py b/cloud_governance/policy/aws/cost_over_usage.py
index e28b1930..616367f7 100644
--- a/cloud_governance/policy/aws/cost_over_usage.py
+++ b/cloud_governance/policy/aws/cost_over_usage.py
@@ -28,7 +28,7 @@ def __init__(self):
def get_user_used_instances(self, user_used_list: list):
"""
- This method return user used instances group by region
+ This method returns user used instances group by region
@return:
"""
region_resources = {}
diff --git a/cloud_governance/policy/aws/ebs_unattached.py b/cloud_governance/policy/aws/ebs_unattached.py
index e882cb65..2470918c 100644
--- a/cloud_governance/policy/aws/ebs_unattached.py
+++ b/cloud_governance/policy/aws/ebs_unattached.py
@@ -52,9 +52,10 @@ def __delete_ebs_unattached(self):
days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE,
extra_purse=ebs_cost, delta_cost=delta_cost)
if unattached_volumes:
- unattached_volumes_data.append([volume.get('VolumeId'),
- self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='Name'),
- self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='User'),
- str(last_detached_days),
- self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='Policy')])
+ unattached_volumes_data.append({'ResourceId': volume.get('VolumeId'),
+ 'Name': self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='Name'),
+ 'User': self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='User'),
+ 'Days': str(last_detached_days),
+ 'Skip': self._get_tag_name_from_tags(tags=volume.get('Tags'), tag_name='Policy')
+ })
return unattached_volumes_data
diff --git a/cloud_governance/policy/aws/ec2_idle.py b/cloud_governance/policy/aws/ec2_idle.py
index 947086db..d78119dc 100644
--- a/cloud_governance/policy/aws/ec2_idle.py
+++ b/cloud_governance/policy/aws/ec2_idle.py
@@ -116,7 +116,7 @@ def __get_metrics_from_cloud_watch(self, instance_id: str, instance_period: int)
def __get_proposed_metrics(self, metrics: list, metric_period: int):
"""
- This method return the metrics
+ This method returns the metrics
@param metrics:
@param metric_period:
@return:
@@ -125,7 +125,7 @@ def __get_proposed_metrics(self, metrics: list, metric_period: int):
def __get_time_difference(self, launch_time: datetime):
"""
- This method return the difference of datetime
+ This method returns the difference of datetime
@param launch_time:
@return:
"""
diff --git a/cloud_governance/policy/aws/ec2_stop.py b/cloud_governance/policy/aws/ec2_stop.py
index bc558717..6f537898 100644
--- a/cloud_governance/policy/aws/ec2_stop.py
+++ b/cloud_governance/policy/aws/ec2_stop.py
@@ -71,10 +71,15 @@ def __fetch_stop_instance(self, instance_days: int, delete_instance_days: int, s
stopped_instance_tags[instance_id] = resource.get('Tags')
ec2_types[instance_id] = resource.get('InstanceType')
block_device_mappings[instance_id] = resource.get('BlockDeviceMappings')
- stopped_instances.append([resource.get('InstanceId'), self._get_tag_name_from_tags(tags=resource.get('Tags'), tag_name='Name'),
- self._get_tag_name_from_tags(tags=resource.get('Tags'), tag_name='User'), str(resource.get('LaunchTime')),
- self._get_tag_name_from_tags(tags=resource.get('Tags'), tag_name='Policy')
- ])
+ stopped_instances.append({
+ 'policy': self._policy,
+ 'ResourceId': resource.get('InstanceId'),
+ 'StoppedDays': days,
+ 'StoppedDate': str(resource.get('UsageOperationUpdateTime')),
+ 'Name': self._get_tag_name_from_tags(tags=resource.get('Tags'), tag_name='Name'),
+ 'User': self._get_tag_name_from_tags(tags=resource.get('Tags'), tag_name='User'),
+ 'LaunchTime': str(resource.get('LaunchTime')),
+ 'Policy': self._get_tag_name_from_tags(tags=resource.get('Tags'), tag_name='Policy')})
if self._dry_run == "no":
for instance_id, tags in stopped_instance_tags.items():
if self._get_policy_value(tags=tags) not in ('NOTDELETE', 'SKIP'):
diff --git a/cloud_governance/policy/aws/empty_roles.py b/cloud_governance/policy/aws/empty_roles.py
index 2482b81d..72d31fbb 100644
--- a/cloud_governance/policy/aws/empty_roles.py
+++ b/cloud_governance/policy/aws/empty_roles.py
@@ -12,7 +12,7 @@ def __init__(self):
def run(self):
"""
- This method return all empty roles, delete if dry_run no
+ This method returns all empty roles, delete if dry_run no
@return:
"""
return self.__delete_empty_roles()
@@ -40,7 +40,12 @@ def __delete_empty_roles(self):
empty_days = self._get_resource_last_used_days(tags=tags)
empty_role = self._check_resource_and_delete(resource_name='IAM Role', resource_id='RoleName', resource_type='CreateRole', resource=get_role, empty_days=empty_days, days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE, tags=tags)
if empty_role:
- zombie_roles.append([empty_role.get('RoleName'), self._get_tag_name_from_tags(tags=tags, tag_name='User'), self._get_policy_value(tags=tags), empty_days])
+ zombie_roles.append({
+ 'ResourceId': empty_role.get('RoleName'),
+ 'Name': empty_role.get('RoleName'),
+ 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'),
+ 'Skip': self._get_policy_value(tags=tags),
+ 'Days': empty_days})
else:
empty_days = 0
self._update_resource_tags(resource_id=role_name, tags=tags, left_out_days=empty_days, resource_left_out=role_empty)
diff --git a/cloud_governance/policy/aws/ip_unattached.py b/cloud_governance/policy/aws/ip_unattached.py
index c26ca878..8cefd079 100644
--- a/cloud_governance/policy/aws/ip_unattached.py
+++ b/cloud_governance/policy/aws/ip_unattached.py
@@ -13,7 +13,7 @@ def __init__(self):
def run(self):
"""
- This method return zombie elastic_ip's and delete if dry_run no
+ This method returns zombie elastic_ip's and delete if dry_run no
@return:
"""
addresses = self._ec2_operations.get_elastic_ips()
@@ -40,9 +40,12 @@ def run(self):
days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE, tags=tags,
extra_purse=eip_cost, delta_cost=delta_cost)
if zombie_eip:
- zombie_addresses.append([address.get('AllocationId'), self._get_tag_name_from_tags(tags=tags),
- self._get_tag_name_from_tags(tags=tags, tag_name='User'), address.get('PublicIp'),
- self._get_policy_value(tags=tags), unused_days])
+ zombie_addresses.append({'ResourceId': address.get('AllocationId'),
+ 'Name': self._get_tag_name_from_tags(tags=tags),
+ 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'),
+ 'PublicIp': address.get('PublicIp'),
+ 'Skip': self._get_policy_value(tags=tags),
+ 'Days': unused_days})
else:
unused_days = 0
self._update_resource_tags(resource_id=address.get('AllocationId'), tags=tags, left_out_days=unused_days, resource_left_out=ip_no_used)
diff --git a/cloud_governance/policy/aws/monthly_report.py b/cloud_governance/policy/aws/monthly_report.py
index 89a03530..d2b3b203 100644
--- a/cloud_governance/policy/aws/monthly_report.py
+++ b/cloud_governance/policy/aws/monthly_report.py
@@ -26,7 +26,7 @@ def __init__(self):
def policy_description(self, policy_name: str):
"""
- This method return the policy description
+ This method returns the policy description
@param policy_name:
@return:
"""
@@ -35,7 +35,7 @@ def policy_description(self, policy_name: str):
'ec2_idle': 'stops the idle instances in the last 7 days. ( CPU < 5%, Network < 5k )',
'ebs_unattached': 'Delete unattached EBS volumes, where the unused days are calculated by the last DeattachedTime',
'ip_unattached': 'Delete all the elastic_ips that are unused',
- 'nat_gateway_unused': ' Delete all unused nat gateways',
+ 'unused_nat_gateway': ' Delete all unused nat gateways',
'zombie_snapshots': 'Delete all the snapshots which the AMI does not use',
's3_inactive': 'Delete the empty buckets which don’t have any content.',
'empty_roles': 'Delete the empty role which does\'t have any policies',
@@ -82,7 +82,7 @@ def send_monthly_report(self):
def row_span(self, cols: int):
"""
- This method return the table data with colspan
+ This method returns the table data with colspan
@param cols:
@return:
"""
diff --git a/cloud_governance/policy/aws/nat_gateway_unused.py b/cloud_governance/policy/aws/nat_gateway_unused.py
deleted file mode 100644
index 43c5f112..00000000
--- a/cloud_governance/policy/aws/nat_gateway_unused.py
+++ /dev/null
@@ -1,51 +0,0 @@
-
-from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy
-
-
-class NatGatewayUnused(NonClusterZombiePolicy):
- """
- This class sends an alert mail for zombie Nat gateways ( based on vpc routes )
- to the user after 4 days and delete after 7 days.
- """
-
- def __init__(self):
- super().__init__()
-
- def __check_nat_gateway_in_routes(self, nat_gateway_id: str):
- route_tables = self._ec2_client.describe_route_tables()['RouteTables']
- nat_gateway_found = False
- for route_table in route_tables:
- for route in route_table.get('Routes'):
- if route.get('NatGatewayId') == nat_gateway_id:
- nat_gateway_found = True
- return nat_gateway_found
-
- def run(self):
- """
- This method return zombie NatGateways, delete if dry_run no
- @return:
- """
- nat_gateways = self._ec2_operations.get_nat_gateways()
- nat_gateway_unused_data = []
- for nat_gateway in nat_gateways:
- if self._get_policy_value(tags=nat_gateway.get('Tags', [])) not in ('NOTDELETE', 'SKIP'):
- nat_gateway_id = nat_gateway.get('NatGatewayId')
- tags = nat_gateway.get('Tags')
- gateway_unused = False
- if not self._check_cluster_tag(tags=tags):
- if nat_gateway.get('State') == 'available':
- if not self.__check_nat_gateway_in_routes(nat_gateway_id=nat_gateway_id):
- gateway_unused = True
- unused_days = self._get_resource_last_used_days(tags=tags)
- zombie_nat_gateway = self._check_resource_and_delete(resource_name='NatGateway',
- resource_id='NatGatewayId',
- resource_type='CreateNatGateway',
- resource=nat_gateway,
- empty_days=unused_days,
- days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE, tags=tags)
- if zombie_nat_gateway:
- nat_gateway_unused_data.append([nat_gateway_id, self._get_tag_name_from_tags(tags=tags, tag_name='User'), zombie_nat_gateway.get('VpcId'), self._get_policy_value(tags=tags), unused_days])
- else:
- unused_days = 0
- self._update_resource_tags(resource_id=nat_gateway_id, tags=tags, left_out_days=unused_days, resource_left_out=gateway_unused)
- return nat_gateway_unused_data
diff --git a/cloud_governance/policy/aws/s3_inactive.py b/cloud_governance/policy/aws/s3_inactive.py
index 118a36a6..b0ae5eb0 100644
--- a/cloud_governance/policy/aws/s3_inactive.py
+++ b/cloud_governance/policy/aws/s3_inactive.py
@@ -15,7 +15,7 @@ def __init__(self):
def run(self):
"""
- This method return all Empty buckets and delete if dry_run no
+ This method returns all Empty buckets and delete if dry_run no
@return:
"""
return self.__delete_s3_inactive()
@@ -49,7 +49,7 @@ def __delete_s3_inactive(self):
self._cloudtrail.set_cloudtrail(region_name=region)
empty_bucket = self._check_resource_and_delete(resource_name='S3 Bucket', resource_id='Name', resource_type='CreateBucket', resource=bucket, empty_days=empty_days, days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE, tags=tags)
if empty_bucket:
- empty_buckets.append([bucket.get('Name'), self._get_tag_name_from_tags(tags=tags, tag_name='User'), str(bucket.get('CreationDate')), str(empty_days), self._get_policy_value(tags=tags)])
+ empty_buckets.append({'ResourceId': bucket.get('Name'), 'Name': bucket.get('Name'), 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'), 'Date': str(bucket.get('CreationDate')), 'Days': str(empty_days), 'Skip': self._get_policy_value(tags=tags)})
else:
empty_days = 0
self._update_resource_tags(resource_id=bucket_name, tags=tags, left_out_days=empty_days, resource_left_out=bucket_empty)
diff --git a/cloud_governance/policy/aws/skipped_resources.py b/cloud_governance/policy/aws/skipped_resources.py
index 60c22432..8bcfdd24 100644
--- a/cloud_governance/policy/aws/skipped_resources.py
+++ b/cloud_governance/policy/aws/skipped_resources.py
@@ -28,7 +28,7 @@ def get_volume_type_prices(self):
def get_resources(self, resource_name: str):
"""
- This method return resource data based on resource name
+ This method returns resource data based on resource name
@param resource_name:
@return:
"""
@@ -45,7 +45,7 @@ def get_resources(self, resource_name: str):
def get_ebs_cost(self, volume_id: str):
"""
- This method return the size of the ebs_volume
+ This method returns the size of the ebs_volume
@param volume_id:
@return:
"""
@@ -57,7 +57,7 @@ def get_ebs_cost(self, volume_id: str):
def get_instance_volume_size(self, resource: dict):
"""
- This method return size of the attached volumes of the instance
+ This method returns size of the attached volumes of the instance
@param resource:
@return:
"""
@@ -106,7 +106,7 @@ def get_not_delete_resources(self):
def run(self):
"""
- This method return all tag "Not_Delete" or "skip" resources
+ This method returns all tag "Not_Delete" or "skip" resources
@return:
"""
resources_data = self.get_not_delete_resources()
diff --git a/cloud_governance/policy/aws/spot_savings_analysis.py b/cloud_governance/policy/aws/spot_savings_analysis.py
new file mode 100644
index 00000000..5c0d7e40
--- /dev/null
+++ b/cloud_governance/policy/aws/spot_savings_analysis.py
@@ -0,0 +1,116 @@
+
+from datetime import datetime
+
+import typeguard
+
+from cloud_governance.common.clouds.aws.athena.pyathena_operations import PyAthenaOperations
+from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations
+from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class SpotSavingsAnalysis:
+ """
+ This class contain the spt savings analysis reports from the athena query
+ that are gathered from the AWS Cost and Usage Reports.
+ To get reports from the athena:
+ 1. Enable the cost-and-usage reports with support of athena integration
+ 2. Create a Database and table of CUR
+ """
+
+ def __init__(self):
+ self.__environment_variables_dict = environment_variables.environment_variables_dict
+ self.__default_round_digits = self.__environment_variables_dict.get('DEFAULT_ROUND_DIGITS')
+ self.__es_index = self.__environment_variables_dict.get('es_index')
+ self.__database_name = self.__environment_variables_dict.get('ATHENA_DATABASE_NAME')
+ self.__table_name = self.__environment_variables_dict.get('ATHENA_TABLE_NAME')
+ self.__es_operations = ElasticSearchOperations()
+
+ def __get_prepared_query(self):
+ """
+ This method prepare the query
+ :return:
+ """
+ current_date = datetime.utcnow()
+ year = current_date.year
+ current_month = current_date.month
+ previous_month = current_month - 1 if current_month - 1 != 0 else 12
+ query = f"""
+ SELECT
+ date_format(line_item_usage_start_date, '%Y-%m-%d') as CurrentDate,
+ date_format(bill_billing_period_start_date, '%Y-%m-%d') as MonthStartDate,
+ line_item_usage_account_id as AccountId,
+ line_item_product_code as ProductCode,
+ product_region as Region,
+ product_instance_type as InstanceType,
+ cost_category_cost_center as CostCenter,
+ cost_category_o_us as CostCategory,
+ cost_category_organization as RHOrg,
+ ROUND(SUM(discount_total_discount), 3) as TotalDiscount,
+ ROUND(SUM(line_item_usage_amount), {self.__default_round_digits}) as UsageAmount,
+ ROUND(SUM(line_item_unblended_cost + discount_total_discount), {self.__default_round_digits}) as UnblendedCost,
+ ROUND(SUM(pricing_public_on_demand_cost), {self.__default_round_digits}) as OnDemand,
+ ROUND(SUM(pricing_public_on_demand_cost - line_item_unblended_cost), {self.__default_round_digits}) as SpotSavings
+ FROM "{self.__database_name}"."{self.__table_name}"
+ WHERE "product_product_name" = 'Amazon Elastic Compute Cloud'
+ AND "line_item_resource_id" LIKE 'i-%'
+ AND "line_item_operation" LIKE 'RunInstance%'
+ AND "product_marketoption" = 'Spot'
+ AND month(bill_billing_period_start_date) in ({previous_month}, {current_month})
+ AND year(bill_billing_period_start_date) = {year}
+ AND pricing_public_on_demand_cost <> 0
+ GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9 ORDER BY MonthStartDate desc
+ """
+ return query
+
+ def __get_query_results(self):
+ """
+ This method get queries data
+ :return:
+ """
+ result = {}
+ query_string = self.__get_prepared_query()
+ if query_string:
+ athena_operations = PyAthenaOperations()
+ result = athena_operations.execute_query(query_string)
+ else:
+ logger.debug(f"query string is not provided, exit without query execution")
+ return result
+
+ @typeguard.typechecked
+ def __get_data_to_upload_to_es(self, athena_data_dictionary: list):
+ """
+ This method returns ready upload dict to upload to elasticsearch
+ :return:
+ """
+ for row_dict in athena_data_dictionary:
+ month_start_date = row_dict.get('MonthStartDate')
+ month_name_year = datetime.strftime(datetime.strptime(month_start_date, '%Y-%m-%d'), '%Y %b')
+ row_dict['Month'] = month_name_year
+ row_dict['filter_date'] = f'{month_start_date}-{month_name_year.split()[-1]}'
+ row_dict['AccountIdCostCenter'] = f'{row_dict.get("AccountId")}-{row_dict.get("CostCenter")}'
+ row_dict['index_id'] = f'{row_dict.get("CurrentDate")}-' \
+ f'{row_dict.get("AccountId")}-' \
+ f'{row_dict.get("Region")}-{row_dict.get("InstanceType")}'
+ row_dict['AWSCostCenter'] = f'AWS-{row_dict.get("CostCenter")}'
+
+ @logger_time_stamp
+ def __collect_reports_and_upload_es(self):
+ """
+ This method collects the data and uploads to elastic search
+ :return:
+ """
+ query_result = self.__get_query_results()
+ if query_result:
+ self.__get_data_to_upload_to_es(athena_data_dictionary=query_result)
+ if query_result:
+ self.__es_operations.upload_data_in_bulk(data_items=query_result, id='index_id', index=self.__es_index)
+
+ @logger_time_stamp
+ def run(self):
+ """
+ This is the starting of the methods
+ :return:
+ """
+ self.__collect_reports_and_upload_es()
diff --git a/cloud_governance/policy/aws/unused_nat_gateway.py b/cloud_governance/policy/aws/unused_nat_gateway.py
new file mode 100644
index 00000000..cdbb214b
--- /dev/null
+++ b/cloud_governance/policy/aws/unused_nat_gateway.py
@@ -0,0 +1,92 @@
+import datetime
+
+
+from cloud_governance.common.clouds.aws.cloudwatch.cloudwatch_operations import CloudWatchOperations
+from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy
+
+
+class UnusedNatGateway(NonClusterZombiePolicy):
+ """
+ This class sends an alert mail for zombie Nat gateways ( based on vpc routes )
+ to the user after 4 days and delete after 7 days.
+ """
+
+ NAMESPACE = 'AWS/NATGateway'
+ UNUSED_DAYS = 1
+
+ def __init__(self):
+ super().__init__()
+ self._cloudwatch = CloudWatchOperations(region=self._region)
+
+ def __check_cloud_watch_logs(self, resource_id: str, days: int = UNUSED_DAYS):
+ """
+ This method returns weather the NatGateway is used in last input days
+ :param resource_id:
+ :param days:
+ :return:
+ """
+ if days == 0:
+ days = 1
+ end_time = datetime.datetime.utcnow()
+ start_time = end_time - datetime.timedelta(days=days)
+ response = self._cloudwatch.get_metric_data(start_time=start_time, end_time=end_time, resource_id=resource_id,
+ resource_type='NatGatewayId', namespace=self.NAMESPACE,
+ metric_names={'ActiveConnectionCount': 'Count'},
+ statistic='Average')['MetricDataResults'][0]
+ for value in response.get('Values'):
+ if value > 0:
+ return False
+ return True
+
+ def __check_nat_gateway_in_routes(self, nat_gateway_id: str):
+ """
+ This method check the nat gateway present in the routes or not.
+ :param nat_gateway_id:
+ :return:
+ """
+ route_tables = self._ec2_client.describe_route_tables()['RouteTables']
+ nat_gateway_found = False
+ for route_table in route_tables:
+ for route in route_table.get('Routes'):
+ if route.get('NatGatewayId') == nat_gateway_id:
+ nat_gateway_found = True
+ return nat_gateway_found
+
+ def run(self):
+ """
+ This method returns zombie NatGateways, delete if dry_run no
+ @return:
+ """
+ nat_gateways = self._ec2_operations.get_nat_gateways()
+ nat_gateway_unused_data = []
+ for nat_gateway in nat_gateways:
+ if self._get_policy_value(tags=nat_gateway.get('Tags', [])) not in ('NOTDELETE', 'SKIP'):
+ nat_gateway_id = nat_gateway.get('NatGatewayId')
+ tags = nat_gateway.get('Tags')
+ gateway_unused = False
+ last_used_days = int(self._ec2_operations.get_tag_value_from_tags(tags=tags, tag_name='LastUsedDay', default_value=1))
+ if not self._check_cluster_tag(tags=tags):
+ if nat_gateway.get('State') == 'available':
+ if not self.__check_nat_gateway_in_routes(nat_gateway_id=nat_gateway_id) or self.__check_cloud_watch_logs(days=last_used_days, resource_id=nat_gateway_id):
+ gateway_unused = True
+ unused_days = self._get_resource_last_used_days(tags=tags)
+ zombie_nat_gateway = self._check_resource_and_delete(resource_name='NatGateway',
+ resource_id='NatGatewayId',
+ resource_type='CreateNatGateway',
+ resource=nat_gateway,
+ empty_days=unused_days,
+ days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE,
+ tags=tags)
+ if zombie_nat_gateway:
+ nat_gateway_unused_data.append(
+ {'ResourceId': nat_gateway_id,
+ 'Name': self._get_tag_name_from_tags(tags=tags, tag_name='Name'),
+ 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'),
+ 'VpcId': zombie_nat_gateway.get('VpcId'),
+ 'Skip': self._get_policy_value(tags=tags),
+ 'Days': unused_days, 'Policy': self._policy})
+ else:
+ unused_days = 0
+ self._update_resource_tags(resource_id=nat_gateway_id, tags=tags, left_out_days=unused_days,
+ resource_left_out=gateway_unused)
+ return nat_gateway_unused_data
diff --git a/cloud_governance/policy/aws/zombie_cluster_resource.py b/cloud_governance/policy/aws/zombie_cluster_resource.py
index 00743617..203e9fac 100644
--- a/cloud_governance/policy/aws/zombie_cluster_resource.py
+++ b/cloud_governance/policy/aws/zombie_cluster_resource.py
@@ -38,7 +38,7 @@ def __init__(self, cluster_prefix: str = None, delete: bool = False, region: str
def all_cluster_instance(self):
"""
- This method return list of cluster's instance tag name that contains openshift tag prefix from all regions
+ This method returns list of cluster's instance tag name that contains openshift tag prefix from all regions
:return: list of cluster's instance tag name
"""
instances_list = []
@@ -63,7 +63,7 @@ def all_cluster_instance(self):
def _cluster_instance(self):
"""
- This method return list of cluster's instance tag name that contains openshift tag prefix
+ This method returns list of cluster's instance tag name that contains openshift tag prefix
:return: list of cluster's instance tag name
"""
instances_list = []
@@ -85,7 +85,7 @@ def _cluster_instance(self):
def __get_cluster_resources(self, resources_list: list, input_resource_id: str, tags: str = 'Tags'):
"""
- This method return all cluster resources keys that start with cluster prefix
+ This method returns all cluster resources keys that start with cluster prefix
:param resources_list:
:param tags:
:return: dictionary of the resources key and id
@@ -181,7 +181,7 @@ def __get_all_zombie_resources(self, exist_resources: dict):
def zombie_cluster_volume(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's volume according to cluster tag name and cluster name data
+ This method returns list of cluster's volume according to cluster tag name and cluster name data
delete only available resource that related to cluster
"""
available_volumes = []
@@ -200,14 +200,14 @@ def zombie_cluster_volume(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
if delete_cluster_resource and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource_id=zombie, resource='ec2_volume')
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource_id=zombie, resource='ec2_volume')
return zombies, cluster_left_out_days
def zombie_cluster_ami(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's ami according to cluster tag name and cluster name data
+ This method returns list of cluster's ami according to cluster tag name and cluster name data
"""
images_data = self.ec2_operations.get_images()
exist_ami = self.__get_cluster_resources(resources_list=images_data, input_resource_id='ImageId')
@@ -225,7 +225,7 @@ def zombie_cluster_ami(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
self.ec2_client.deregister_image(ImageId=zombie)
logger.info(f'deregister_image: {zombie}')
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.ec2_client.deregister_image(ImageId=zombie)
logger.info(f'deregister_image: {zombie}')
except Exception as err:
@@ -234,7 +234,7 @@ def zombie_cluster_ami(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
def zombie_cluster_snapshot(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's snapshot according to cluster tag name and cluster name data
+ This method returns list of cluster's snapshot according to cluster tag name and cluster name data
"""
snapshots_data = self.ec2_operations.get_snapshots()
exist_snapshot = self.__get_cluster_resources(resources_list=snapshots_data, input_resource_id='SnapshotId')
@@ -247,7 +247,7 @@ def zombie_cluster_snapshot(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
if delete_cluster_resource and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='ebs_snapshots', resource_id=zombie)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='ebs_snapshots', resource_id=zombie)
return zombies, cluster_left_out_days
@@ -318,7 +318,7 @@ def __get_zombies_by_vpc_id(self, vpc_id: str, resources: list, output_tag: str,
def zombie_cluster_security_group(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of zombie cluster's security groups compare to existing instances and cluster name data
+ This method returns list of zombie cluster's security groups compare to existing instances and cluster name data
:return: list of zombie cluster's security groups
"""
security_groups = self.ec2_operations.get_security_groups()
@@ -338,7 +338,7 @@ def zombie_cluster_security_group(self, vpc_id: str = '', cluster_tag_vpc: str =
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
@@ -346,7 +346,7 @@ def zombie_cluster_security_group(self, vpc_id: str = '', cluster_tag_vpc: str =
def zombie_cluster_elastic_ip(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of zombie cluster's elastic ip according to existing instances and cluster name data
+ This method returns list of zombie cluster's elastic ip according to existing instances and cluster name data
"""
exist_elastic_ip_association = []
exist_elastic_ip_allocation = []
@@ -377,14 +377,14 @@ def zombie_cluster_elastic_ip(self, vpc_id: str = '', cluster_tag_vpc: str = '')
if delete_cluster_resource and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, cluster_tag=cluster_tag)
zombies = {**zombies_all}
return zombies, cluster_left_out_days
def zombie_cluster_network_interface(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of zombie cluster's network interface according to existing instances and cluster name data
+ This method returns list of zombie cluster's network interface according to existing instances and cluster name data
"""
network_interfaces_data = self.ec2_operations.get_network_interface()
exist_network_interface = self.__get_cluster_resources(resources_list=network_interfaces_data, input_resource_id='NetworkInterfaceId', tags='TagSet')
@@ -406,14 +406,14 @@ def zombie_cluster_network_interface(self, vpc_id: str = '', cluster_tag_vpc: st
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', resource_id=zombie_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', resource_id=zombie_id, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_load_balancer(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's load balancer according to cluster vpc and cluster name data
+ This method returns list of cluster's load balancer according to cluster vpc and cluster name data
"""
exist_load_balancer = {}
@@ -441,13 +441,13 @@ def zombie_cluster_load_balancer(self, vpc_id: str = '', cluster_tag_vpc: str =
if delete_cluster_resource and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_load_balancer_v2(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's load balancer according to cluster vpc and cluster name data
+ This method returns list of cluster's load balancer according to cluster vpc and cluster name data
"""
exist_load_balancer = {}
load_balancers_data = self.ec2_operations.get_load_balancers_v2()
@@ -474,13 +474,13 @@ def zombie_cluster_load_balancer_v2(self, vpc_id: str = '', cluster_tag_vpc: str
if delete_cluster_resource and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def __get_all_exist_vpcs(self):
"""
- This method return all exist vpc ids (for supporting Network ACL - missing OpenShift tags)
+ This method returns all exist vpc ids (for supporting Network ACL - missing OpenShift tags)
:return:
"""
vpcs_data = self.ec2_operations.get_vpcs()
@@ -491,7 +491,7 @@ def __get_all_exist_vpcs(self):
def zombie_cluster_vpc(self):
"""
- This method return list of cluster's vpc according to cluster tag name and cluster name data
+ This method returns list of cluster's vpc according to cluster tag name and cluster name data
"""
vpcs_data = self.ec2_operations.get_vpcs()
exist_vpc = self.__get_cluster_resources(resources_list=vpcs_data, input_resource_id='VpcId')
@@ -510,14 +510,14 @@ def zombie_cluster_vpc(self):
if delete_cluster_resource and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, pending_resources=delete_dict, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, pending_resources=delete_dict, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_subnet(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's subnet according to cluster tag name and cluster name data
+ This method returns list of cluster's subnet according to cluster tag name and cluster name data
"""
subnets_data = self.ec2_operations.get_subnets()
exist_subnet = self.__get_cluster_resources(resources_list=subnets_data, input_resource_id='SubnetId')
@@ -537,14 +537,14 @@ def zombie_cluster_subnet(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_route_table(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's route table according to cluster tag name and cluster name data
+ This method returns list of cluster's route table according to cluster tag name and cluster name data
"""
route_tables_data = self.ec2_operations.get_route_tables()
exist_route_table = self.__get_cluster_resources(resources_list=route_tables_data, input_resource_id='RouteTableId')
@@ -564,14 +564,14 @@ def zombie_cluster_route_table(self, vpc_id: str = '', cluster_tag_vpc: str = ''
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='route_table', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='route_table', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_internet_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's route table internet gateway according to cluster tag name and cluster name data
+ This method returns list of cluster's route table internet gateway according to cluster tag name and cluster name data
"""
internet_gateways_data = self.ec2_operations.get_internet_gateways()
exist_internet_gateway = self.__get_cluster_resources(resources_list=internet_gateways_data, input_resource_id='InternetGatewayId')
@@ -593,7 +593,7 @@ def zombie_cluster_internet_gateway(self, vpc_id: str = '', cluster_tag_vpc: str
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
@@ -601,7 +601,7 @@ def zombie_cluster_internet_gateway(self, vpc_id: str = '', cluster_tag_vpc: str
def zombie_cluster_dhcp_option(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's dhcp option according to cluster tag name and cluster name data
+ This method returns list of cluster's dhcp option according to cluster tag name and cluster name data
"""
dhcp_options_data = self.ec2_operations.get_dhcp_options()
exist_dhcp_option = self.__get_cluster_resources(resources_list=dhcp_options_data, input_resource_id='DhcpOptionsId')
@@ -619,7 +619,7 @@ def zombie_cluster_dhcp_option(self, vpc_id: str = '', cluster_tag_vpc: str = ''
else:
self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
if vpc_id:
self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie, vpc_id=vpc_id)
else:
@@ -628,7 +628,7 @@ def zombie_cluster_dhcp_option(self, vpc_id: str = '', cluster_tag_vpc: str = ''
def zombie_cluster_vpc_endpoint(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of cluster's vpc endpoint according to cluster tag name and cluster name data
+ This method returns list of cluster's vpc endpoint according to cluster tag name and cluster name data
"""
vpc_endpoints_data = self.ec2_operations.get_vpce()
exist_vpc_endpoint = self.__get_cluster_resources(resources_list=vpc_endpoints_data, input_resource_id='VpcEndpointId')
@@ -650,14 +650,14 @@ def zombie_cluster_vpc_endpoint(self, vpc_id: str = '', cluster_tag_vpc: str = '
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', resource_id=zombie_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', resource_id=zombie_id, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_nat_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of zombie cluster's nat gateway according to cluster tag name and cluster name data
+ This method returns list of zombie cluster's nat gateway according to cluster tag name and cluster name data
"""
nat_gateways_data = self.ec2_operations.get_nat_gateways()
exist_nat_gateway = self.__get_cluster_resources(resources_list=nat_gateways_data, input_resource_id='NatGatewayId')
@@ -676,14 +676,14 @@ def zombie_cluster_nat_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = ''
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', resource_id=zombie_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', resource_id=zombie_id, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_network_acl(self, vpc_id: str = '', cluster_tag_vpc: str = ''):
"""
- This method return list of zombie cluster's network acl according to existing vpc id and cluster name data
+ This method returns list of zombie cluster's network acl according to existing vpc id and cluster name data
"""
exist_network_acl = {}
network_acls_data = self.ec2_operations.get_nacls()
@@ -713,14 +713,14 @@ def zombie_cluster_network_acl(self, vpc_id: str = '', cluster_tag_vpc: str = ''
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
for zombie_id in zombie_ids:
self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag)
return zombies, cluster_left_out_days
def zombie_cluster_role(self):
"""
- This method return list of cluster's role in all regions according to cluster name and cluster name data
+ This method returns list of cluster's role in all regions according to cluster name and cluster name data
* Role is a global resource, need to scan for live cluster in all regions
"""
exist_role_name_tag = {}
@@ -751,13 +751,13 @@ def zombie_cluster_role(self):
if delete_cluster_resource and self.delete:
self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_role')
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_role')
return zombies, cluster_left_out_days
def zombie_cluster_user(self):
"""
- This method return list of cluster's user according to cluster name and cluster name data
+ This method returns list of cluster's user according to cluster name and cluster name data
* User is a global resource, need to scan for live cluster in all regions
"""
exist_user_name_tag = {}
@@ -785,14 +785,14 @@ def zombie_cluster_user(self):
if delete_cluster_resource and self.delete:
self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_user')
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_user')
return zombies, cluster_left_out_days
def zombie_cluster_s3_bucket(self, cluster_stamp: str = 'image-registry'):
"""
- This method return list of cluster's s3 bucket according to cluster name and cluster name data
+ This method returns list of cluster's s3 bucket according to cluster name and cluster name data
* S3 is a global resource, need to scan for live cluster in all regions
"""
exist_bucket_name_tag = {}
@@ -824,7 +824,7 @@ def zombie_cluster_s3_bucket(self, cluster_stamp: str = 'image-registry'):
if delete_cluster_resource and self.delete:
self.delete_s3_resource.delete_zombie_s3_resource(resource_type='s3_bucket', resource_id=zombie)
else:
- if self._force_delete:
+ if self._force_delete and self.delete:
self.delete_s3_resource.delete_zombie_s3_resource(resource_type='s3_bucket', resource_id=zombie)
return zombies, cluster_left_out_days
diff --git a/cloud_governance/policy/aws/zombie_snapshots.py b/cloud_governance/policy/aws/zombie_snapshots.py
index 7a8ca2d7..58b15873 100644
--- a/cloud_governance/policy/aws/zombie_snapshots.py
+++ b/cloud_governance/policy/aws/zombie_snapshots.py
@@ -25,7 +25,7 @@ def _get_image_ids_from_description(self, snapshot_description: str):
def run(self):
"""
- This method return all the zombie snapshots, delete if dry_run no
+ This method returns all the zombie snapshots, delete if dry_run no
@return:
"""
snapshots = self._ec2_operations.get_snapshots()
@@ -51,12 +51,13 @@ def run(self):
days_to_delete_resource=self.DAYS_TO_DELETE_RESOURCE,
tags=tags)
if zombie_snapshot:
- zombie_snapshots.append([snapshot.get('SnapshotId'),
- self._get_tag_name_from_tags(tags=tags),
- self._get_tag_name_from_tags(tags=tags, tag_name='User'),
- f'{str(snapshot.get("VolumeSize"))}Gb',
- self._get_policy_value(tags=snapshot.get('Tags')), str(unused_days)
- ])
+ zombie_snapshots.append({'ResourceId': snapshot.get('SnapshotId'),
+ 'Name': self._get_tag_name_from_tags(tags=tags),
+ 'User': self._get_tag_name_from_tags(tags=tags, tag_name='User'),
+ 'Size': f'{str(snapshot.get("VolumeSize"))}Gb',
+ 'Skip': self._get_policy_value(tags=snapshot.get('Tags')),
+ 'Days': str(unused_days)
+ })
else:
unused_days = 0
self._update_resource_tags(resource_id=snapshot_id, tags=tags, left_out_days=unused_days,
diff --git a/cloud_governance/policy/azure/cost_billing_reports.py b/cloud_governance/policy/azure/cost_billing_reports.py
index 2a80a74a..ad7c11fa 100644
--- a/cloud_governance/policy/azure/cost_billing_reports.py
+++ b/cloud_governance/policy/azure/cost_billing_reports.py
@@ -17,9 +17,6 @@ class CostBillingReports:
This class is responsible for generation cost billing report for Budget, Actual, Forecast
"""
- COST_CENTER_OWNER = 'Shai'
- COST_CENTER_OWNER_OTHERS = 'Others'
-
def __init__(self):
self.__environment_variables_dict = environment_variables.environment_variables_dict
self.__total_account = self.__environment_variables_dict.get('TOTAL_ACCOUNTS', '')
@@ -29,7 +26,7 @@ def __init__(self):
self.gdrive_operations = GoogleDriveOperations()
self.__gsheet_id = self.__environment_variables_dict.get('SPREADSHEET_ID')
self.update_to_gsheet = UploadToGsheet()
- self.__cost_center, self.__allocated_budget, self.__years = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.azure_operations.subscription_id, dir_path='/tmp')
+ self.__cost_center, self.__allocated_budget, self.__years, self.__owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.azure_operations.subscription_id, dir_path='/tmp')
self.__common_data = self.get_common_data()
def get_common_data(self):
@@ -44,7 +41,7 @@ def get_common_data(self):
upload_data['AllocatedBudget'] = 0
upload_data['CostCenter'] = int(self.__cost_center)
upload_data['CloudName'] = self.azure_operations.cloud_name
- upload_data['Owner'] = self.COST_CENTER_OWNER_OTHERS
+ upload_data['Owner'] = self.__owner
return upload_data
@logger_time_stamp
@@ -66,12 +63,12 @@ def get_data_from_costs(self, cost_data_rows: list, cost_data_columns: list, cos
if cost_center > 0:
common_data['CostCenter'] = cost_center
if subscription_id:
- cost_center, allocated_budget, years = self.update_to_gsheet.get_cost_center_budget_details(account_id=subscription_id, dir_path='/tmp')
+ cost_center, allocated_budget, years, owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=subscription_id, dir_path='/tmp')
if cost_center:
common_data['CostCenter'] = int(cost_center)
- common_data['Owner'] = self.COST_CENTER_OWNER
+ common_data['Owner'] = owner
else:
- common_data['Owner'] = self.COST_CENTER_OWNER_OTHERS
+ common_data['Owner'] = owner
else:
allocated_budget, years = self.__allocated_budget, self.__years
for index, item in enumerate(cost_data_rows):
@@ -87,12 +84,12 @@ def get_data_from_costs(self, cost_data_rows: list, cost_data_columns: list, cos
common_data['Account'] = item[key]
elif column.get('name') == 'SubscriptionId':
common_data['AccountId'] = item[key]
- cost_center, allocated_budget, years = self.update_to_gsheet.get_cost_center_budget_details(account_id=item[key], dir_path='/tmp')
+ cost_center, allocated_budget, years, owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=item[key], dir_path='/tmp')
if cost_center:
common_data['CostCenter'] = int(cost_center)
- common_data['Owner'] = self.COST_CENTER_OWNER
+ common_data['Owner'] = owner
else:
- common_data['Owner'] = self.COST_CENTER_OWNER_OTHERS
+ common_data['Owner'] = owner
else:
if column.get('type') == 'Datetime':
start_date = item[key].split('T')[0]
diff --git a/cloud_governance/policy/common_policies/__init__.py b/cloud_governance/policy/common_policies/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloud_governance/policy/common_policies/send_aggregated_alerts.py b/cloud_governance/policy/common_policies/send_aggregated_alerts.py
new file mode 100644
index 00000000..741a7444
--- /dev/null
+++ b/cloud_governance/policy/common_policies/send_aggregated_alerts.py
@@ -0,0 +1,159 @@
+import json
+import logging
+import os
+import tempfile
+from datetime import date, datetime, timedelta
+
+import typeguard
+from botocore.exceptions import ClientError
+
+from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations
+from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
+from cloud_governance.common.jira.jira import logger
+from cloud_governance.common.logger.init_logger import handler
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+from cloud_governance.common.mails.mail_message import MailMessage
+from cloud_governance.common.mails.postfix import Postfix
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class SendAggregatedAlerts:
+ """
+ This class send alerts to users which conditions are not satisfied by the policies
+ """
+
+ FILE_NAME = 'resources.json'
+ GLOBAL_REGION = 'us-east-1'
+ TODAY_DATE = str(date.today()).replace('-', '/')
+
+ def __init__(self):
+ self.__environment_variables = environment_variables.environment_variables_dict
+ self.__bucket_name = self.__environment_variables.get('BUCKET_NAME')
+ self.__bucket_key = self.__environment_variables.get('BUCKET_KEY')
+ self.__policies = self.__environment_variables.get('POLICIES_TO_ALERT')
+ self.__s3_operations = S3Operations(region_name='us-east-2', bucket=self.__bucket_name, logs_bucket_key=self.__bucket_key)
+ self.__active_regions = EC2Operations().get_active_regions()
+ self.__kerberos_users = self.__get_kerberos_users_for_iam_users()
+ self.__global_region_policies = ['s3-inactive', 'empty-roles']
+ self.__mail_alert_days = self.__environment_variables.get('MAIL_ALERT_DAYS')
+ self.__policy_action_days = self.__environment_variables.get('POLICY_ACTIONS_DAYS')
+ self.__mail_message = MailMessage()
+ self.__postfix = Postfix()
+
+ @logger_time_stamp
+ def __get_kerberos_users_for_iam_users(self):
+ """
+ This method returns the users which IAM users are not kerberos username
+ :return:
+ """
+ responses = {}
+ users = self.__environment_variables.get('KERBEROS_USERS')
+ for iam_user, kerberos_user in users.items():
+ responses[iam_user.lower()] = kerberos_user.lower()
+ return responses
+
+ def __get_users_agg_result(self, policy_result: list, agg_users_result: dict, policy_name: str, region: str):
+ """
+ This method returns the aggregated users resources list
+ :param agg_users_result:
+ :param policy_result:
+ :return:
+ """
+ if policy_result:
+ for response in policy_result:
+ if type(response) == dict:
+ skip_policy = response.get('Skip')
+ if skip_policy in ('NA', '', None):
+ user = response.pop('User').lower()
+ response['Region'] = region
+ response['Policy'] = policy_name
+ if user in self.__kerberos_users.keys():
+ user = self.__kerberos_users.get(user)
+ agg_users_result.setdefault(user, []).append(response)
+
+ def __get_policy_data_in_bucket(self, region: str, policy: str):
+ """
+ This method returns the policy data in s3 bucket
+ :param region:
+ :param policy:
+ :return:
+ """
+ try:
+ policy_save_path = f'{self.__bucket_key}/{region}/{policy}'
+ bucket_path_file = self.__s3_operations.get_last_objects(bucket=self.__bucket_name, key_prefix=f'{policy_save_path}/{self.TODAY_DATE}')
+ policy_s3_response = self.__s3_operations.get_last_s3_policy_content(s3_file_path=bucket_path_file, file_name=self.FILE_NAME)
+ return json.loads(policy_s3_response) if policy_s3_response else []
+ except ClientError as err:
+ logger.info(err)
+ return []
+
+ @logger_time_stamp
+ def __get_policy_users_list(self):
+ """
+ This method gets the latest policy responses
+ :return:
+ """
+ agg_users_result = {}
+ for policy in self.__policies:
+ run_global_region = True if policy in self.__global_region_policies else False
+ for region in self.__active_regions:
+ if (region == self.GLOBAL_REGION and run_global_region) or not run_global_region:
+ self.__get_users_agg_result(policy_result=self.__get_policy_data_in_bucket(region=region, policy=policy),
+ agg_users_result=agg_users_result, policy_name=policy, region=region)
+ if region == self.GLOBAL_REGION and run_global_region:
+ break
+ return agg_users_result
+
+ def __get_policy_agg_data_by_region(self, policy_data: dict):
+ """
+ This method returns the policy data agg by region
+ :param policy_data:
+ :return:
+ """
+ agg_policy_region_result = {}
+ for policy_name, policy_region_data in policy_data.items():
+ agg_policy_region_result[policy_name] = {}
+ for region_data in policy_region_data:
+ region_name = region_data.get('Region').lower()
+ agg_policy_region_result[policy_name].setdefault(region_name, []).append(region_data)
+ return agg_policy_region_result
+
+ @logger_time_stamp
+ def __get_policy_agg_data(self, user_policy_data: list):
+ """
+ This method returns the data agg by policy
+ :param user_policy_data:
+ :return:
+ """
+ agg_policy_result = {}
+ for result in user_policy_data:
+ policy_name = result.get('Policy').lower()
+ days = int(result.get('Days', 0))
+ if days in self.__mail_alert_days or days in self.__policy_action_days:
+ result['Action'] = 'Deleted' if days in self.__policy_action_days else 'Monitoring'
+ result['DeletedDay'] = (datetime.now() + timedelta(days=self.__policy_action_days[0] - days)).date()
+ agg_policy_result.setdefault(policy_name, []).append(result)
+ return self.__get_policy_agg_data_by_region(policy_data=agg_policy_result)
+
+ @logger_time_stamp
+ def __send_mail_alerts_to_users(self):
+ """
+ This method send mail alerts to users
+ :return:
+ """
+ policy_agg_users_list = self.__get_policy_users_list()
+ for user, user_policy_data in policy_agg_users_list.items():
+ handler.setLevel(logging.WARN)
+ agg_policy_data = self.__get_policy_agg_data(user_policy_data=user_policy_data)
+ if agg_policy_data:
+ handler.setLevel(logging.INFO)
+ subject, body = self.__mail_message.get_agg_policies_mail_message(user=user, user_resources=agg_policy_data)
+ self.__postfix.send_email_postfix(subject=subject, content=body, to=user, cc=[], mime_type='html')
+
+ @logger_time_stamp
+ def run(self):
+ """
+ This method start the other methods
+ :return:
+ """
+ self.__send_mail_alerts_to_users()
diff --git a/cloud_governance/policy/gcp/__init__.py b/cloud_governance/policy/gcp/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloud_governance/policy/gcp/cost_billing_reports.py b/cloud_governance/policy/gcp/cost_billing_reports.py
new file mode 100644
index 00000000..668a54fd
--- /dev/null
+++ b/cloud_governance/policy/gcp/cost_billing_reports.py
@@ -0,0 +1,293 @@
+import json
+import os
+from datetime import datetime, timedelta
+from ast import literal_eval
+
+from typeguard import typechecked
+
+from cloud_governance.common.clouds.gcp.google_account import GoogleAccount
+from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
+from cloud_governance.common.google_drive.upload_to_gsheet import UploadToGsheet
+from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
+from cloud_governance.main.environment_variables import environment_variables
+from cloud_governance.common.logger.init_logger import logger
+
+
+class CostBillingReports:
+ """
+ This class is responsible for generation cost billing report for Budget, Actual, Forecast
+ """
+
+ DEFAULT_YEARS = 12
+ DEFAULT_ROUND_DIGITS = 3
+
+ def __init__(self):
+ self.__environment_variables_dict = environment_variables.environment_variables_dict
+ self.__database_name = self.__environment_variables_dict.get('GCP_DATABASE_NAME', '')
+ self.__table_name = self.__environment_variables_dict.get('GCP_DATABASE_TABLE_NAME', '')
+ self.__gcp_account = GoogleAccount()
+ self.__gsheet_id = self.__environment_variables_dict.get('SPREADSHEET_ID')
+ self.__cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper()
+ self.update_to_gsheet = UploadToGsheet()
+ self.elastic_upload = ElasticUpload()
+
+ @logger_time_stamp
+ def __next_twelve_months(self):
+ """
+ This method returns the next 12 month, year
+ :return:
+ """
+ months = 12
+ year = datetime.now().year
+ next_month = datetime.now().month + 1
+ month_year = []
+ for idx in range(months):
+ month = str((idx+next_month) % months)
+ c_year = year
+ if len(month) == 1:
+ month = f'0{month}'
+ if month == '00':
+ month = 12
+ year = year+1
+ month_year.append((str(month), c_year))
+ return month_year
+
+ @typechecked()
+ @logger_time_stamp
+ def __prepare_usage_query(self, first_year_month: str = None, second_year_month: str = None):
+ """
+ This method prepare the query for usage
+ :param first_year_month: #YYYYMM
+ :param second_year_month: #YYYYMM
+ :return:
+ """
+ if not first_year_month and not second_year_month:
+ current_month = datetime.now().replace(day=1)
+ past_month = current_month - timedelta(days=1)
+ first_year_month = past_month.strftime("%Y%m")
+ second_year_month = current_month.strftime("%Y%m")
+ logger.info(f'StartMonth: {first_year_month}, EndMonth: {second_year_month}')
+ fetch_monthly_invoice_query = f"""
+ SELECT ifnull(project.ancestors[SAFE_OFFSET(1)].display_name, 'NA') as folder_name,
+ ifnull(project.ancestry_numbers, 'NA') as folder_id, invoice.month, ifnull(project.id, 'GCP-refund/credit') as project_name, ifnull(project.number, '000000000000') as project_id,
+ (SUM(CAST(cost AS NUMERIC)) + SUM(IFNULL((SELECT SUM(CAST(c.amount AS NUMERIC))
+ FROM UNNEST(credits) AS c), 0))) AS total_cost
+ FROM `{self.__database_name}.{self.__table_name}`
+ where invoice.month BETWEEN '{first_year_month}' AND '{second_year_month}'
+ GROUP BY 1, 2, 3, 4, 5
+ ORDER BY 3
+ """
+ fetch_monthly_folders_query = f"""
+ SELECT TO_JSON_STRING(project.ancestors) as project_folders, project.number, invoice.month, ifnull(project.ancestry_numbers, 'NA') as folder_id
+ FROM `{self.__database_name}.{self.__table_name}`
+ where invoice.month BETWEEN '{first_year_month}' AND '{second_year_month}' GROUP BY 1, 2, 3, 4 ORDER BY invoice.month
+ """
+ return [fetch_monthly_invoice_query, fetch_monthly_folders_query]
+
+ @typechecked()
+ @logger_time_stamp
+ def __organized_results(self, data_rows: list):
+ """
+ This method organize the results to be uploaded to elastic search
+ :param data_rows:
+ :return:
+ """
+ compress_gcp_data = {} # compress data based on budget_id
+ for row in data_rows:
+ month = row.get('Month')
+ cost_center, allocated_budget, years, owner = 0, 0, '', 'Others'
+ project_budget_account_id = 0
+ for idx, _id in enumerate((row.get('folder_ids')+[row.get('ProjectId')])[::-1]): # start from reverse [root, sub_child, child]
+ cost_center, allocated_budget, years, owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=_id, dir_path='/tmp')
+ if cost_center > 0:
+ project_budget_account_id = _id
+ break
+ parent_index = len(row.get("folder_ids"))
+ index = f'{project_budget_account_id}-{month}'
+ if index in compress_gcp_data:
+ compress_gcp_data[index]['Actual'] = round(compress_gcp_data[index]['Actual'] + row.get('Actual'), 3)
+ compress_gcp_data[index]['Projects'].append({
+ 'Project': row.get('Project'),
+ 'Actual': round(row.get('Actual'), self.DEFAULT_ROUND_DIGITS),
+ 'ProjectId': row.get('ProjectId')
+ })
+ if parent_index > compress_gcp_data[index]['total_folders']:
+ compress_gcp_data[index]['Account'] = row.get(f'parent{parent_index}', 'NA')
+ compress_gcp_data[index]['AccountId'] = row.get(f'parent{parent_index}_id', 'NA')
+ else:
+ project_cost_data = {'CloudName': self.__cloud_name, 'CostCenter': cost_center, 'Owner': owner,
+ 'Budget': round(allocated_budget / self.DEFAULT_YEARS, self.DEFAULT_ROUND_DIGITS),
+ 'Forecast': 0,
+ 'AllocatedBudget': round(allocated_budget, self.DEFAULT_ROUND_DIGITS),
+ 'BudgetId': project_budget_account_id,
+ 'Account': row.get(f'parent{parent_index}', 'NA'),
+ 'AccountId': row.get(f'parent{parent_index}_id', 'NA'),
+ 'Actual': round(row.get('Actual'), self.DEFAULT_ROUND_DIGITS),
+ 'filter_date': row.get('filter_date'), 'Month': row.get('Month'),
+ 'start_date': row.get('start_date'), 'timestamp': row.get('timestamp'),
+ 'Projects': [{'Project': row.get('Project'),
+ 'Actual': round(row.get('Actual'), self.DEFAULT_ROUND_DIGITS),
+ 'ProjectId': row.get('ProjectId')}],
+ 'index_id': f"{row.get('start_date')}-{row.get(f'parent{parent_index}', 'NA').lower()}",
+ 'total_folders': parent_index}
+ compress_gcp_data[index] = project_cost_data
+ return self.__second_layer_filter(items=list(compress_gcp_data.values()))
+
+ @typechecked()
+ @logger_time_stamp
+ def __second_layer_filter(self, items: list):
+ """
+ This method aggregates the results which have the same Account name
+ :param items:
+ :return:
+ """
+ filtered_result = {}
+ for item in items:
+ account = item.get('Account')
+ month = item.get('Month')
+ index = f'{account}-{month}'
+ if index in filtered_result:
+ filtered_result[index]['Budget'] += item.get('Budget')
+ filtered_result[index]['Actual'] += item.get('Actual')
+ if item.get('BudgetId') != filtered_result[index]['BudgetId']:
+ filtered_result[index]['AllocatedBudget'] += item.get('AllocatedBudget')
+ filtered_result[index]['Projects'].extend(item.get('Projects'))
+ else:
+ filtered_result[index] = item
+ return list(filtered_result.values())
+
+ # @Todo Add forecast values in future
+ @typechecked()
+ @logger_time_stamp
+ def __forecast_for_next_months(self, cost_data: list):
+ """
+ This method returns the forecast of next twelve months data
+ :param cost_data:
+ :return:
+ """
+ forecast_cost_data = []
+ month_years = self.__next_twelve_months()
+ month = (datetime.now().month - 1) % 12
+ if month == 0:
+ month = 12
+ if len(str(month)) == 1:
+ month = f'0{month}'
+ year = datetime.now().year
+ cache_start_date = f'{year}-{str(month)}-01'
+ for data in cost_data:
+ if cache_start_date == data.get('start_date') and data.get('CostCenter') > 0:
+ for m_y in month_years:
+ m, y = m_y[0], m_y[1]
+ start_date = f'{y}-{m}-01'
+ timestamp = datetime.strptime(start_date, "%Y-%m-%d")
+ index_id = f'{start_date}-{data.get("Account").lower()}'
+ month = datetime.strftime(timestamp, "%Y %b")
+ projects = []
+ for project in data.get('Projects'):
+ project['Actual'] = 0
+ projects.append(project)
+ forecast_cost_data.append({
+ **data,
+ 'Actual': 0,
+ 'start_date': start_date,
+ 'timestamp': timestamp,
+ 'index_id': index_id,
+ 'Projects': projects,
+ 'filter_date': f'{start_date}-{month.split()[-1]}',
+ 'Month': month}
+ )
+ return forecast_cost_data
+
+ @typechecked()
+ @logger_time_stamp
+ def __get_aggregated_folder_details(self, query_data: list):
+ """
+ This method gives the unique folder_names from the data
+ :param query_data:
+ :return:
+ """
+ project_folders = {}
+ for data in query_data:
+ index = f'{data.get("number")}'
+ month = data.get('month')
+ project_folder_id = data.get('folder_id')
+ insert_data = False
+ if index not in project_folders:
+ insert_data = True
+ else:
+ insert_data = month >= project_folders.get(index).get('month')
+ if insert_data:
+ updated_data = {'month': month, 'folder_id': project_folder_id}
+ for folders in literal_eval(data.get('project_folders')):
+ folder_id = folders.get('resource_name').split('/')[-1]
+ folder_name = folders.get('display_name')
+ updated_data[folder_id] = folder_name
+ project_folders[index] = updated_data
+ return project_folders
+
+ @typechecked()
+ @logger_time_stamp
+ def __get_parent_folders(self, folder_ids: list, folders_data: dict, project_id: str):
+ """
+ This method returns the list of parent folders of Project
+ :param folder_ids:
+ :param folders_data:
+ :param project_id:
+ :return:
+ """
+ parent_folders = {}
+ for idx, _id in enumerate(folder_ids):
+ parent_folders.update({
+ f'parent{idx + 1}': folders_data[project_id].get(_id),
+ f'parent{idx + 1}_id': _id
+ })
+ return parent_folders
+
+ @logger_time_stamp
+ def __get_big_query_data(self):
+ """
+ This method collect the data from the big query and filter the data
+ :return:
+ """
+ cost_usage_queries = self.__prepare_usage_query()
+ query_rows = self.__gcp_account.query_list(cost_usage_queries)
+ folders_data = self.__get_aggregated_folder_details(query_rows[1])
+ agg_data = {}
+ for cst_row in query_rows[0]:
+ project_id, bill_month, total_cost = cst_row.get('project_id').strip(), cst_row.get('month'), float(cst_row.get('total_cost'))
+ folder_ids = folders_data.get(project_id).get('folder_id').split('/')[2:-1] if folders_data.get(project_id) else cst_row.get('folder_id').split('/')[2:-1]
+ folder_name = cst_row.get('folder_name')
+ index = f"{project_id}-{bill_month}"
+ parents_folders = self.__get_parent_folders(folder_ids, folders_data, project_id) if project_id in folders_data else {}
+ if agg_data.get(index):
+ total_cost = float(cst_row.get('total_cost')) + agg_data[index]['Actual']
+ agg_data[index] = {
+ 'folder_name': parents_folders.get(f'parent{len(folder_ids)}', 'NA'),
+ 'start_date': f'{bill_month[:4]}-{bill_month[4:]}-01',
+ 'Project': cst_row.get('project_name'), 'ProjectId': project_id, 'Actual': total_cost,
+ 'Account': parents_folders.get('parent1', 'NA'), 'Forecast': 0, 'folder_ids': folder_ids, **parents_folders
+ }
+ agg_data[index]['timestamp'] = datetime.strptime(agg_data[index]['start_date'], '%Y-%m-%d')
+ month = datetime.strftime(agg_data[index]['timestamp'], "%Y %b")
+ agg_data[index]['Month'] = month
+ agg_data[index]['filter_date'] = f'{agg_data[index]["start_date"]}-{month.split()[-1]}'
+ return self.__organized_results(list(agg_data.values()))
+
+ @logger_time_stamp
+ def __get_cost_and_upload(self):
+ """
+ This method collect the cost and uploads to the ElasticSearch"
+ :return:
+ """
+ collected_data = self.__get_big_query_data()
+ forecast_data = self.__forecast_for_next_months(cost_data=collected_data)
+ upload_data = collected_data + forecast_data
+ self.elastic_upload.es_upload_data(items=upload_data, set_index='index_id')
+
+ @logger_time_stamp
+ def run(self):
+ """
+ This method run the gcp cost explorer methods
+ :return:
+ """
+ self.__get_cost_and_upload()
diff --git a/cloud_governance/policy/ibm/cost_billing_reports.py b/cloud_governance/policy/ibm/cost_billing_reports.py
index 5e130ad9..efbfe29a 100644
--- a/cloud_governance/policy/ibm/cost_billing_reports.py
+++ b/cloud_governance/policy/ibm/cost_billing_reports.py
@@ -23,8 +23,7 @@ def __init__(self):
self.__ibm_account = IBMAccount()
self.__elastic_upload = ElasticUpload()
self.update_to_gsheet = UploadToGsheet()
- self.owner = self.__environment_variables_dict.get('COST_CENTER_OWNER')
- self.cost_center, self.__account_budget, self.__years = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.__ibm_account.short_account_id)
+ self.cost_center, self.__account_budget, self.__years, self.__owner = self.update_to_gsheet.get_cost_center_budget_details(account_id=self.__ibm_account.short_account_id)
def prepare_es_data(self, month: str, year: str, usage_cost: float = 0, next_invoice: float = 0):
"""This method prepares the data to upload to the es"""
@@ -42,7 +41,7 @@ def prepare_es_data(self, month: str, year: str, usage_cost: float = 0, next_inv
'Month': month,
'CostCenter': self.cost_center,
'CloudName': 'IBM Cloud',
- 'Owner': self.owner,
+ 'Owner': self.__owner,
'Forecast': round(next_invoice, 3),
'Actual': round(usage_cost, 3),
'filter_date': f'{start_date}-{month.split()[-1]}',
@@ -66,10 +65,17 @@ def get_cost_usage_details(self):
if past_usage_cost:
es_data = self.prepare_es_data(usage_cost=round(past_usage_cost.get('resources').get('billable_cost'), 3), month=str(last_month), year=str(last_month_year))
upload_es_data[es_data['index_id']] = es_data
- for next_month in range(month+1, month+11):
- new_year = date + relativedelta(month=next_month)
- es_data = self.prepare_es_data(month=new_year.strftime("%m"), year=str(new_year.year))
- upload_es_data[es_data['index_id']] = es_data
+ for next_month in range(self.MONTHS):
+ next_month = (next_month + month) % self.MONTHS
+ if next_month != month:
+ c_year = year
+ if len(str(next_month)) != 2:
+ next_month = f'0{next_month}'
+ if next_month == '00':
+ year += 1
+ next_month = str(12)
+ es_data = self.prepare_es_data(month=str(next_month), year=str(c_year))
+ upload_es_data[es_data['index_id']] = es_data
if upload_es_data:
self.__elastic_upload.es_upload_data(items=list(upload_es_data.values()), set_index='index_id')
return list(upload_es_data.values())
diff --git a/cloud_governance/policy/ibm/ibm_cost_report.py b/cloud_governance/policy/ibm/ibm_cost_report.py
index 0af571e7..d0f5bd34 100644
--- a/cloud_governance/policy/ibm/ibm_cost_report.py
+++ b/cloud_governance/policy/ibm/ibm_cost_report.py
@@ -27,7 +27,7 @@ def __init__(self):
@typechecked
def collect_tags_from_machines(self, tags: list):
"""
- This method return tags from list of string tags
+ This method returns tags from list of string tags
@param tags:
@return:
"""
diff --git a/cloud_governance/policy/ibm/tag_vm.py b/cloud_governance/policy/ibm/tag_vm.py
index 3462e1fd..b66b9987 100644
--- a/cloud_governance/policy/ibm/tag_vm.py
+++ b/cloud_governance/policy/ibm/tag_vm.py
@@ -13,7 +13,7 @@ def __init__(self):
def get_virtual_machine_username(self, vm_id: str):
"""
- This method return the virtual machine username from the billing order lists
+ This method returns the virtual machine username from the billing order lists
@param vm_id:
@return:
"""
diff --git a/cloud_governance/policy/policy_operations/aws/tag_cluster/remove_cluster_tags.py b/cloud_governance/policy/policy_operations/aws/tag_cluster/remove_cluster_tags.py
index a23fb720..51baf46d 100644
--- a/cloud_governance/policy/policy_operations/aws/tag_cluster/remove_cluster_tags.py
+++ b/cloud_governance/policy/policy_operations/aws/tag_cluster/remove_cluster_tags.py
@@ -66,7 +66,7 @@ def remove_instance_tags(self, instance_list: list, tags: list):
def get_cluster(self, clusters: list):
"""
- This method return cluster, and it tags
+ This method returns cluster, and it tags
@param clusters:
@return:
"""
@@ -205,7 +205,7 @@ def cluster_images(self, instance_tags: dict):
def cluster_snapshot(self, instance_tags: dict):
"""
- This method return list of cluster's snapshot according to cluster tag name
+ This method returns list of cluster's snapshot according to cluster tag name
@return:
"""
snapshots_data = self.ec2_operations.get_snapshots()
@@ -458,7 +458,7 @@ def cluster_role(self, instance_tags: dict):
def cluster_user(self, instance_tags: dict):
"""
- This method return list of cluster's user according to cluster name
+ This method returns list of cluster's user according to cluster name
@param instance_tags:
@return:
"""
diff --git a/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_operations.py b/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_operations.py
index 2bd5fd97..16a74638 100644
--- a/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_operations.py
+++ b/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_operations.py
@@ -1,3 +1,5 @@
+from datetime import datetime
+
import boto3
from cloud_governance.common.clouds.aws.cloudtrail.cloudtrail_operations import CloudTrailOperations
@@ -11,7 +13,7 @@ class TagClusterOperations:
This class tags AWS resources
"""
- def __init__(self, input_tags: dict, cluster_name: str, cluster_prefix: str, region: str, dry_run: str, cluster_only: bool):
+ def __init__(self, region: str, input_tags: dict = None, cluster_name: str = None, cluster_prefix: str = None, dry_run: str = None, cluster_only: bool = None):
self.cluster_only = cluster_only
self.cluster_prefix = cluster_prefix
self.utils = Utils(region=region)
@@ -27,6 +29,7 @@ def __init__(self, input_tags: dict, cluster_name: str, cluster_prefix: str, r
self.cloudtrail = CloudTrailOperations(region_name='us-east-1')
self._get_username_from_instance_id_and_time = CloudTrailOperations(region_name=region).get_username_by_instance_id_and_time
self.dry_run = dry_run
+ self.iam_users = self.iam_operations.get_iam_users_list()
def _input_tags_list_builder(self):
"""
@@ -67,3 +70,29 @@ def _fill_na_tags(self, user: str = None):
else:
tags.append({'Key': key, 'Value': value})
return tags
+
+ def get_user_name_from_name_tag(self, tags: list):
+ """
+ This method retuns the username from the name tag verified with iam users
+ :param tags:
+ :return:
+ """
+ user_name = self.ec2_operations.get_tag_value_from_tags(tags=tags, tag_name='User')
+ if user_name in self.iam_users:
+ return user_name
+ else:
+ name_tag = self.ec2_operations.get_tag_value_from_tags(tags=tags, tag_name='Name')
+ for user in self.iam_users:
+ if user in name_tag:
+ return user
+ return None
+
+ def get_username(self, start_time: datetime, resource_id: str, resource_type: str, tags: list):
+ """
+ This method returns the username
+ :return:
+ """
+ iam_username = self.get_user_name_from_name_tag(tags=tags)
+ if not iam_username:
+ return self._get_username_from_instance_id_and_time(start_time=start_time, resource_id=resource_id, resource_type=resource_type)
+ return iam_username
diff --git a/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_resouces.py b/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_resouces.py
index 5b9fa619..50f69510 100644
--- a/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_resouces.py
+++ b/cloud_governance/policy/policy_operations/aws/tag_cluster/tag_cluster_resouces.py
@@ -12,6 +12,7 @@ class TagClusterResources(TagClusterOperations):
"""
SHORT_ID = 5
+ NA_VALUE = 'NA'
def __init__(self, cluster_name: str = None, cluster_prefix: str = None, input_tags: dict = None,
region: str = 'us-east-2', dry_run: str = 'yes', cluster_only: bool = False):
@@ -107,7 +108,7 @@ def __remove_tags_start_with_aws(self, tags: list):
def __generate_cluster_resources_list_by_tag(self, resources_list: list, input_resource_id: str,
tags: str = 'Tags'):
"""
- This method return resource list that related to input resource id according to cluster's tag name and update the tags
+ This method returns resource list that related to input resource id according to cluster's tag name and update the tags
@param resources_list:
@param input_resource_id:
@param ids:
@@ -141,10 +142,7 @@ def __generate_cluster_resources_list_by_tag(self, resources_list: list, input_r
for cluster_name, cluster_id in cluster_ids.items():
if self.dry_run == 'no':
try:
- if self.cluster_name in cluster_name:
- self.ec2_client.create_tags(Resources=cluster_id, Tags=cluster_tags.get(cluster_name))
- else:
- self.ec2_client.create_tags(Resources=cluster_id, Tags=cluster_tags.get(cluster_name))
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=cluster_id, tags=cluster_tags.get(cluster_name))
logger.info(f'{input_resource_id}:: {cluster_name}, count: {len(cluster_id)}, {cluster_id}, {cluster_tags.get(cluster_name)}')
except Exception as err:
logger.info(err)
@@ -153,7 +151,7 @@ def __generate_cluster_resources_list_by_tag(self, resources_list: list, input_r
def __generate_cluster_resources_list_by_vpc(self, resources_list: list, input_resource_id: str):
"""
- This method return resource list that related to input resource id according to cluster's vpc id
+ This method returns resource list that related to input resource id according to cluster's vpc id
@param resources_list:
@param input_resource_id:
@return:
@@ -174,12 +172,12 @@ def __generate_cluster_resources_list_by_vpc(self, resources_list: list, input_r
if self.cluster_name:
if self.cluster_name in cluster_tag[0].get('Key'):
if self.dry_run == 'no':
- self.ec2_client.create_tags(Resources=[resource_id], Tags=all_tags)
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=[resource_id], tags=all_tags)
logger.info(all_tags)
result_resources_list.append(resource_id)
else:
if self.dry_run == 'no':
- self.ec2_client.create_tags(Resources=[resource_id], Tags=all_tags)
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=[resource_id], tags=all_tags)
logger.info(all_tags)
result_resources_list.append(resource_id)
break
@@ -229,11 +227,14 @@ def __validate_existing_tag(self, tags: list):
@param tags:
@return:
"""
+ check_tags = ['User', 'Project', 'Manager', 'Owner', 'Email']
+ tag_count = 0
for tag in tags:
- for key, value in self.input_tags.items():
- if tag.get('Key') == key:
- return True
- return False
+ if tag.get('Key') in check_tags:
+ tag_count += 1
+ if tag.get('Value') == 'NA':
+ return False
+ return tag_count == len(check_tags)
def update_cluster_tags(self, resources: list):
"""
@@ -248,24 +249,22 @@ def update_cluster_tags(self, resources: list):
for instance in resources:
for item in instance:
instance_id = item['InstanceId']
- if item.get('Tags'):
+ tags = item.get('Tags')
+ if tags:
# search that not exist permanent tags in the resource
- if not self.__validate_existing_tag(item.get('Tags')):
- for tag in item['Tags']:
+ if not self.__validate_existing_tag(tags):
+ for tag in tags:
if self.cluster_prefix in tag.get('Key'):
add_tags = self.__append_input_tags()
cluster_name = tag.get('Key').split('/')[-1]
- if cluster_name in cluster_instances:
- add_tags = self.__filter_resource_tags_by_add_tags(tags=item.get('Tags'),
- search_tags=cluster_tags[
- cluster_name])
+ user = self.ec2_operations.get_tag_value_from_tags(tags=tags, tag_name='User')
+ if cluster_name in cluster_instances and user and user != 'NA':
+ add_tags = self.__filter_resource_tags_by_add_tags(tags=tags, search_tags=cluster_tags[cluster_name])
if add_tags:
cluster_instances[cluster_name].append(instance_id)
break
else:
- username = self._get_username_from_instance_id_and_time(
- start_time=item.get('LaunchTime'), resource_id=instance_id,
- resource_type='AWS::EC2::Instance')
+ username = self.get_username(start_time=item.get('LaunchTime'), resource_id=instance_id, resource_type='AWS::EC2::Instance', tags=tags)
if username:
if username == 'AutoScaling':
add_tags.extend(self._fill_na_tags(user=username))
@@ -275,10 +274,14 @@ def update_cluster_tags(self, resources: list):
if not self.__check_user_in_username_tags(user_tags):
try:
user = self.iam_client.get_user(UserName=username)['User']
- username = self.cloudtrail.get_username_by_instance_id_and_time(
+ temp_username = self.cloudtrail.get_username_by_instance_id_and_time(
start_time=user.get('CreateDate'), resource_id=username,
resource_type='AWS::IAM::User')
- user_tags = self.iam_operations.get_user_tags(username=username)
+ if temp_username:
+ add_tags.append({'Key': 'User', 'Value': temp_username})
+ user_tags = self.iam_operations.get_user_tags(username=temp_username)
+ else:
+ add_tags.append({'Key': 'User', 'Value': username})
except:
add_tags.append({'Key': 'User', 'Value': username})
if user_tags:
@@ -294,17 +297,13 @@ def update_cluster_tags(self, resources: list):
add_tags = self.__filter_resource_tags_by_add_tags(tags=item.get('Tags'),
search_tags=add_tags)
if add_tags:
- cluster_instances[cluster_name] = [instance_id]
+ cluster_instances.setdefault(cluster_name, []).append(instance_id)
cluster_tags[cluster_name] = add_tags
break
for cluster_instance_name, instance_ids in cluster_instances.items():
if self.dry_run == 'no':
try:
- if self.cluster_name:
- if cluster_instance_name == self.cluster_name:
- self.ec2_client.create_tags(Resources=instance_ids, Tags=cluster_tags.get(cluster_instance_name))
- else:
- self.ec2_client.create_tags(Resources=instance_ids, Tags=cluster_tags.get(cluster_instance_name))
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=instance_ids, tags=cluster_tags.get(cluster_instance_name))
logger.info(f'Cluster :: {cluster_instance_name} count: {len(instance_ids)} :: InstanceId :: {instance_ids} :: {cluster_tags.get(cluster_instance_name)}')
except Exception as err:
logger.info(err)
@@ -318,7 +317,7 @@ def update_cluster_tags(self, resources: list):
def cluster_instance(self):
"""
- This method return list of cluster's instance according to cluster tag name,
+ This method returns list of cluster's instance according to cluster tag name,
The instances list is different from other resources
it will search for full cluster name (including random suffix string) in case of user input cluster name was given
@return:
@@ -339,7 +338,7 @@ def cluster_instance(self):
def cluster_volume(self):
"""
- This method return list of cluster's volume according to cluster tag name
+ This method returns list of cluster's volume according to cluster tag name
@return:
"""
volumes_data = self.ec2_operations.get_volumes()
@@ -353,7 +352,7 @@ def cluster_volume(self):
def cluster_ami(self):
"""
- This method return list of cluster's ami according to cluster tag name
+ This method returns list of cluster's ami according to cluster tag name
@return:
"""
images_data = self.ec2_operations.get_images()
@@ -367,7 +366,7 @@ def cluster_ami(self):
def cluster_snapshot(self):
"""
- This method return list of cluster's snapshot according to cluster tag name
+ This method returns list of cluster's snapshot according to cluster tag name
@return:
"""
snapshots_data = self.ec2_operations.get_snapshots()
@@ -381,14 +380,14 @@ def cluster_snapshot(self):
def __get_security_group_data(self):
"""
- This method return security group data
+ This method returns security group data
@return:
"""
return self.ec2_operations.get_security_groups()
def cluster_security_group(self):
"""
- This method return list of cluster's security group according to cluster tag name
+ This method returns list of cluster's security group according to cluster tag name
@return:
"""
security_group_ids = self.__generate_cluster_resources_list_by_tag(
@@ -398,7 +397,7 @@ def cluster_security_group(self):
def cluster_elastic_ip(self):
"""
- This method return list of cluster's elastic ip according to cluster tag name
+ This method returns list of cluster's elastic ip according to cluster tag name
@return:
"""
elastic_ips_data = self.ec2_operations.get_elastic_ips()
@@ -409,7 +408,7 @@ def cluster_elastic_ip(self):
def cluster_network_interface(self):
"""
- This method return list of cluster's network interface according to cluster tag name
+ This method returns list of cluster's network interface according to cluster tag name
@return:
"""
network_interfaces_data = self.ec2_operations.get_network_interface()
@@ -422,7 +421,7 @@ def cluster_network_interface(self):
def cluster_load_balancer(self):
"""
- This method return list of cluster's load balancer according to cluster vpc
+ This method returns list of cluster's load balancer according to cluster vpc
@return:
"""
result_resources_list = []
@@ -470,7 +469,7 @@ def cluster_load_balancer(self):
def cluster_load_balancer_v2(self):
"""
- This method return list of cluster's load balancer according to cluster vpc
+ This method returns list of cluster's load balancer according to cluster vpc
@return:
"""
result_resources_list = []
@@ -518,7 +517,7 @@ def cluster_load_balancer_v2(self):
def cluster_vpc(self):
"""
- This method return list of cluster's vpc according to cluster tag name
+ This method returns list of cluster's vpc according to cluster tag name
@return:
"""
vpcs_data = self.ec2_operations.get_vpcs()
@@ -545,7 +544,7 @@ def get_cluster_vpc(self):
def cluster_subnet(self):
"""
- This method return list of cluster's subnet according to cluster tag name
+ This method returns list of cluster's subnet according to cluster tag name
@return:
"""
subnets_data = self.ec2_operations.get_subnets()
@@ -556,7 +555,7 @@ def cluster_subnet(self):
def cluster_route_table(self):
"""
- This method return list of cluster's route table according to cluster tag name
+ This method returns list of cluster's route table according to cluster tag name
@return:
"""
route_tables_data = self.ec2_operations.get_route_tables()
@@ -567,7 +566,7 @@ def cluster_route_table(self):
def cluster_internet_gateway(self):
"""
- This method return list of cluster's route table internet gateway according to cluster tag name
+ This method returns list of cluster's route table internet gateway according to cluster tag name
@return:
"""
internet_gateways_data = self.ec2_operations.get_internet_gateways()
@@ -579,7 +578,7 @@ def cluster_internet_gateway(self):
def cluster_dhcp_option(self):
"""
- This method return list of cluster's dhcp option according to cluster tag name
+ This method returns list of cluster's dhcp option according to cluster tag name
@return:
"""
dhcp_options_data = self.ec2_operations.get_dhcp_options()
@@ -590,7 +589,7 @@ def cluster_dhcp_option(self):
def cluster_vpc_endpoint(self):
"""
- This method return list of cluster's vpc endpoint according to cluster tag name
+ This method returns list of cluster's vpc endpoint according to cluster tag name
@return:
"""
vpc_endpoints_data = self.ec2_operations.get_vpce()
@@ -601,7 +600,7 @@ def cluster_vpc_endpoint(self):
def cluster_nat_gateway(self):
"""
- This method return list of cluster's nat gateway according to cluster tag name
+ This method returns list of cluster's nat gateway according to cluster tag name
@return:
"""
nat_gateways_data = self.ec2_operations.get_nat_gateways()
@@ -612,7 +611,7 @@ def cluster_nat_gateway(self):
def cluster_network_acl(self):
"""
- This method return list of cluster's network acl according to cluster vpc id
+ This method returns list of cluster's network acl according to cluster vpc id
Missing OpenShift Tags for it based on VPCs
@return:
"""
@@ -624,7 +623,7 @@ def cluster_network_acl(self):
def cluster_role(self, cluster_names: list = []):
"""
- This method return list of cluster's role according to cluster name
+ This method returns list of cluster's role according to cluster name
@param cluster_names:
@return:
"""
@@ -674,7 +673,7 @@ def cluster_role(self, cluster_names: list = []):
def cluster_user(self, cluster_names: list = []):
"""
- This method return list of cluster's user according to cluster name
+ This method returns list of cluster's user according to cluster name
@param cluster_names:
@return:
"""
@@ -730,13 +729,22 @@ def __filter_resource_tags_by_add_tags(self, tags: list, search_tags: list):
for search_tag in search_tags:
found = False
for tag in tags:
- if tag.get('Key') == search_tag.get('Key'):
+ if tag.get('Key') == search_tag.get('Key') and tag.get('Value') != 'NA':
found = True
+ break
if not found:
add_tags.append(search_tag)
else:
add_tags.extend(search_tags)
- return add_tags
+ filter_tags = {}
+ for tag in add_tags:
+ key = tag.get('Key')
+ value = tag.get('Value')
+ if key in filter_tags and filter_tags[key].get('Value') == self.NA_VALUE:
+ filter_tags[key] = {'Key': key, 'Value': value}
+ else:
+ filter_tags[key] = {'Key': key, 'Value': value}
+ return list(filter_tags.values())
def __remove_launchTime(self, tags: list):
"""
@@ -748,7 +756,7 @@ def __remove_launchTime(self, tags: list):
def cluster_s3_bucket(self, cluster_names: list = []):
"""
- This method return list of cluster's s3 bucket according to cluster name
+ This method returns list of cluster's s3 bucket according to cluster name
@param cluster_names:
@return:
"""
diff --git a/cloud_governance/policy/policy_operations/aws/tag_non_cluster/non_cluster_operations.py b/cloud_governance/policy/policy_operations/aws/tag_non_cluster/non_cluster_operations.py
index 8db6b2eb..f7b3519d 100644
--- a/cloud_governance/policy/policy_operations/aws/tag_non_cluster/non_cluster_operations.py
+++ b/cloud_governance/policy/policy_operations/aws/tag_non_cluster/non_cluster_operations.py
@@ -10,6 +10,8 @@
class NonClusterOperations:
+ NA_VALUE = 'NA'
+
def __init__(self, region: str = 'us-east-2', dry_run: str = 'yes', input_tags: dict = ''):
self.region = region
self.dry_run = dry_run
@@ -21,6 +23,7 @@ def __init__(self, region: str = 'us-east-2', dry_run: str = 'yes', input_tags:
self.iam_client = IAMOperations()
self.ec2_operations = EC2Operations(region=region)
self.utils = Utils(region=region)
+ self.iam_users = self.iam_client.get_iam_users_list()
def _get_instances_data(self, instance_id: str = ''):
"""
@@ -63,13 +66,21 @@ def _get_tags_of_resources(self, tags: list, search_tags: list):
for search_tag in search_tags:
found = False
for tag in tags:
- if tag.get('Key') == search_tag.get('Key'):
+ if tag.get('Key') == search_tag.get('Key') and tag.get('Value') != 'NA':
found = True
if not found:
add_tags.append(search_tag)
else:
add_tags.extend(search_tags)
- return add_tags
+ filter_tags = {}
+ for tag in add_tags:
+ key = tag.get('Key')
+ value = tag.get('Value')
+ if key in filter_tags and filter_tags[key].get('Value') == self.NA_VALUE:
+ filter_tags[key] = {'Key': key, 'Value': value}
+ else:
+ filter_tags[key] = {'Key': key, 'Value': value}
+ return list(filter_tags.values())
def _fill_na_tags(self, user: str = None):
"""
@@ -89,15 +100,15 @@ def _fill_na_tags(self, user: str = None):
tags.append({'Key': key, 'Value': value})
return tags
- def _get_username_from_cloudtrail(self, start_time: datetime, resource_id: str, resource_type: str):
+ def _get_username_from_cloudtrail(self, start_time: datetime, resource_id: str, resource_type: str, end_time: datetime = None):
"""
- This method return username fom cloudtrail
+ This method returns username fom cloudtrail
@param start_time:
@param resource_id:
@param resource_type:
@return:
"""
- return self.cloudtrail.get_username_by_instance_id_and_time(start_time=start_time, resource_id=resource_id, resource_type=resource_type)
+ return self.cloudtrail.get_username_by_instance_id_and_time(start_time=start_time, resource_id=resource_id, resource_type=resource_type, end_time=end_time)
def _get_resource_data(self, resource_method: callable):
"""
@@ -121,7 +132,7 @@ def _convert_datetime_format(self, date_time: datetime):
def _build_tag(self, key: str, value: any):
"""
- This method return Key value pair
+ This method returns Key value pair
@param key:
@param value:
@return:
@@ -152,7 +163,7 @@ def _get_tags_from_instance_item(self, instance_item: dict):
def _get_tags_fom_attachments(self, attachments: list):
"""
- This method return tags from attachments
+ This method returns tags from attachments
@param attachments:
@return:
"""
@@ -183,4 +194,43 @@ def _get_tags_from_snapshot_description_images(self, description: str):
username = self._get_username_from_cloudtrail(start_time=start_time, resource_id=image_id, resource_type='AWS::EC2::Ami')
return tags, username
+ def get_user_name_from_name_tag(self, tags: list = None, resource_name: str = None):
+ """
+ This method retuns the username from the name tag verified with iam users
+ :param resource_name:
+ :param tags:
+ :return:
+ """
+ name_tag = self.ec2_operations.get_tag_value_from_tags(tags=tags, tag_name='Name') if tags else resource_name
+ for user in self.iam_users:
+ if user in name_tag:
+ return user
+ return None
+
+ def get_username(self, start_time: datetime, resource_id: str, resource_type: str, tags: list, resource_name: str = '', end_time: datetime = None):
+ """
+ This method returns the username
+ :return:
+ """
+ iam_username = self.get_user_name_from_name_tag(tags=tags, resource_name=resource_name)
+ if not iam_username:
+ iam_username = self.get_user_name_from_name_tag(resource_name=resource_name)
+ if not iam_username:
+ return self._get_username_from_cloudtrail(start_time=start_time, resource_id=resource_id, resource_type=resource_type, end_time=end_time)
+ return iam_username
+ def validate_existing_tag(self, tags: list):
+ """
+ This method validates that permanent tag exists in tags list
+ @param tags:
+ @return:
+ """
+ check_tags = ['User', 'Project', 'Manager', 'Owner', 'Email']
+ tag_count = 0
+ if tags:
+ for tag in tags:
+ if tag.get('Key') in check_tags:
+ tag_count += 1
+ if tag.get('Value') == 'NA':
+ return False
+ return tag_count == len(check_tags)
diff --git a/cloud_governance/policy/policy_operations/aws/tag_non_cluster/tag_non_cluster_resources.py b/cloud_governance/policy/policy_operations/aws/tag_non_cluster/tag_non_cluster_resources.py
index 69b97b5d..0b14d684 100644
--- a/cloud_governance/policy/policy_operations/aws/tag_non_cluster/tag_non_cluster_resources.py
+++ b/cloud_governance/policy/policy_operations/aws/tag_non_cluster/tag_non_cluster_resources.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timedelta
from cloud_governance.common.logger.init_logger import logger
from cloud_governance.policy.policy_operations.aws.tag_non_cluster.non_cluster_operations import NonClusterOperations
@@ -36,7 +36,7 @@ def __get_instance_tags(self, launch_time: datetime, instance_id: str, tags: lis
@param tags:
@return:
"""
- username = self._get_username_from_cloudtrail(start_time=launch_time, resource_id=instance_id, resource_type='AWS::EC2::Instance')
+ username = self.get_username(start_time=launch_time, resource_id=instance_id, resource_type='AWS::EC2::Instance', tags=tags)
search_tags = []
user_tags = []
if not username:
@@ -67,15 +67,17 @@ def non_cluster_update_ec2(self, instances_list: list = None):
for item in instance:
instance_id = item.get('InstanceId')
launch_time = item.get('LaunchTime')
- add_tags = self.__get_instance_tags(launch_time=launch_time, instance_id=instance_id, tags=item.get('Tags'))
- if add_tags:
- if self.dry_run == 'no':
- try:
- self.ec2_client.create_tags(Resources=[instance_id], Tags=add_tags)
- logger.info(f'Added tags to instance: {instance_id} total: {len(add_tags)} tags: {add_tags}')
- except Exception as err:
- logger.info(err)
- instances_ids.append(instance_id)
+ tags = item.get('Tags')
+ if not self.validate_existing_tag(tags=tags):
+ add_tags = self.__get_instance_tags(launch_time=launch_time, instance_id=instance_id, tags=tags)
+ if add_tags:
+ if self.dry_run == 'no':
+ try:
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=[instance_id], tags=add_tags)
+ logger.info(f'Added tags to instance: {instance_id} total: {len(add_tags)} tags: {add_tags}')
+ except Exception as err:
+ logger.info(err)
+ instances_ids.append(instance_id)
logger.info(f'non_cluster_ec2 count: {len(sorted(instances_ids))} {sorted(instances_ids)}')
return sorted(instances_ids)
@@ -90,37 +92,39 @@ def update_volumes(self, volumes_data: list = None):
volume_ids = []
for volume in volumes_data:
volume_id = volume.get('VolumeId')
- username = self._get_username_from_cloudtrail(start_time=volume.get('CreateTime'), resource_id=volume_id, resource_type='AWS::EC2::Volume')
- search_tags = []
- if not username:
- get_tags, username = self._get_tags_fom_attachments(attachments=volume.get('Attachments'))
- search_tags.extend(get_tags)
- else:
- search_tags.extend(self._append_input_tags())
- if username:
- user_tags = self.iam_client.get_user_tags(username=username)
- if not user_tags:
- search_tags.extend(self._fill_na_tags(user=username))
+ tags = volume.get('Tags')
+ if not self.validate_existing_tag(tags=tags):
+ username = self.get_username(start_time=volume.get('CreateTime'), resource_id=volume_id, resource_type='AWS::EC2::Volume', tags=tags)
+ search_tags = []
+ if not username:
+ get_tags, username = self._get_tags_fom_attachments(attachments=volume.get('Attachments'))
+ search_tags.extend(get_tags)
else:
- search_tags.extend(user_tags)
- search_tags.append({'Key': 'Email', 'Value': f'{username}@redhat.com'})
- search_tags.append(self._build_tag(key='LaunchTime', value=volume.get('CreateTime')))
- else:
- search_tags.extend(self._fill_na_tags())
- search_tags.extend(self._append_input_tags())
- search_tags.append(self._build_tag(key='LaunchTime', value=volume.get('CreateTime')))
- if not self.__check_name_in_tags(volume.get('Tags')):
- tag_name = f'{username}-{volume_id[-self.SHORT_RESOURCE_ID:]}' if username else f'{volume_id[:self.SHORT_RESOURCE_NAME]}-{self.region}-{volume_id[-self.SHORT_RESOURCE_ID:]}'
- search_tags.append({'Key': 'cg-Name', 'Value': tag_name})
- volume_tags = self._get_tags_of_resources(tags=volume.get('Tags'), search_tags=search_tags)
- if volume_tags:
- if self.dry_run == 'no':
- try:
- self.ec2_client.create_tags(Resources=[volume_id], Tags=volume_tags)
- logger.info(f'added tags to volume_id: {volume_id} total: {len(volume_tags)} tags: {volume_tags}')
- except Exception as err:
- logger.info(err)
- volume_ids.append(volume_id)
+ search_tags.extend(self._append_input_tags())
+ if username:
+ user_tags = self.iam_client.get_user_tags(username=username)
+ if not user_tags:
+ search_tags.extend(self._fill_na_tags(user=username))
+ else:
+ search_tags.extend(user_tags)
+ search_tags.append({'Key': 'Email', 'Value': f'{username}@redhat.com'})
+ search_tags.append(self._build_tag(key='LaunchTime', value=volume.get('CreateTime')))
+ else:
+ search_tags.extend(self._fill_na_tags())
+ search_tags.extend(self._append_input_tags())
+ search_tags.append(self._build_tag(key='LaunchTime', value=volume.get('CreateTime')))
+ if not self.__check_name_in_tags(volume.get('Tags')):
+ tag_name = f'{username}-{volume_id[-self.SHORT_RESOURCE_ID:]}' if username else f'{volume_id[:self.SHORT_RESOURCE_NAME]}-{self.region}-{volume_id[-self.SHORT_RESOURCE_ID:]}'
+ search_tags.append({'Key': 'cg-Name', 'Value': tag_name})
+ volume_tags = self._get_tags_of_resources(tags=volume.get('Tags'), search_tags=search_tags)
+ if volume_tags:
+ if self.dry_run == 'no':
+ try:
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=[volume_id], tags=volume_tags)
+ logger.info(f'added tags to volume_id: {volume_id} total: {len(volume_tags)} tags: {volume_tags}')
+ except Exception as err:
+ logger.info(err)
+ volume_ids.append(volume_id)
logger.info(f'non_cluster_volumes count: {len(sorted(volume_ids))} {sorted(volume_ids)}')
return sorted(volume_ids)
@@ -135,43 +139,51 @@ def update_snapshots(self, snapshots: list = None):
snapshot_ids = []
for snapshot in snapshots:
snapshot_id = snapshot.get('SnapshotId')
- username = self._get_username_from_cloudtrail(start_time=snapshot.get('StartTime'), resource_id=snapshot_id, resource_type='AWS::EC2::Snapshot')
- search_tags = []
- if not username:
- if snapshot.get('Description') and 'Created' in snapshot.get('Description'):
- image_tags, username = self._get_tags_from_snapshot_description_images(description=snapshot.get('Description'))
- if not username:
- instance_id = snapshot.get('Description').split(" ")[2].split("(")[1][:-1]
- instances = self._get_instances_data(instance_id)
- if instances:
- for item in instances:
- if item.get('InstanceId') == instance_id:
- item_tags, username = self._get_tags_from_instance_item(instance_item=item)
- else:
- search_tags.extend(self._append_input_tags())
- if username:
- user_tags = self.iam_client.get_user_tags(username=username)
- search_tags.append({'Key': 'Email', 'Value': f'{username}@redhat.com'})
- if not user_tags:
- search_tags.extend(self._fill_na_tags(user=username))
+ tags = snapshot.get('Tags')
+ if not self.validate_existing_tag(tags=tags):
+ username = self.get_username(start_time=snapshot.get('StartTime'), resource_id=snapshot_id, resource_type='AWS::EC2::Snapshot', tags=tags)
+ if 'vm_import_image' in username:
+ start_time = snapshot.get('StartTime') + timedelta(seconds=5)
+ end_time = start_time + timedelta(minutes=30)
+ assume_username = self.get_username(start_time=start_time, resource_id=snapshot_id, resource_type='AWS::EC2::Snapshot', tags=tags, end_time=end_time)
+ if assume_username:
+ username = assume_username
+ search_tags = []
+ if not username:
+ if snapshot.get('Description') and 'Created' in snapshot.get('Description'):
+ image_tags, username = self._get_tags_from_snapshot_description_images(description=snapshot.get('Description'))
+ if not username:
+ instance_id = snapshot.get('Description').split(" ")[2].split("(")[1][:-1]
+ instances = self._get_instances_data(instance_id)
+ if instances:
+ for item in instances:
+ if item.get('InstanceId') == instance_id:
+ item_tags, username = self._get_tags_from_instance_item(instance_item=item)
else:
- search_tags.extend(user_tags)
- else:
- search_tags.extend(self._fill_na_tags())
- search_tags.extend(self._append_input_tags())
- if not self.__check_name_in_tags(snapshot.get('Tags')):
- tag_name = f'{username}-{snapshot_id[-self.SHORT_RESOURCE_ID:]}' if username else f'{snapshot_id[:self.SHOT_SNAPSHOT_ID]}-{self.region}-{snapshot_id[-self.SHORT_RESOURCE_ID:]}'
- search_tags.append({'Key': 'cg-Name', 'Value': tag_name})
- search_tags.append(self._build_tag(key='LaunchTime', value=snapshot.get('StartTime')))
- snapshot_tags = self._get_tags_of_resources(tags=snapshot.get('Tags'), search_tags=search_tags)
- if snapshot_tags:
- if self.dry_run == 'no':
- try:
- self.ec2_client.create_tags(Resources=[snapshot_id], Tags=snapshot_tags)
- logger.info(f'added tags to snapshots: {snapshot_id} total: {len(snapshot_tags)} tags: {snapshot_tags}')
- except Exception as err:
- logger.info(err)
- snapshot_ids.append(snapshot_id)
+ search_tags.extend(self._append_input_tags())
+ if username:
+ user_tags = self.iam_client.get_user_tags(username=username)
+ search_tags.append({'Key': 'Email', 'Value': f'{username}@redhat.com'})
+ if not user_tags:
+ search_tags.extend(self._fill_na_tags(user=username))
+ else:
+ search_tags.extend(user_tags)
+ else:
+ search_tags.extend(self._fill_na_tags())
+ search_tags.extend(self._append_input_tags())
+ if not self.__check_name_in_tags(snapshot.get('Tags')):
+ tag_name = f'{username}-{snapshot_id[-self.SHORT_RESOURCE_ID:]}' if username else f'{snapshot_id[:self.SHOT_SNAPSHOT_ID]}-{self.region}-{snapshot_id[-self.SHORT_RESOURCE_ID:]}'
+ search_tags.append({'Key': 'cg-Name', 'Value': tag_name})
+ search_tags.append(self._build_tag(key='LaunchTime', value=snapshot.get('StartTime')))
+ snapshot_tags = self._get_tags_of_resources(tags=snapshot.get('Tags'), search_tags=search_tags)
+ if snapshot_tags:
+ if self.dry_run == 'no':
+ try:
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=[snapshot_id], tags=snapshot_tags)
+ logger.info(f'added tags to snapshots: {snapshot_id} total: {len(snapshot_tags)} tags: {snapshot_tags}')
+ except Exception as err:
+ logger.info(err)
+ snapshot_ids.append(snapshot_id)
logger.info(f'non_cluster_snapshot count: {len(sorted(snapshot_ids))} {sorted(snapshot_ids)}')
return sorted(snapshot_ids)
@@ -187,31 +199,34 @@ def update_ami(self, images: list = None):
image_ids = []
for image in images:
image_id = image.get('ImageId')
+ tags = image.get('Tags')
+ image_name = image.get('Name')
start_time = datetime.fromisoformat(image.get('CreationDate')[:-1] + '+00:00')
- username = self._get_username_from_cloudtrail(start_time=start_time, resource_id=image_id, resource_type='AWS::EC2::Ami')
- search_tags = []
- search_tags.extend(self._append_input_tags())
- if username:
- user_tags = self.iam_client.get_user_tags(username=username)
- search_tags.append({'Key': 'Email', 'Value': f'{username}@redhat.com'})
- if not user_tags:
- search_tags.extend(self._fill_na_tags(user=username))
+ if not self.validate_existing_tag(tags=tags):
+ username = self.get_username(start_time=start_time, resource_id=image_id, resource_type='AWS::EC2::Ami', tags=tags, resource_name=image_name)
+ search_tags = []
+ search_tags.extend(self._append_input_tags())
+ if username:
+ user_tags = self.iam_client.get_user_tags(username=username)
+ search_tags.append({'Key': 'Email', 'Value': f'{username}@redhat.com'})
+ if not user_tags:
+ search_tags.extend(self._fill_na_tags(user=username))
+ else:
+ search_tags.extend(user_tags)
else:
- search_tags.extend(user_tags)
- else:
- search_tags.extend(self._fill_na_tags())
- if not self.__check_name_in_tags(image.get('Tags')):
- tag_name = f'{username}-{image_id[-self.SHORT_RESOURCE_ID:]}' if username else f'{image_id[:self.SHORT_RESOURCE_NAME]}-{self.region}-{image_id[-self.SHORT_RESOURCE_ID:]}'
- search_tags.append({'Key': 'cg-Name', 'Value': tag_name})
- search_tags.append(self._build_tag(key='LaunchTime', value=start_time))
- image_tags = self._get_tags_of_resources(tags=image.get('Tags'), search_tags=search_tags)
- if image_tags:
- if self.dry_run == 'no':
- try:
- self.ec2_client.create_tags(Resources=[image_id], Tags=image_tags)
- logger.info(f'added tags to image: {image_id} total: {len(image_tags)} tags: {image_tags}')
- except Exception as err:
- logger.info(err)
- image_ids.append(image_id)
+ search_tags.extend(self._fill_na_tags())
+ if not self.__check_name_in_tags(image.get('Tags')):
+ tag_name = f'{username}-{image_id[-self.SHORT_RESOURCE_ID:]}' if username else f'{image_id[:self.SHORT_RESOURCE_NAME]}-{self.region}-{image_id[-self.SHORT_RESOURCE_ID:]}'
+ search_tags.append({'Key': 'cg-Name', 'Value': tag_name})
+ search_tags.append(self._build_tag(key='LaunchTime', value=start_time))
+ image_tags = self._get_tags_of_resources(tags=image.get('Tags'), search_tags=search_tags)
+ if image_tags:
+ if self.dry_run == 'no':
+ try:
+ self.utils.tag_aws_resources(client_method=self.ec2_client.create_tags, resource_ids=[image_id], tags=image_tags)
+ logger.info(f'added tags to image: {image_id} total: {len(image_tags)} tags: {image_tags}')
+ except Exception as err:
+ logger.info(err)
+ image_ids.append(image_id)
logger.info(f'non_cluster_amis count: {len(sorted(image_ids))} {sorted(image_ids)}')
return sorted(image_ids)
diff --git a/cloud_governance/policy/policy_operations/aws/tag_non_cluster/update_na_tag_resources.py b/cloud_governance/policy/policy_operations/aws/tag_non_cluster/update_na_tag_resources.py
index 0ef1f548..8a037eca 100644
--- a/cloud_governance/policy/policy_operations/aws/tag_non_cluster/update_na_tag_resources.py
+++ b/cloud_governance/policy/policy_operations/aws/tag_non_cluster/update_na_tag_resources.py
@@ -46,7 +46,7 @@ def __get_resource_ids(self, resource_id: str, resource_name: str):
def __get_key_value(self, key, value):
"""
- This method return key-value pairs
+ This method returns key-value pairs
:param key:
:param value:
:return:
diff --git a/cloud_governance/policy/policy_operations/aws/tag_user/tag_iam_user.py b/cloud_governance/policy/policy_operations/aws/tag_user/tag_iam_user.py
index eb7229cf..e359c61a 100644
--- a/cloud_governance/policy/policy_operations/aws/tag_user/tag_iam_user.py
+++ b/cloud_governance/policy/policy_operations/aws/tag_user/tag_iam_user.py
@@ -1,4 +1,5 @@
import csv
+import os.path
import re
from ast import literal_eval
@@ -77,6 +78,7 @@ def __write_into_csv_file(self, tag_keys: list, tag_values: dict):
else:
file.write(' , ')
file.write('\n')
+ logger.info(f'Generated the file: {self.file_name}')
def generate_user_csv(self):
"""
@@ -89,7 +91,7 @@ def generate_user_csv(self):
tag_values = {}
for user in users:
user_name = user.get('UserName')
- if '-' not in user_name:
+ if user_name.count('-') <= 3:
user_tags = self.IAMOperations.get_user_tags(username=user_name)
tag_values[user_name] = {}
for tag in user_tags:
@@ -105,8 +107,6 @@ def generate_user_csv(self):
break
tag_keys = list(sorted(tag_keys))
self.__write_into_csv_file(tag_keys=tag_keys, tag_values=tag_values)
- with open(self.file_name) as file:
- logger.info(file.read())
def __filter_tags_user_tags(self, user_tags: list, append_tags: list):
"""
@@ -121,8 +121,7 @@ def __filter_tags_user_tags(self, user_tags: list, append_tags: list):
found = False
for user_tag in user_tags:
if user_tag.get('Key').strip() == append_tag.get('Key').strip():
- if user_tag.get('Value').strip() == append_tag.get('Value').strip():
- found = True
+ found = True
if not found:
add_tags.append(append_tag)
else:
@@ -176,30 +175,55 @@ def update_user_tags(self):
"""
count = 0
updated_usernames = []
- with open(self.file_name) as file:
- csvreader = csv.reader(file)
- header = next(csvreader)
- rows = []
- for row in csvreader:
- rows.append(row)
- json_data = self.__get_json_data(header, rows)
- for key, tags in json_data.items():
- try:
- user_tags = self.IAMOperations.get_user_tags(username=key)
- tags.append({'Key': 'User', 'Value': key})
- tags.extend(self.get_user_details_from_ldap(user_name=key))
- filter_tags = self.__filter_tags_user_tags(user_tags, tags)
- if filter_tags:
- self.iam_client.tag_user(UserName=key, Tags=filter_tags)
- logger.info(f'Username :: {key} {filter_tags}')
+ if os.path.exists(self.file_name):
+ with open(self.file_name) as file:
+ csvreader = csv.reader(file)
+ header = next(csvreader)
+ rows = []
+ for row in csvreader:
+ rows.append(row)
+ json_data = self.__get_json_data(header, rows)
+ for key, tags in json_data.items():
+ if self.tag_iam_user_tags(username=key, tags=tags):
updated_usernames.append(key)
count += 1
- except Exception as err:
- logger.info(err)
+ else:
+ users_list = self.get_detail_resource_list(func_name=self.iam_client.list_users, input_tag='Users',
+ check_tag='Marker')
+ for user in users_list:
+ username = user.get('UserName')
+ if username.count('-') <= 3: # assumed if username contains 3 hyphens, it is cluster user
+ if self.tag_iam_user_tags(username=username):
+ updated_usernames.append(username)
+ count += 1
logger.info(f'Updated Tags of IAM Users = {count} :: Usernames {updated_usernames}')
return count
- def __format_tags(self, username: str, headers: list):
+ def tag_iam_user_tags(self, username: str, tags: list = None):
+ """
+ This method tags the IAM User tags
+ :param tags:
+ :param username:
+ :return:
+ """
+ try:
+ if not tags:
+ tags = []
+ user_tags = self.IAMOperations.get_user_tags(username=username)
+ tags.append({'Key': 'User', 'Value': username})
+ tags.extend(self.get_user_details_from_ldap(user_name=username))
+ filter_tags = self.__filter_tags_user_tags(user_tags, tags)
+ if filter_tags:
+ self.iam_client.tag_user(UserName=username, Tags=filter_tags)
+ logger.info(f'Username :: {username} {filter_tags}')
+ return True
+ except Exception as err:
+ logger.error(err)
+ return False
+
+ def __format_tags(self, username: str, headers: list = None):
+ if not headers:
+ headers = ['User']
tags = {'User': username}
user_tags = self.IAMOperations.get_user_tags(username=username)
for user_tag in user_tags:
@@ -212,19 +236,32 @@ def delete_update_user_from_doc(self):
This method removes IAM user if not in the IAM list
@return:
"""
- iam_file = pd.read_csv(self.file_name)
- iam_users = [user['UserName'] for user in self.IAMOperations.get_users()]
- csv_iam_users = list(iam_file['User'])
- for index, user in enumerate(csv_iam_users):
- if user not in iam_users:
- self.__google_drive_operations.delete_rows(spreadsheet_id=self.__SPREADSHEET_ID,
- sheet_name=self.__sheet_name, row_number=index + 1)
- logger.info(f'removed user {user}')
+ self.__google_drive_operations.create_work_sheet(gsheet_id=self.__SPREADSHEET_ID, sheet_name=self.__sheet_name)
+ iam_users = [user['UserName'] for user in
+ self.get_detail_resource_list(func_name=self.iam_client.list_users, input_tag='Users',
+ check_tag='Marker') if user['UserName'].count('-') <= 3]
+ csv_iam_users = []
+ iam_file = pd.DataFrame(columns=['User', "Project"])
+ if os.path.exists(self.file_name):
+ iam_file = pd.read_csv(self.file_name)
+ if not iam_file.empty:
+ csv_iam_users = list(iam_file['User'])
+ for index, user in enumerate(csv_iam_users):
+ if user not in iam_users:
+ self.__google_drive_operations.delete_rows(spreadsheet_id=self.__SPREADSHEET_ID,
+ sheet_name=self.__sheet_name, row_number=index + 1)
+ logger.info(f'removed user {user}')
+ else:
+ iam_file = pd.DataFrame(columns=['User'])
append_data = []
for user in iam_users:
- if '-' not in user:
+ if user.count('-') <= 3:
if user not in csv_iam_users:
- tags = self.__format_tags(username=user, headers=list(iam_file.columns))
+ if not iam_file.empty:
+ tags = self.__format_tags(username=user, headers=list(iam_file.columns))
+ else:
+ append_data.append(['User'])
+ tags = self.__format_tags(username=user)
df2 = pd.DataFrame.from_dict([tags])
iam_file = pd.concat([iam_file, df2], ignore_index=True)
iam_file = iam_file.fillna('')
diff --git a/cloud_governance/policy/policy_operations/aws/zombie_cluster/delete_ec2_resources.py b/cloud_governance/policy/policy_operations/aws/zombie_cluster/delete_ec2_resources.py
index 895c41be..2c866520 100644
--- a/cloud_governance/policy/policy_operations/aws/zombie_cluster/delete_ec2_resources.py
+++ b/cloud_governance/policy/policy_operations/aws/zombie_cluster/delete_ec2_resources.py
@@ -165,8 +165,9 @@ def __delete_load_balancer_v2(self, resource_id: str):
@typeguard.typechecked
def __delete_volume(self, resource_id: str):
try:
- self.client.delete_volume(VolumeId=resource_id)
- logger.info(f'delete_volume: {resource_id}')
+ logger.info(f'Cluster volumes are handled by ebs_unattached')
+ # self.client.delete_volume(VolumeId=resource_id)
+ # logger.info(f'delete_volume: {resource_id}')
except Exception as err:
logger.exception(f'Cannot delete_volume: {resource_id}, {err}')
@@ -260,22 +261,31 @@ def __delete_security_group(self, resource_id: str, vpc_id: str):
security_groups = self.ec2_operations.get_security_groups()
vpc_security_groups = self.__get_cluster_references(resource_id=vpc_id, resource_list=security_groups, input_resource_id='VpcId', output_result='')
for vpc_security_group in vpc_security_groups:
- if vpc_security_group.get('GroupName') == 'default':
- if vpc_security_group.get('IpPermissions'):
- for ip_permission in vpc_security_group.get('IpPermissions'):
- self.client.revoke_security_group_ingress(GroupId=vpc_security_group.get('GroupId'), IpPermissions=[ip_permission])
- logger.info(f'Removed the Ingress rules of Security Group {resource_id} :: {ip_permission}')
- else:
- if vpc_security_group.get('Tags'):
- if self.__is_cluster_resource(tags=vpc_security_group.get('Tags'), cluster_tag=self.cluster_tag):
- logger.info(vpc_security_group.get('GroupId'))
- if vpc_security_group.get('IpPermissions'):
- for ip_permission in vpc_security_group.get('IpPermissions'):
- if ip_permission.get('UserIdGroupPairs'):
- for user_id_group_pair in ip_permission.get('UserIdGroupPairs'):
- if user_id_group_pair.get('GroupId') == resource_id:
- self.client.revoke_security_group_ingress(GroupId=vpc_security_group.get('GroupId'), IpPermissions=[ip_permission])
- logger.info(f'Removed the Ingress rules of Security Group {resource_id} from {vpc_security_group.get("GroupId")}')
+ if resource_id != vpc_security_group.get('GroupId'):
+ if vpc_security_group.get('GroupName') == 'default':
+ logger.info(f'Removing the {resource_id} ingress rule from Default Security Group: {vpc_security_group.get("GroupId")}')
+ if vpc_security_group.get('IpPermissions'):
+ for ip_permission in vpc_security_group.get('IpPermissions'):
+ if ip_permission.get('UserIdGroupPairs'):
+ for user_id_group_pair in ip_permission.get('UserIdGroupPairs'):
+ if user_id_group_pair.get('GroupId') == resource_id:
+ ingress_rule = {'FromPort': ip_permission.get('FromPort'), 'IpProtocol': ip_permission.get('IpProtocol'), 'IpRanges': ip_permission.get('IpRanges'), 'Ipv6Ranges': ip_permission.get('Ipv6Ranges'), 'PrefixListIds': ip_permission.get('PrefixListIds'), 'ToPort': ip_permission.get('ToPort'), 'UserIdGroupPairs': [user_id_group_pair]}
+ self.client.revoke_security_group_ingress(GroupId=vpc_security_group.get('GroupId'), IpPermissions=[ingress_rule])
+ logger.info(f'Removed the Ingress rules of Security Group {vpc_security_group.get("GroupId")} :: {ingress_rule}')
+ else:
+ if vpc_security_group.get('Tags'):
+ if self.__is_cluster_resource(tags=vpc_security_group.get('Tags'), cluster_tag=self.cluster_tag):
+ logger.info(vpc_security_group.get('GroupId'))
+ if vpc_security_group.get('IpPermissions'):
+ for ip_permission in vpc_security_group.get('IpPermissions'):
+ if ip_permission.get('UserIdGroupPairs'):
+ for user_id_group_pair in ip_permission.get('UserIdGroupPairs'):
+ if user_id_group_pair.get('GroupId') == resource_id:
+ ingress_rule = {'FromPort': ip_permission.get('FromPort'), 'IpProtocol': ip_permission.get('IpProtocol'), 'IpRanges': ip_permission.get('IpRanges'),
+ 'Ipv6Ranges': ip_permission.get('Ipv6Ranges'), 'PrefixListIds': ip_permission.get('PrefixListIds'),
+ 'ToPort': ip_permission.get('ToPort'), 'UserIdGroupPairs': [user_id_group_pair]}
+ self.client.revoke_security_group_ingress(GroupId=vpc_security_group.get('GroupId'), IpPermissions=[ingress_rule])
+ logger.info(f'Removed the Ingress rules of Security Group {resource_id} from {ingress_rule}')
network_interfaces = self.ec2_operations.get_network_interface()
network_interface_ids = self.__get_cluster_references(resource_id=vpc_id, resource_list=network_interfaces,
input_resource_id='VpcId',
diff --git a/cloud_governance/policy/policy_operations/aws/zombie_cluster/run_zombie_cluster_resources.py b/cloud_governance/policy/policy_operations/aws/zombie_cluster/run_zombie_cluster_resources.py
index af4d93dc..b7f2c35e 100644
--- a/cloud_governance/policy/policy_operations/aws/zombie_cluster/run_zombie_cluster_resources.py
+++ b/cloud_governance/policy/policy_operations/aws/zombie_cluster/run_zombie_cluster_resources.py
@@ -82,7 +82,7 @@ def __get_resource_list(region, delete: bool = False, resource: str = '', cluste
def zombie_cluster_resource(delete: bool = False, region: str = 'us-east-2', resource: str = '', cluster_tag: str = '',
resource_name: str = '', service_type: str = ''):
"""
- This method return zombie cluster resources,
+ This method returns zombie cluster resources,
How its works? if not exist an instance cluster, the resource is zombie
if delete true it will delete the zombie resource
:return: list of zombie resources
diff --git a/cloud_governance/policy/policy_operations/aws/zombie_cluster/zombie_cluster_common_methods.py b/cloud_governance/policy/policy_operations/aws/zombie_cluster/zombie_cluster_common_methods.py
index 0eb4df60..3c545d4f 100644
--- a/cloud_governance/policy/policy_operations/aws/zombie_cluster/zombie_cluster_common_methods.py
+++ b/cloud_governance/policy/policy_operations/aws/zombie_cluster/zombie_cluster_common_methods.py
@@ -15,7 +15,7 @@
class ZombieClusterCommonMethods:
DAYS_TO_TRIGGER_RESOURCE_MAIL = 4
- DAYS_TO_DELETE_RESOURCE = 7
+ DAYS_TO_DELETE_RESOURCE = environment_variables.environment_variables_dict.get('DAYS_TO_DELETE_RESOURCE')
def __init__(self, region: str, force_delete: bool = False):
self.__environment_variables_dict = environment_variables.environment_variables_dict
@@ -32,6 +32,7 @@ def __init__(self, region: str, force_delete: bool = False):
self.__ldap_host_name = self.__environment_variables_dict.get('LDAP_HOST_NAME', '')
self._special_user_mails = self.__environment_variables_dict.get('special_user_mails', '{}')
self._account_admin = self.__environment_variables_dict.get('account_admin', '')
+ self.__email_alert = self.__environment_variables_dict.get('EMAIL_ALERT') if self.__environment_variables_dict.get('EMAIL_ALERT') else False
self._ldap = LdapSearch(ldap_host_name=self.__ldap_host_name)
self._mail = Postfix()
self._mail_description = MailMessage()
@@ -45,7 +46,7 @@ def _literal_eval(self, data: any):
def get_tag_name_from_tags(self, tags: list, tag_name: str):
"""
- This method return tag_name from resource_tags
+ This method returns tag_name from resource_tags
@param tags:
@param tag_name:
@return:
@@ -75,7 +76,7 @@ def get_zombie_cluster_user_tag(self, zombies: dict, resources: list, resource_i
def _get_tags_of_zombie_resources(self, resources: list, resource_id_name: str, zombies: dict, aws_service: str,
aws_tag: str = 'Tags'):
"""
- This method return tags of the resource i.e {resource_id: tags}
+ This method returns tags of the resource i.e {resource_id: tags}
@param resources:
@param tags:
@return:
@@ -177,7 +178,7 @@ def update_resource_tags(self, tags: list, tag_name: str, tag_value: str):
def get_cluster_delete_days(self, tags: list) -> int:
"""
- This method return the ClusterDeleteDays tag
+ This method returns the ClusterDeleteDays tag
@param tags:
@return:
"""
@@ -188,7 +189,6 @@ def get_cluster_delete_days(self, tags: list) -> int:
cluster_delete_days = int(cluster_delete_days) + 1
return cluster_delete_days
- @logger_time_stamp
def trigger_mail(self, tags: list, resource_id: str, days: int, resources: list, message_type: str):
"""
This method send triggering mail
@@ -254,7 +254,6 @@ def collect_notify_cluster_data(self, resource_data: dict, cluster_left_out_days
delete_data.setdefault(cluster_tag, []).append({func_name: delete_tag_data[cluster_tag]})
return notify_data, delete_data, cluster_data
- @logger_time_stamp
def send_mails_to_cluster_user(self, notify_data: dict, delete_data: dict, cluster_data: dict):
"""
This method send mail to the user to notify cluster status
@@ -263,17 +262,17 @@ def send_mails_to_cluster_user(self, notify_data: dict, delete_data: dict, clust
@param delete_data:
@return:
"""
- for cluster_tag, resource_ids in notify_data.items():
- self.update_resource_tags(tags=cluster_data[cluster_tag], tag_name='Name', tag_value=cluster_tag)
- self.trigger_mail(tags=cluster_data[cluster_tag], resource_id=cluster_tag,
- days=self.DAYS_TO_TRIGGER_RESOURCE_MAIL,
- resources=resource_ids, message_type='notification')
- for cluster_tag, resource_ids in delete_data.items():
- self.update_resource_tags(tags=cluster_data[cluster_tag], tag_name='Name', tag_value=cluster_tag)
- self.trigger_mail(tags=cluster_data[cluster_tag], resource_id=cluster_tag,
- days=self.DAYS_TO_DELETE_RESOURCE, resources=resource_ids, message_type='delete')
+ if self.__email_alert:
+ for cluster_tag, resource_ids in notify_data.items():
+ self.update_resource_tags(tags=cluster_data[cluster_tag], tag_name='Name', tag_value=cluster_tag)
+ self.trigger_mail(tags=cluster_data[cluster_tag], resource_id=cluster_tag,
+ days=self.DAYS_TO_TRIGGER_RESOURCE_MAIL,
+ resources=resource_ids, message_type='notification')
+ for cluster_tag, resource_ids in delete_data.items():
+ self.update_resource_tags(tags=cluster_data[cluster_tag], tag_name='Name', tag_value=cluster_tag)
+ self.trigger_mail(tags=cluster_data[cluster_tag], resource_id=cluster_tag,
+ days=self.DAYS_TO_DELETE_RESOURCE, resources=resource_ids, message_type='delete')
- @logger_time_stamp
def _check_zombie_cluster_deleted_days(self, resources: dict, cluster_left_out_days: dict, zombie: str, cluster_tag: str):
"""
This method check the cluster delete days and return the clusters
diff --git a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py
index ae13efbf..c5f37a9d 100644
--- a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py
+++ b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py
@@ -9,6 +9,7 @@
from cloud_governance.common.clouds.aws.price.resources_pricing import ResourcesPricing
from cloud_governance.common.clouds.aws.s3.s3_operations import S3Operations
from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
+from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations
from cloud_governance.common.ldap.ldap_search import LdapSearch
from cloud_governance.common.logger.init_logger import logger
from cloud_governance.common.mails.mail_message import MailMessage
@@ -19,7 +20,7 @@
class NonClusterZombiePolicy:
- DAYS_TO_DELETE_RESOURCE = 7
+ DAYS_TO_DELETE_RESOURCE = environment_variables.environment_variables_dict.get('DAYS_TO_DELETE_RESOURCE')
DAYS_TO_NOTIFY_ADMINS = 6
DAYS_TO_TRIGGER_RESOURCE_MAIL = 4
DAILY_HOURS = 24
@@ -48,9 +49,13 @@ def __init__(self):
self._mail_description = MailMessage()
self.__ldap_host_name = self.__environment_variables_dict.get('LDAP_HOST_NAME', '')
self._ldap = LdapSearch(ldap_host_name=self.__ldap_host_name)
+ self.__email_alert = self.__environment_variables_dict.get('EMAIL_ALERT') if self.__environment_variables_dict.get('EMAIL_ALERT') else False
+ self.__manager_email_alert = self.__environment_variables_dict.get('MANAGER_EMAIL_ALERT')
self._admins = ['athiruma@redhat.com', 'ebattat@redhat.com']
self._es_upload = ElasticUpload()
self.resource_pricing = ResourcesPricing()
+ self._es_operations = ElasticSearchOperations()
+ self._es_index = self.__environment_variables_dict.get('es_index')
def set_dryrun(self, value: str):
self._dry_run = value
@@ -95,7 +100,7 @@ def _get_tag_name_from_tags(self, tags: list, tag_name: str = 'Name'):
def _calculate_days(self, create_date: datetime):
"""
- This method return the days
+ This method returns the days
@return:
"""
today = datetime.date.today()
@@ -164,30 +169,32 @@ def _trigger_mail(self, tags: list, resource_id: str, days: int, resource_type:
@param resource_id:
@return:
"""
- try:
- special_user_mails = self._literal_eval(self._special_user_mails)
- user, resource_name = self._get_tag_name_from_tags(tags=tags, tag_name='User'), self._get_tag_name_from_tags(
- tags=tags, tag_name='Name')
- if not resource_name:
- resource_name = self._get_tag_name_from_tags(tags=tags, tag_name='cg-Name')
- to = user if user not in special_user_mails else special_user_mails[user]
- ldap_data = self._ldap.get_user_details(user_name=to)
- cc = [self._account_admin, f'{ldap_data.get("managerId")}@redhat.com']
- name = to
- if ldap_data:
- name = ldap_data.get('displayName')
- subject, body = self._mail_description.resource_message(name=name, days=days,
- notification_days=self.DAYS_TO_TRIGGER_RESOURCE_MAIL,
- delete_days=self.DAYS_TO_DELETE_RESOURCE,
- resource_name=resource_name, resource_id=resource_id,
- resource_type=resource_type, msgadmins=self.DAYS_TO_NOTIFY_ADMINS, extra_purse=kwargs.get('extra_purse'))
- if not kwargs.get('admins'):
- self._mail.send_email_postfix(to=to, content=body, subject=subject, cc=cc, resource_id=resource_id, message_type=kwargs.get('message_type'), extra_purse=kwargs.get('delta_cost', 0))
- else:
- kwargs['admins'].append(f'{ldap_data.get("managerId")}@redhat.com')
- self._mail.send_email_postfix(to=kwargs.get('admins'), content=body, subject=subject, cc=[], resource_id=resource_id, message_type=kwargs.get('message_type'), extra_purse=kwargs.get('delta_cost', 0))
- except Exception as err:
- logger.info(err)
+ if self.__email_alert:
+ try:
+ special_user_mails = self._literal_eval(self._special_user_mails)
+ user, resource_name = self._get_tag_name_from_tags(tags=tags, tag_name='User'), self._get_tag_name_from_tags(
+ tags=tags, tag_name='Name')
+ if not resource_name:
+ resource_name = self._get_tag_name_from_tags(tags=tags, tag_name='cg-Name')
+ to = user if user not in special_user_mails else special_user_mails[user]
+ ldap_data = self._ldap.get_user_details(user_name=to)
+ cc = [self._account_admin, f'{ldap_data.get("managerId")}@redhat.com'] if self.__manager_email_alert else []
+ name = to
+ if ldap_data:
+ name = ldap_data.get('displayName')
+ subject, body = self._mail_description.resource_message(name=name, days=days,
+ notification_days=self.DAYS_TO_TRIGGER_RESOURCE_MAIL,
+ delete_days=self.DAYS_TO_DELETE_RESOURCE,
+ resource_name=resource_name, resource_id=resource_id,
+ resource_type=resource_type, msgadmins=self.DAYS_TO_NOTIFY_ADMINS, extra_purse=kwargs.get('extra_purse'))
+ if not kwargs.get('admins'):
+ self._mail.send_email_postfix(to=to, content=body, subject=subject, cc=cc, resource_id=resource_id, message_type=kwargs.get('message_type'), extra_purse=kwargs.get('delta_cost', 0))
+ else:
+ if self.__manager_email_alert:
+ kwargs['admins'].append(f'{ldap_data.get("managerId")}@redhat.com')
+ self._mail.send_email_postfix(to=kwargs.get('admins'), content=body, subject=subject, cc=[], resource_id=resource_id, message_type=kwargs.get('message_type'), extra_purse=kwargs.get('delta_cost', 0))
+ except Exception as err:
+ logger.info(err)
def _update_tag_value(self, tags: list, tag_name: str, tag_value: str):
"""
@@ -231,7 +238,7 @@ def __delete_resource_on_name(self, resource_id: str):
self._ec2_client.delete_volume(VolumeId=resource_id)
elif self._policy == 'ip_unattached':
self._ec2_client.release_address(AllocationId=resource_id)
- elif self._policy == 'nat_gateway_unused':
+ elif self._policy == 'unused_nat_gateway':
self._ec2_client.delete_nat_gateway(NatGatewayId=resource_id)
elif self._policy == 'zombie_snapshots':
self._ec2_client.delete_snapshot(SnapshotId=resource_id)
@@ -287,7 +294,7 @@ def _update_resource_tags(self, resource_id: str, left_out_days: int, tags: list
self._s3_client.put_bucket_tagging(Bucket=resource_id, Tagging={'TagSet': tags})
elif self._policy == 'empty_roles':
self._iam_client.tag_role(RoleName=resource_id, Tags=tags)
- elif self._policy in ('ip_unattached', 'nat_gateway_unused', 'zombie_snapshots'):
+ elif self._policy in ('ip_unattached', 'unused_nat_gateway', 'zombie_snapshots'):
self._ec2_client.create_tags(Resources=[resource_id], Tags=tags)
except Exception as err:
logger.info(f'Exception raised: {err}: {resource_id}')
diff --git a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py
index cc3bcdf0..2e22b291 100644
--- a/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py
+++ b/cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py
@@ -1,5 +1,6 @@
import importlib
import inspect
+from datetime import datetime
from cloud_governance.common.logger.init_logger import logger
from cloud_governance.policy.policy_operations.aws.zombie_non_cluster.run_zombie_non_cluster_policies import NonClusterZombiePolicy
@@ -26,9 +27,22 @@ def run(self):
else:
logger.info(f'key: {cls[0]}, count: {len(response)}, {response}')
policy_result = response
+
+ if self._es_operations.check_elastic_search_connection():
+ if policy_result:
+ for policy_dict in policy_result:
+ policy_dict['region_name'] = self._region
+ policy_dict['account'] = self._account
+ self._es_operations.upload_to_elasticsearch(data=policy_dict, index=self._es_index)
+ logger.info(f'Uploaded the policy results to elasticsearch index: {self._es_index}')
+ else:
+ logger.error(f'No data to upload on @{self._account} at {datetime.utcnow()}')
+ else:
+ logger.error('ElasticSearch host is not pingable, Please check ')
+
if self._policy_output:
- if self._policy not in ('ec2_idle', 'ebs_in_use', 'ec2_run'):
- beautify_data = self._beautify_upload_data(upload_resource_data=response)
- policy_result = {'count': len(beautify_data), self._policy: beautify_data}
+ # if self._policy not in ('ec2_idle', 'ebs_in_use', 'ec2_run', 's3_inactive', 'zombie_snapshots', 'nat_gateway_unused'):
+ # beautify_data = self._beautify_upload_data(upload_resource_data=response)
+ # policy_result = {'count': len(beautify_data), self._policy: beautify_data}
logger.info(policy_result)
self._s3operations.save_results_to_s3(policy=self._policy.replace('_', '-'), policy_output=self._policy_output, policy_result=policy_result)
diff --git a/cloud_governance/policy/policy_operations/gcp/__init__.py b/cloud_governance/policy/policy_operations/gcp/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloud_governance/policy/policy_operations/gcp/gcp_policy_runner.py b/cloud_governance/policy/policy_operations/gcp/gcp_policy_runner.py
new file mode 100644
index 00000000..01675d33
--- /dev/null
+++ b/cloud_governance/policy/policy_operations/gcp/gcp_policy_runner.py
@@ -0,0 +1,29 @@
+import importlib
+import inspect
+
+from cloud_governance.common.jira.jira import logger
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class GcpPolicyRunner:
+ """
+ This method run the azure policies
+ """
+
+ def __init__(self):
+ self.__environment_variables_dict = environment_variables.environment_variables_dict
+ self._policy = self.__environment_variables_dict.get('policy')
+ self._cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME')
+
+ def run(self):
+ """
+ Run the azure policies
+ @return:
+ """
+ azure_policies = importlib.import_module(f'cloud_governance.policy.gcp.{self._policy}')
+ logger.info(f'Account: {self._cloud_name}, Policy: {self._policy}')
+ for cls in inspect.getmembers(azure_policies, inspect.isclass):
+ if self._policy.replace('_', '') == cls[0].lower():
+ response = cls[1]().run()
+ if isinstance(response, list) and len(response) > 0:
+ logger.info(f'key: {cls[0]}, count: {len(response)}, {response}')
diff --git a/cloud_governance/policy/policy_operations/gitleaks/gitleaks.py b/cloud_governance/policy/policy_operations/gitleaks/gitleaks.py
index 31cf3670..a559fac4 100644
--- a/cloud_governance/policy/policy_operations/gitleaks/gitleaks.py
+++ b/cloud_governance/policy/policy_operations/gitleaks/gitleaks.py
@@ -39,7 +39,7 @@ def __delete_gitleaks_report(self):
def __get_gitleaks_report(self):
"""
- This method return dict report content
+ This method returns dict report content
"""
report_file = self.__report_file_full_path
if os.path.isfile(report_file):
diff --git a/cloud_governance/policy/policy_runners/__init__.py b/cloud_governance/policy/policy_runners/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/cloud_governance/policy/policy_runners/common_policy_runner.py b/cloud_governance/policy/policy_runners/common_policy_runner.py
new file mode 100644
index 00000000..4f54fd7a
--- /dev/null
+++ b/cloud_governance/policy/policy_runners/common_policy_runner.py
@@ -0,0 +1,29 @@
+import importlib
+import inspect
+
+from cloud_governance.common.logger.init_logger import logger
+from cloud_governance.main.environment_variables import environment_variables
+
+
+class CommonPolicyRunner:
+ """
+ This method run the Common policies
+ """
+
+ def __init__(self):
+ self.__environment_variables_dict = environment_variables.environment_variables_dict
+ self._policy = self.__environment_variables_dict.get('policy')
+ self._cloud_name = self.__environment_variables_dict.get('PUBLIC_CLOUD_NAME')
+
+ def run(self):
+ """
+ Run the Common policies
+ @return:
+ """
+ azure_policies = importlib.import_module(f'cloud_governance.policy.common_policies.{self._policy}')
+ logger.info(f'Account: {self._cloud_name}, Policy: {self._policy}, CLOUD_NAME: {self._cloud_name}')
+ for cls in inspect.getmembers(azure_policies, inspect.isclass):
+ if self._policy.replace('_', '') == cls[0].lower():
+ response = cls[1]().run()
+ if isinstance(response, list) and len(response) > 0:
+ logger.info(f'key: {cls[0]}, count: {len(response)}, {response}')
diff --git a/clouds_data_ware_house/cloud_resource_orchestration/clouds/aws/instance_types_pricing.py b/clouds_data_ware_house/cloud_resource_orchestration/clouds/aws/instance_types_pricing.py
new file mode 100644
index 00000000..700f12fa
--- /dev/null
+++ b/clouds_data_ware_house/cloud_resource_orchestration/clouds/aws/instance_types_pricing.py
@@ -0,0 +1,74 @@
+import json
+import os
+
+import boto3
+from pkg_resources import resource_filename
+
+
+class InstanceTypes:
+
+ def __init__(self):
+ self.ec2_client = boto3.client('ec2', region_name='us-east-1')
+ self.__client = boto3.client('pricing', region_name='us-east-1')
+
+ def get_instance_types(self, region_name: str):
+ """This method fetch all instance types"""
+ instance_types = []
+ ec2_client = boto3.client('ec2', region_name=region_name)
+ response = ec2_client.describe_instance_types()
+ instance_types.extend([ins_type['InstanceType'] for ins_type in response.get('InstanceTypes')])
+ while 'NextToken' in response:
+ response = ec2_client.describe_instance_types(NextToken=response.get('NextToken'))
+ instance_types.extend([ins_type['InstanceType'] for ins_type in response.get('InstanceTypes')])
+ return sorted(instance_types)
+
+ def get_region_name(self, region_code):
+ """
+ This method return region name
+ @param region_code:
+ @return:
+ """
+ default_region = 'us-east-1'
+ endpoint_file = resource_filename('botocore', 'data/endpoints.json')
+ try:
+ with open(endpoint_file, 'r') as f:
+ data = json.load(f)
+ return data['partitions'][0]['regions'][region_code]['description']
+ except IOError:
+ return default_region
+
+ def instance_price(self, region_name: str, instance_type: str):
+ """This method give price of instance type in a region"""
+ FLT = '[{{"Field": "tenancy", "Value": "shared", "Type": "TERM_MATCH"}},' \
+ '{{"Field": "operatingSystem", "Value": "Linux", "Type": "TERM_MATCH"}},' \
+ '{{"Field": "preInstalledSw", "Value": "NA", "Type": "TERM_MATCH"}},' \
+ '{{"Field": "instanceType", "Value": "{t}", "Type": "TERM_MATCH"}},' \
+ '{{"Field": "location", "Value": "{r}", "Type": "TERM_MATCH"}},' \
+ '{{"Field": "capacitystatus", "Value": "Used", "Type": "TERM_MATCH"}}]'
+ f = FLT.format(r=self.get_region_name(region_name), t=instance_type)
+ try:
+ data = self.__client.get_products(ServiceCode='AmazonEC2', Filters=json.loads(f))
+ od = json.loads(data['PriceList'][0])['terms']['OnDemand']
+ id1 = list(od)[0]
+ id2 = list(od[id1]['priceDimensions'])[0]
+ return od[id1]['priceDimensions'][id2]['pricePerUnit']['USD']
+ except Exception as err:
+ return 0
+
+ def instance_prices(self):
+ """This method get the instance prices based on instance_type"""
+ # regions = self.ec2_client.describe_regions()['Regions']
+ # aws_pricing = {}
+ # for region in regions:
+ region_pricing = {}
+ instance_types = self.get_instance_types(region_name='us-west-2')
+ for instance_type in instance_types:
+ price = self.instance_price(region_name='us-west-2', instance_type=instance_type)
+ if float(price) > 0:
+ region_pricing[instance_type] = round(float(price), 4)
+ # aws_pricing[region['RegionName']] = region_pricing
+ with open('instances_price.json', 'w') as file:
+ json.dump(region_pricing, file, indent=4)
+
+
+# InstanceTypes().instance_prices()
diff --git a/clouds_data_ware_house/cloud_resource_orchestration/clouds/aws/instances_price.json b/clouds_data_ware_house/cloud_resource_orchestration/clouds/aws/instances_price.json
index 0e21982e..dc99790d 100644
--- a/clouds_data_ware_house/cloud_resource_orchestration/clouds/aws/instances_price.json
+++ b/clouds_data_ware_house/cloud_resource_orchestration/clouds/aws/instances_price.json
@@ -123,6 +123,7 @@
"c6in.4xlarge": 0.9072,
"c6in.8xlarge": 1.8144,
"c6in.large": 0.1134,
+ "c6in.metal": 7.2576,
"c6in.xlarge": 0.2268,
"c7g.12xlarge": 1.74,
"c7g.16xlarge": 2.32,
@@ -131,8 +132,8 @@
"c7g.8xlarge": 1.16,
"c7g.large": 0.0725,
"c7g.medium": 0.0363,
+ "c7g.metal": 2.32,
"c7g.xlarge": 0.145,
- "cc2.8xlarge": 2.0,
"d2.2xlarge": 1.38,
"d2.4xlarge": 2.76,
"d2.8xlarge": 5.52,
@@ -206,6 +207,12 @@
"i3en.large": 0.226,
"i3en.metal": 10.848,
"i3en.xlarge": 0.452,
+ "i4g.16xlarge": 4.9421,
+ "i4g.2xlarge": 0.6178,
+ "i4g.4xlarge": 1.2355,
+ "i4g.8xlarge": 2.471,
+ "i4g.large": 0.1544,
+ "i4g.xlarge": 0.3089,
"i4i.16xlarge": 5.491,
"i4i.2xlarge": 0.686,
"i4i.32xlarge": 10.9824,
@@ -363,6 +370,7 @@
"m6idn.4xlarge": 1.273,
"m6idn.8xlarge": 2.5459,
"m6idn.large": 0.1591,
+ "m6idn.metal": 10.1837,
"m6idn.xlarge": 0.3182,
"m6in.12xlarge": 3.3415,
"m6in.16xlarge": 4.4554,
@@ -372,7 +380,17 @@
"m6in.4xlarge": 1.1138,
"m6in.8xlarge": 2.2277,
"m6in.large": 0.1392,
+ "m6in.metal": 8.9107,
"m6in.xlarge": 0.2785,
+ "m7g.12xlarge": 1.9584,
+ "m7g.16xlarge": 2.6112,
+ "m7g.2xlarge": 0.3264,
+ "m7g.4xlarge": 0.6528,
+ "m7g.8xlarge": 1.3056,
+ "m7g.large": 0.0816,
+ "m7g.medium": 0.0408,
+ "m7g.metal": 2.6112,
+ "m7g.xlarge": 0.1632,
"p2.16xlarge": 14.4,
"p2.8xlarge": 7.2,
"p2.xlarge": 0.9,
@@ -510,6 +528,7 @@
"r6idn.4xlarge": 1.5631,
"r6idn.8xlarge": 3.1262,
"r6idn.large": 0.1954,
+ "r6idn.metal": 12.505,
"r6idn.xlarge": 0.3908,
"r6in.12xlarge": 4.1839,
"r6in.16xlarge": 5.5786,
@@ -519,7 +538,17 @@
"r6in.4xlarge": 1.3946,
"r6in.8xlarge": 2.7893,
"r6in.large": 0.1743,
+ "r6in.metal": 11.1571,
"r6in.xlarge": 0.3487,
+ "r7g.12xlarge": 2.5704,
+ "r7g.16xlarge": 3.4272,
+ "r7g.2xlarge": 0.4284,
+ "r7g.4xlarge": 0.8568,
+ "r7g.8xlarge": 1.7136,
+ "r7g.large": 0.1071,
+ "r7g.medium": 0.0536,
+ "r7g.metal": 3.4272,
+ "r7g.xlarge": 0.2142,
"t1.micro": 0.02,
"t2.2xlarge": 0.3712,
"t2.large": 0.0928,
@@ -551,6 +580,7 @@
"t4g.xlarge": 0.1344,
"trn1.2xlarge": 1.3438,
"trn1.32xlarge": 21.5,
+ "trn1n.32xlarge": 24.78,
"u-12tb1.112xlarge": 109.2,
"u-18tb1.112xlarge": 163.8,
"u-3tb1.56xlarge": 27.3,
@@ -602,4 +632,4 @@
"z1d.large": 0.186,
"z1d.metal": 4.464,
"z1d.xlarge": 0.372
-}
+}
\ No newline at end of file
diff --git a/cloudsensei/README.md b/cloudsensei/README.md
new file mode 100644
index 00000000..03dc3967
--- /dev/null
+++ b/cloudsensei/README.md
@@ -0,0 +1,88 @@
+## CloudSensei
+
+CloudSensei is an effort to uncover potential cloud resource leaks which might lead to inefficient cloud management. Currently, CloudSensei helps generate a daily report allowing stakeholders to action on long-running EC2 instances.
+
+#### To-Do:
+Eventually, CloudSensei will be flipped to work as a Slackbot, allowing users to join a “read-only” Slack channel to review daily expense reports for instance.
+
+### How it works?
+To implement this functionality, CloudSensei utilizes AWS Lambda + EventBridge.
+The EventBridge Scheduler (CronJob) will run on every day 17:00hrs IST.
+
+
+#### How to send Slack notifications on Slack?
+1. Create a new Slack bot on your slack workspace, add it to desired channel
+2. Generate [OAuth](https://api.slack.com/authentication/token-types#bot) Token for Slack Bot
+3. Use [Block Kit](https://api.slack.com/block-kit) to build message formats.
+4. Use Slack [postMessage API](https://api.slack.com/methods/chat.postMessage) to post messages to Slack channel
+
+#### Steps to create Slack bot:
+1. Go to [api.slack.com](https://api.slack.com/)
+2. Click on **Your apps** and click on **Manage your apps**.
+3. Click on **Create New App**.
+4. Select create from scratch.
+ 1. Enter necessary fields and create app
+5. A Basic information tab will open, select options **Bots**.
+6. Click on **OAuth & Permissions** on left panel
+ 1. Click on
+ 2. Under the scopes, add only **Bots Token Scopes**
+ 3. Under the **OAuth Tokens for Your Workspace**, Submit **Install to Workspace** and allow access.
+
+#### Adding Slack bot to your channel
+
+To Create a Lambda function & integrate with EventBridge you must export some env variables:
+
+Fill the env.txt file to export environment variables
+
+To store the data of long-running instances in elastic search
+export below variables
+```commandline
+ACCOUNT_ID=$
+AWS_DEFAULT_REGION=$
+RESOURCE_DAYS=$
+SEND_AGG_MAIL=$
+ES_SERVER=$
+```
+
+To send mail of the long-running instances
+export below variables
+```commandline
+ACCOUNT_ID=$
+AWS_DEFAULT_REGION=$
+RESOURCE_DAYS=$
+SES_HOST_ADDRESS=$
+SES_HOST_PORT=$
+SES_USER_ID=$
+SES_PASSWORD=$
+TO_ADDRESS=$
+CC_ADDRESS=$
+```
+
+To send Slack notifications in Slack channel
+export below variables
+```commandline
+ACCOUNT_ID=$
+AWS_DEFAULT_REGION=$
+RESOURCE_DAYS=$
+SLACK_API_TOKEN=$
+SLACK_CHANNEL_NAME=$
+```
+
+Note: Use env.txt to export above varibales
+
+```commandline
+git clone https://github.com/redhat-performance/cloud-governance
+cd cloud-governance/cloudsensei/
+./run.sh deploy
+# Copy the tfstate file backup, incase it is deleted, we cannot retrieve it
+```
+
+To delete the Lambda + Event_bridge service [ ** must have the tfstate file]
+```commandline
+cd cloudsensei
+./run.sh destroy
+```
+
+##### Limits of BlockKit
+
+1. Can only send 50 item per block.
diff --git a/cloudsensei/agg_lambda/lambda_function.py b/cloudsensei/agg_lambda/lambda_function.py
new file mode 100644
index 00000000..cfd96c5f
--- /dev/null
+++ b/cloudsensei/agg_lambda/lambda_function.py
@@ -0,0 +1,43 @@
+import json
+import logging
+import os
+from datetime import datetime
+from time import time
+
+from es_operations import ESOperations
+from send_email import send_email_with_ses
+
+
+def lambda_handler(event, context):
+ """
+ This lambda function sends notifications to slack on long running resources on AWS Cloud
+ :param event:
+ :param context:
+ :return:
+ """
+ start_time = time()
+ logging.info(f"{lambda_handler.__name__} started at {datetime.utcnow()}")
+ aws_accounts = ["perf-dept", "openshift-perfscale", "openshift-psap"]
+ code = 400
+ message = "Something went wrong check your es_data"
+ es_operations = ESOperations()
+ email_body = ""
+ subject = "Weekly Cloud Report: Long running instances in the Perf&Scale AWS Accounts"
+ for account in aws_accounts:
+ current_date = str(datetime.utcnow().date())
+ index_id = f"{account}-{current_date}"
+ es_data = es_operations.get_es_data_by_id(index_id)
+ if es_data:
+ email_body += f"Cloud Report: Long running instances in the @{account} account
"
+ email_body += es_data.get('_source').get('body')
+ email_body += "
"
+ response = send_email_with_ses(body=email_body, subject=subject, to=os.environ.get('TO_ADDRESS'), cc=os.environ.get('CC_ADDRESS'))
+ if response:
+ code = 200
+ message = "Successfully sent an emails"
+ end_time = time()
+ return {
+ 'statusCode': code,
+ 'body': json.dumps(message),
+ 'total_running_time': f"{end_time - start_time} s"
+ }
diff --git a/cloudsensei/agg_lambda/run.sh b/cloudsensei/agg_lambda/run.sh
new file mode 100755
index 00000000..81cbffbf
--- /dev/null
+++ b/cloudsensei/agg_lambda/run.sh
@@ -0,0 +1,22 @@
+PROJECT_NAME="AggFunction"
+SUCCESS_OUTPUT_PATH="/dev/null"
+ERROR_LOG="$(mktemp -d)/stderr.log"
+
+
+echo "Clearing if previously created zip file"
+PROJECT_PATH="$PWD/$PROJECT_NAME.zip"
+if [ -f $PROJECT_PATH ]; then
+ rm -rf $PROJECT_PATH
+ rm -rf ./package
+ echo "Deleted Previously created zip file"
+fi
+
+pip install --upgrade pip
+pip install --target ./package -r ../requirements.txt > $SUCCESS_OUTPUT_PATH
+pushd package
+zip -r ../$PROJECT_NAME.zip . > $SUCCESS_OUTPUT_PATH
+popd
+zip -g $PROJECT_NAME.zip lambda_function.py > $SUCCESS_OUTPUT_PATH
+zip -g $PROJECT_NAME.zip ../es_operations.py > $SUCCESS_OUTPUT_PATH
+zip -g $PROJECT_NAME.zip ../send_email.py > $SUCCESS_OUTPUT_PATH
+aws lambda update-function-code --function-name CloudSenseiAggFunction --zip-file fileb://$PROJECT_PATH --region $AWS_DEFAULT_REGION > $SUCCESS_OUTPUT_PATH
diff --git a/cloudsensei/email_template.j2 b/cloudsensei/email_template.j2
new file mode 100644
index 00000000..5c4019b9
--- /dev/null
+++ b/cloudsensei/email_template.j2
@@ -0,0 +1,26 @@
+{% set style="border-collapse:collapse;border:2px solid black;padding: 10px" %}
+
+
+
+
+ {% for keys in keys_list %}
+ | {{keys}} |
+ {% endfor %}
+
+
+
+ {% for user, region_list in resources_list.items() %}
+ {% for region_name, resources_list in region_list.items() %}
+ {% for resources in resources_list %}
+ {% set _ = resources.update({'User': user, 'Region': region_name}) %}
+
+ {% for key in keys_list %}
+ | {{resources[key]}} |
+ {% endfor %}
+
+ {% endfor %}
+ {% endfor %}
+ {% endfor %}
+
+
+