diff --git a/app.py b/app.py index 45992f42..c2db7734 100644 --- a/app.py +++ b/app.py @@ -1,23 +1,23 @@ #!/usr/bin/env python3 import os -from pathlib import Path -from typing import List import aws_cdk as cdk -from aws_cdk import aws_config, aws_stepfunctions +from git import Repo from stacks.control_broker_stack import ( ControlBrokerStack, ) from stacks.pipeline_stack import GitHubCDKPipelineStack from stacks.test_stack import TestStack -from stacks.client_stack import ClientStack +from stacks.endpoint_stack import EndpointStack +from utils.environment import is_pipeline_synth -STACK_VERSION = "V0x6x3" +STACK_VERSION = "V0x7x0" app = cdk.App() -continuously_deployed = app.node.try_get_context( - "control-broker/continuous-deployment/enabled" +continuously_deployed = ( + app.node.try_get_context("control-broker/continuous-deployment/enabled") + or is_pipeline_synth() ) deploy_stage = None if continuously_deployed: @@ -39,19 +39,23 @@ f"ControlBrokerTestStack{STACK_VERSION}", control_broker_outer_state_machine=control_broker_stack.outer_eval_engine_state_machine, control_broker_roles=control_broker_stack.Input_reader_roles, - env=env + env=env, ) if app.node.try_get_context("control-broker/client/enabled"): - ClientStack( + EndpointStack( deploy_stage or app, - f"ControlBrokerClientStack{STACK_VERSION}", + f"ControlBrokerEndpointStack{STACK_VERSION}", control_broker_outer_state_machine=control_broker_stack.outer_eval_engine_state_machine, control_broker_roles=control_broker_stack.Input_reader_roles, control_broker_eval_results_bucket=control_broker_stack.eval_results_reports_bucket, - env=env + env=env, ) if continuously_deployed: + try: + current_branch = Repo().active_branch.name + except TypeError: + current_branch = None pipeline_stack = GitHubCDKPipelineStack( app, "ControlBrokerCICDDeployment", @@ -59,6 +63,7 @@ **app.node.try_get_context( "control-broker/continuous-deployment/github-config" ), + github_repo_branch=current_branch ) pipeline_stack.pipeline.add_stage(deploy_stage) app.synth() diff --git a/cdk.json b/cdk.json index 4881eb24..d4c154b8 100644 --- a/cdk.json +++ b/cdk.json @@ -26,14 +26,12 @@ "aws" ], "performance-testing-example-template": "supplementary_files/ExampleStack.template.json", - "control-broker/config-rule/enabled": true, "control-broker/continuous-deployment/enabled": false, "control-broker/continuous-deployment/github-config": { "github_repo_name": "control-broker", - "github_repo_owner": "VerticalRelevance", - "github_repo_branch": "main" + "github_repo_owner": "VerticalRelevance" }, - "control-broker/post-deployment-testing/enabled": true, + "control-broker/post-deployment-testing/enabled": false, "control-broker/client/enabled": true, "control-broker/secret-config/secrets-manager-secret-id": "control-broker/secret-config" } diff --git a/docs/open_api/.openapi-generator-ignore b/docs/open_api/.openapi-generator-ignore new file mode 100644 index 00000000..7484ee59 --- /dev/null +++ b/docs/open_api/.openapi-generator-ignore @@ -0,0 +1,23 @@ +# OpenAPI Generator Ignore +# Generated by openapi-generator https://github.com/openapitools/openapi-generator + +# Use this file to prevent files from being overwritten by the generator. +# The patterns follow closely to .gitignore or .dockerignore. + +# As an example, the C# client generator defines ApiClient.cs. +# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line: +#ApiClient.cs + +# You can match any string of characters against a directory, file or extension with a single asterisk (*): +#foo/*/qux +# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux + +# You can recursively match patterns against a directory, file or extension with a double asterisk (**): +#foo/**/qux +# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux + +# You can also negate patterns with an exclamation (!). +# For example, you can ignore all files in a docs folder with the file extension .md: +#docs/*.md +# Then explicitly reverse the ignore rule for a single file: +#!docs/README.md diff --git a/docs/open_api/.openapi-generator/FILES b/docs/open_api/.openapi-generator/FILES new file mode 100644 index 00000000..869df55c --- /dev/null +++ b/docs/open_api/.openapi-generator/FILES @@ -0,0 +1,2 @@ +.openapi-generator-ignore +README.md diff --git a/docs/open_api/.openapi-generator/VERSION b/docs/open_api/.openapi-generator/VERSION new file mode 100644 index 00000000..1e20ec35 --- /dev/null +++ b/docs/open_api/.openapi-generator/VERSION @@ -0,0 +1 @@ +5.4.0 \ No newline at end of file diff --git a/docs/open_api/README.md b/docs/open_api/README.md new file mode 100644 index 00000000..6c6352fd --- /dev/null +++ b/docs/open_api/README.md @@ -0,0 +1,2 @@ +# OpenAPI JSON +This is a OpenAPI JSON built by the [openapi-generator](https://github.com/openapitools/openapi-genreator) project. \ No newline at end of file diff --git a/docs/open_api/openapi.json b/docs/open_api/openapi.json new file mode 100644 index 00000000..ad98c6a8 --- /dev/null +++ b/docs/open_api/openapi.json @@ -0,0 +1,103 @@ +{ + "openapi" : "3.0.1", + "info" : { + "title" : "ControlBrokerEndpoint", + "version" : "2022-05-05 12:19:57UTC" + }, + "servers" : [ { + "url" : "https://v8i28ze8df.execute-api.us-east-1.amazonaws.com/{basePath}", + "variables" : { + "basePath" : { } + } + } ], + "tags" : [ { + "name" : "aws:cloudformation:stack-id", + "x-amazon-apigateway-tag-value" : "arn:aws:cloudformation:us-east-1:446960196218:stack/ControlBrokerEndpointStackV0x7x0/0cfbb700-cc6c-11ec-bc72-0e5c5ed49289" + }, { + "name" : "aws:cloudformation:stack-name", + "x-amazon-apigateway-tag-value" : "ControlBrokerEndpointStackV0x7x0" + }, { + "name" : "aws:cloudformation:logical-id", + "x-amazon-apigateway-tag-value" : "ControlBrokerEndpoint7427912C" + } ], + "paths" : { + "/" : { + "post" : { + "responses" : { + "200": { + "description": "Control Broker response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ControlBrokerRequestStatus" + } + } + } + }, + }, + "security" : [ { + "ControlBrokerClientAuthorizer" : [ ] + } ], + "x-amazon-apigateway-integration" : { + "payloadFormatVersion" : "2.0", + "type" : "aws_proxy", + "httpMethod" : "POST", + "uri" : "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:446960196218:function:ControlBrokerEndpointStackV-InvokedByApigwFD5B66D0-Kmorz6WfyJpc/invocations", + "connectionType" : "INTERNET" + } + } + } + }, + "components" : { + "schemas" : { + "ControlBrokerConsumerInputs" : { + "properties":{ + "InputType": { + "type":"string" + }, + "Bucket": { + "type":"string" + }, + "InputKeys": { + "type":"array" + }, + "ConsumerMetadata": { + "type":"object" + } + } + }, + "ControlBrokerRequestStatus": { + "properties":{ + "RequestorIsAuthorized": { + "type":"boolean" + }, + "EvalEngineHasReadAccessToinputs": { + "type":"boolean" + }, + "ResultsReportS3Uri": { + "type":"string" + }, + "EvalEngineSfnExecutionArn": { + "type":"string" + } + } + } + }, + "securitySchemes" : { + "ControlBrokerClientAuthorizer" : { + "in" : "header", + "name" : "Authorization", + "type" : "apiKey", + "x-amazon-apigateway-authorizer" : { + "identitySource" : "$request.header.Authorization", + "authorizerUri" : "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:446960196218:function:ControlBrokerEndpointStac-ControlBrokerClientAutho-aNRC35BFIV3W/invocations", + "authorizerPayloadFormatVersion" : "2.0", + "authorizerResultTtlInSeconds" : 0, + "type" : "request", + "enableSimpleResponses" : true + } + } + } + }, + "x-amazon-apigateway-importexport-version" : "1.0" +} \ No newline at end of file diff --git a/docs/open_api/openapitools.json b/docs/open_api/openapitools.json new file mode 100644 index 00000000..9cbc6d5d --- /dev/null +++ b/docs/open_api/openapitools.json @@ -0,0 +1,7 @@ +{ + "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json", + "spaces": 2, + "generator-cli": { + "version": "5.4.0" + } +} diff --git a/docs/open_api/stage-definition.yaml b/docs/open_api/stage-definition.yaml new file mode 100644 index 00000000..6d3a9ca6 --- /dev/null +++ b/docs/open_api/stage-definition.yaml @@ -0,0 +1,44 @@ +openapi: "3.0.1" +info: + title: "ControlBrokerEndpoint" + version: "2022-05-05 12:19:57UTC" +servers: +- url: "https://v8i28ze8df.execute-api.us-east-1.amazonaws.com/{basePath}" + variables: + basePath: + default: "" +tags: +- name: "aws:cloudformation:stack-id" + x-amazon-apigateway-tag-value: "arn:aws:cloudformation:us-east-1:446960196218:stack/ControlBrokerEndpointStackV0x7x0/0cfbb700-cc6c-11ec-bc72-0e5c5ed49289" +- name: "aws:cloudformation:stack-name" + x-amazon-apigateway-tag-value: "ControlBrokerEndpointStackV0x7x0" +- name: "aws:cloudformation:logical-id" + x-amazon-apigateway-tag-value: "ControlBrokerEndpoint7427912C" +paths: + /: + post: + responses: + default: + description: "Default response for POST /" + security: + - ControlBrokerClientAuthorizer: [] + x-amazon-apigateway-integration: + payloadFormatVersion: "2.0" + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:446960196218:function:ControlBrokerEndpointStackV-InvokedByApigwFD5B66D0-Kmorz6WfyJpc/invocations" + connectionType: "INTERNET" +components: + securitySchemes: + ControlBrokerClientAuthorizer: + type: "apiKey" + name: "Authorization" + in: "header" + x-amazon-apigateway-authorizer: + identitySource: "$request.header.Authorization" + authorizerUri: "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:446960196218:function:ControlBrokerEndpointStac-ControlBrokerClientAutho-aNRC35BFIV3W/invocations" + authorizerPayloadFormatVersion: "2.0" + authorizerResultTtlInSeconds: 0 + type: "request" + enableSimpleResponses: true +x-amazon-apigateway-importexport-version: "1.0" diff --git a/requirements.txt b/requirements.txt index 286e998a..5ac4ba36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,8 @@ constructs==10.0.119 decorator==5.1.1 exceptiongroup==1.0.0rc3 executing==0.8.3 +gitdb==4.0.9 +GitPython==3.1.27 iniconfig==1.1.1 ipdb==0.13.9 ipython==8.2.0 @@ -44,11 +46,12 @@ python-dateutil==2.8.2 pytz-deprecation-shim==0.1.0.post0 s3transfer==0.5.2 six==1.16.0 +smmap==5.0.0 stack-data==0.2.0 toml==0.10.2 tomli==2.0.1 traitlets==5.1.1 -typing_extensions==4.2.0 +typing-extensions==4.2.0 tzdata==2022.1 tzlocal==4.2 urllib3==1.26.9 diff --git a/stacks/client_stack.py b/stacks/client_stack.py deleted file mode 100644 index 6b5467ac..00000000 --- a/stacks/client_stack.py +++ /dev/null @@ -1,395 +0,0 @@ -import os -import json -from typing import List, Sequence -from os import path - -from aws_cdk import ( - Duration, - Stack, - RemovalPolicy, - CfnOutput, - aws_config, - aws_dynamodb, - aws_s3, - aws_sqs, - aws_s3_deployment, - aws_lambda, - aws_stepfunctions, - aws_iam, - aws_logs, - aws_events, - aws_apigatewayv2, - aws_apigatewayv2_alpha, # experimental as of 4.25.22 - aws_apigatewayv2_integrations_alpha, # experimental as of 4.25.22 - aws_apigatewayv2_authorizers_alpha, # experimental as of 4.25.22 - aws_lambda_python_alpha, # experimental as of 4.25.22 -) -from constructs import Construct - - -class ClientStack(Stack): - """Client Layer""" - - def __init__( - self, - *args, - control_broker_outer_state_machine: aws_stepfunctions.StateMachine, - control_broker_roles: List[aws_iam.Role], - control_broker_eval_results_bucket: aws_s3.Bucket, - **kwargs, - ): - """Create a ClientStack. - - :param control_broker_outer_state_machine: The outer state machine to call when invoking the control broker during tests. - :type control_broker_outer_state_machine: aws_stepfunctions.StateMachine - :param control_broker_principals: The principals to which we need to give S3 access for our input bucket. - :type control_broker_principals: List[aws_iam.IPrincipal] - :param control_broker_eval_results_bucket: The bucket owned by ControlBroker to host Evaluation ResultsReports. - :type control_broker_eval_results_bucket: aws_s3.Bcuket - """ - super().__init__(*args, **kwargs) - - self.control_broker_outer_state_machine = control_broker_outer_state_machine - self.control_broker_eval_results_bucket = control_broker_eval_results_bucket - - self.apigw() - # self.consumer_client_retry() - - def apigw(self): - - # auth - lambda - - lambda_authorizer = aws_lambda.Function( - self, - "ControlBrokerClientAuthorizer", - runtime=aws_lambda.Runtime.PYTHON_3_9, - handler="lambda_function.lambda_handler", - timeout=Duration.seconds(60), - memory_size=1024, - code=aws_lambda.Code.from_asset( - "./supplementary_files/lambdas/apigw_authorizer" - ), - ) - - authorizer_lambda = aws_apigatewayv2_authorizers_alpha.HttpLambdaAuthorizer( - "ControlBrokerClientAuthorizer", - lambda_authorizer, - response_types=[aws_apigatewayv2_authorizers_alpha.HttpLambdaResponseType.SIMPLE], - results_cache_ttl = Duration.seconds(0), - identity_source = [ - "$request.header.Authorization", # Authorization must be present in headers or 401, e.g. r = requests.post(url,auth = auth, ...) - ] - ) - - # auth - iam - - authorizer_iam = aws_apigatewayv2_authorizers_alpha.HttpIamAuthorizer() - - # integration - - lambda_invoked_by_apigw = aws_lambda.Function( - self, - "InvokedByApigw", - runtime=aws_lambda.Runtime.PYTHON_3_9, - handler="lambda_function.lambda_handler", - timeout=Duration.seconds(60), - memory_size=1024, - code=aws_lambda.Code.from_asset( - "./supplementary_files/lambdas/invoked_by_apigw" - ), - environment = { - "ControlBrokerOuterSfnArn" : self.control_broker_outer_state_machine.state_machine_arn, - "ControlBrokerEvalResultsReportsBucket": self.control_broker_eval_results_bucket.bucket_name - } - ) - - lambda_invoked_by_apigw.role.add_to_policy( - aws_iam.PolicyStatement( - actions=[ - "states:StartExecution", - ], - resources=[ - self.control_broker_outer_state_machine.state_machine_arn - ], - ) - ) - - integration = aws_apigatewayv2_integrations_alpha.HttpLambdaIntegration( - "ControlBrokerClient", - lambda_invoked_by_apigw - ) - - # api - - self.http_api = aws_apigatewayv2_alpha.HttpApi( - self, - "ControlBrokerClient", - # default_authorizer = authorizer - ) - - self.path = "/" - - routes = self.http_api.add_routes( - path=self.path, - methods=[ - aws_apigatewayv2_alpha.HttpMethod.POST - ], - integration=integration, - authorizer=authorizer_lambda - # authorizer=authorizer_iam - ) - - self.apigw_full_invoke_url = path.join(self.http_api.url.rstrip("/"),self.path.strip('/')) - - CfnOutput(self, "ApigwInvokeUrl", value=self.apigw_full_invoke_url) - - def consumer_client_retry(self): - - # object exists - - self.lambda_object_exists = aws_lambda.Function( - self, - "ObjectExists", - runtime=aws_lambda.Runtime.PYTHON_3_9, - handler="lambda_function.lambda_handler", - timeout=Duration.seconds(60), - memory_size=1024, # todo power-tune - code=aws_lambda.Code.from_asset( - "./supplementary_files/lambdas/s3_head_object" - ), - ) - - self.lambda_object_exists.role.add_to_policy( - aws_iam.PolicyStatement( - actions=[ - "s3:HeadObject", - "s3:GetObject", - "s3:List*", - ], - resources=[ - self.control_broker_eval_results_bucket.bucket_arn, - f"{self.control_broker_eval_results_bucket.bucket_arn}*" - ], - ) - ) - - # sign apigw request - - self.lambda_sign_apigw_request = aws_lambda_python_alpha.PythonFunction( - self, - "SignApigwRequest", - entry="./supplementary_files/lambdas/sign_apigw_request", - runtime= aws_lambda.Runtime.PYTHON_3_9, - index="lambda_function.py", - handler="lambda_handler", - timeout=Duration.seconds(60), - memory_size=1024, - environment = { - "ApigwInvokeUrl" : self.apigw_full_invoke_url - }, - layers=[ - aws_lambda_python_alpha.PythonLayerVersion( - self, - "aws_requests_auth", - entry="./supplementary_files/lambda_layers/aws_requests_auth", - compatible_runtimes=[ - aws_lambda.Runtime.PYTHON_3_9 - ] - ), - aws_lambda_python_alpha.PythonLayerVersion(self, - "requests", - entry="./supplementary_files/lambda_layers/requests", - compatible_runtimes=[ - aws_lambda.Runtime.PYTHON_3_9 - ] - ), - ] - ) - - # s3 select - - self.lambda_s3_select = aws_lambda.Function( - self, - "S3Select", - runtime=aws_lambda.Runtime.PYTHON_3_9, - handler="lambda_function.lambda_handler", - timeout=Duration.seconds(60), - memory_size=1024, - code=aws_lambda.Code.from_asset("./supplementary_files/lambdas/s3_select"), - ) - - self.lambda_s3_select.role.add_to_policy( - aws_iam.PolicyStatement( - actions=[ - "s3:HeadObject", - "s3:GetObject", - "s3:List*", - "s3:SelectObjectContent", - ], - resources=[ - self.control_broker_eval_results_bucket.bucket_arn, - f"{self.control_broker_eval_results_bucket.bucket_arn}/*", - ], - ) - ) - - # sfn - - log_group_consumer_client_sfn = aws_logs.LogGroup( - self, - "ConsumerClientLogs", - log_group_name=f"/aws/vendedlogs/states/ConsumerClientLogs-{self.stack_name}", - removal_policy=RemovalPolicy.DESTROY, - ) - - self.role_consumer_client_sfn = aws_iam.Role( - self, - "ConsumerClientSfn", - assumed_by=aws_iam.ServicePrincipal("states.amazonaws.com"), - ) - - self.role_consumer_client_sfn.add_to_policy( - aws_iam.PolicyStatement( - actions=[ - # "logs:*", - "logs:CreateLogDelivery", - "logs:GetLogDelivery", - "logs:UpdateLogDelivery", - "logs:DeleteLogDelivery", - "logs:ListLogDeliveries", - "logs:PutResourcePolicy", - "logs:DescribeResourcePolicies", - "logs:DescribeLogGroups", - ], - resources=[ - "*", - log_group_consumer_client_sfn.log_group_arn, - f"{log_group_consumer_client_sfn.log_group_arn}*", - ], - ) - ) - - self.role_consumer_client_sfn.add_to_policy( - aws_iam.PolicyStatement( - actions=[ - "lambda:InvokeFunction", - ], - resources=[ - self.lambda_sign_apigw_request.function_arn, - self.lambda_object_exists.function_arn, - self.lambda_s3_select.function_arn - ], - ) - ) - - self.sfn_consumer_client = aws_stepfunctions.CfnStateMachine( - self, - "ConsumerClient", - # state_machine_type="EXPRESS", - state_machine_type="STANDARD", - role_arn=self.role_consumer_client_sfn.role_arn, - logging_configuration=aws_stepfunctions.CfnStateMachine.LoggingConfigurationProperty( - destinations=[ - aws_stepfunctions.CfnStateMachine.LogDestinationProperty( - cloud_watch_logs_log_group=aws_stepfunctions.CfnStateMachine.CloudWatchLogsLogGroupProperty( - log_group_arn=log_group_consumer_client_sfn.log_group_arn - ) - ) - ], - # include_execution_data=False, - # level="ERROR", - include_execution_data=True, - level="ALL" - ), - definition_string=json.dumps( - { - "StartAt": "SignApigwRequest", - "States": { - "SignApigwRequest": { - "Type": "Task", - "Next": "CheckResultsReportExists", - "ResultPath": "$.SignApigwRequest", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": self.lambda_sign_apigw_request.function_name, - "Payload.$": "$" - }, - "ResultSelector": { - "Payload.$": "$.Payload" - }, - }, - "CheckResultsReportExists": { - "Type": "Task", - "Next": "GetResultsReportIsCompliantBoolean", - "ResultPath": "$.CheckResultsReportExists", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": self.lambda_object_exists.function_name, - "Payload": { - "S3Uri.$":"$.SignApigwRequest.Payload.ControlBrokerRequestStatus.ResultsReportS3Uri" - } - }, - "ResultSelector": { - "Payload.$": "$.Payload" - }, - "Retry": [ - { - "ErrorEquals": [ - "ObjectDoesNotExistException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 6, - "BackoffRate": 2.0 - } - ], - "Catch": [ - { - "ErrorEquals":[ - "States.ALL" - ], - "Next": "ResultsReportDoesNotYetExist" - } - ] - }, - "ResultsReportDoesNotYetExist": { - "Type":"Fail" - }, - "GetResultsReportIsCompliantBoolean": { - "Type": "Task", - "Next": "ChoiceIsComplaint", - "ResultPath": "$.GetResultsReportIsCompliantBoolean", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": self.lambda_s3_select.function_name, - "Payload": { - "S3Uri.$":"$.SignApigwRequest.Payload.ControlBrokerRequestStatus.ResultsReportS3Uri", - "Expression": "SELECT * from S3Object s", - }, - }, - "ResultSelector": {"S3SelectResult.$": "$.Payload.Selected"}, - }, - "ChoiceIsComplaint": { - "Type":"Choice", - "Default":"CompliantFalse", - "Choices":[ - { - "Variable":"$.GetResultsReportIsCompliantBoolean.S3SelectResult.ControlBrokerResultsReport.Evaluation.IsCompliant", - "BooleanEquals":True, - "Next":"CompliantTrue" - } - ] - }, - "CompliantTrue": { - "Type":"Succeed" - }, - "CompliantFalse": { - "Type":"Fail" - } - } - } - ) - ) - - self.sfn_consumer_client.node.add_dependency(self.role_consumer_client_sfn) - - \ No newline at end of file diff --git a/stacks/control_broker_stack.py b/stacks/control_broker_stack.py index eee54fbb..1211959f 100644 --- a/stacks/control_broker_stack.py +++ b/stacks/control_broker_stack.py @@ -58,7 +58,8 @@ def __init__( self.deploy_outer_sfn() self.Input_reader_roles: List[aws_iam.Role] = [ - self.lambda_opa_eval_python_subprocess.role, + self.lambda_evaluate_cloudformation_by_opa.role, + self.lambda_pac_evaluation_router.role, ] self.outer_eval_engine_state_machine = ( @@ -124,6 +125,21 @@ def deploy_utils(self): ], ) + # converted inputs + + self.bucket_converted_inputs = aws_s3.Bucket( + self, + "ConvertedInputs", + removal_policy=RemovalPolicy.DESTROY, + auto_delete_objects=True, + block_public_access=aws_s3.BlockPublicAccess( + block_public_acls=True, + ignore_public_acls=True, + block_public_policy=True, + restrict_public_buckets=True, + ), + ) + # results reports self.bucket_eval_results_reports = aws_s3.Bucket( @@ -184,21 +200,361 @@ def s3_deploy_local_assets(self): def deploy_inner_sfn_lambdas(self): - # opa eval - python subprocess - single threaded + # pac evaluation router - self.lambda_opa_eval_python_subprocess = aws_lambda.Function( + self.lambda_pac_evaluation_router = aws_lambda.Function( self, - "OpaEvalPythonSubprocessSingleThreaded", + "PaCEvaluationRouter", runtime=aws_lambda.Runtime.PYTHON_3_9, handler="lambda_function.lambda_handler", timeout=Duration.seconds(60), memory_size=10240, # todo power-tune code=aws_lambda.Code.from_asset( - "./supplementary_files/lambdas/opa_eval/python_subprocess" + "./supplementary_files/lambdas/pac_evaluation_router" ), + environment={ + "PaCBucketRouting": json.dumps( + {"OPA": self.bucket_opa_policies.bucket_name} + ), + "ConvertedInputsBucket": self.bucket_converted_inputs.bucket_name, + }, ) - self.lambda_opa_eval_python_subprocess.role.add_to_policy( + self.lambda_pac_evaluation_router.role.add_to_policy( + aws_iam.PolicyStatement( + actions=[ + "cloudformation:ValidateTemplate", + "cloudformation:DescribeType", + "cloudformation:Get*", # FIXME + "cloudformation:Describe*", # FIXME + ], + resources=["*"], + ) + ) + self.lambda_pac_evaluation_router.role.add_to_policy( + aws_iam.PolicyStatement( + actions=[ + "cloudcontrol:GetResource", + "cloudcontrol:*", # FIXME + ], + resources=["*"], + ) + ) + self.lambda_pac_evaluation_router.role.add_to_policy( + aws_iam.PolicyStatement( + actions=[ + "s3:PutObject", + ], + resources=[ + self.bucket_converted_inputs.arn_for_objects("*"), + ], + ) + ) + self.lambda_pac_evaluation_router.role.add_to_policy( + aws_iam.PolicyStatement( + # Get*, List* for all services with a cloudcontrol provisionable resource + # required fro cloudcontrol.get_resource() + actions=[ + "acmpca:Get*", + "acmpca:List*", + "aps:Get*", + "aps:List*", + "accessanalyzer:Get*", + "accessanalyzer:List*", + "amplify:Get*", + "amplify:List*", + "amplifyuibuilder:Get*", + "amplifyuibuilder:List*", + "apigateway:Get*", + "apigateway:List*", + "appflow:Get*", + "appflow:List*", + "appintegrations:Get*", + "appintegrations:List*", + "apprunner:Get*", + "apprunner:List*", + "appstream:Get*", + "appstream:List*", + "appsync:Get*", + "appsync:List*", + "applicationinsights:Get*", + "applicationinsights:List*", + "athena:Get*", + "athena:List*", + "auditmanager:Get*", + "auditmanager:List*", + "autoscaling:Get*", + "autoscaling:List*", + "backup:Get*", + "backup:List*", + "batch:Get*", + "batch:List*", + "budgets:Get*", + "budgets:List*", + "ce:Get*", + "ce:List*", + "cur:Get*", + "cur:List*", + "cassandra:Get*", + "cassandra:List*", + "certificatemanager:Get*", + "certificatemanager:List*", + "chatbot:Get*", + "chatbot:List*", + "cloudformation:Get*", + "cloudformation:List*", + "cloudfront:Get*", + "cloudfront:List*", + "cloudtrail:Get*", + "cloudtrail:List*", + "cloudwatch:Get*", + "cloudwatch:List*", + "codeartifact:Get*", + "codeartifact:List*", + "codeguruprofiler:Get*", + "codeguruprofiler:List*", + "codegurureviewer:Get*", + "codegurureviewer:List*", + "codestarconnections:Get*", + "codestarconnections:List*", + "codestarnotifications:Get*", + "codestarnotifications:List*", + "config:Get*", + "config:List*", + "connect:Get*", + "connect:List*", + "customerprofiles:Get*", + "customerprofiles:List*", + "databrew:Get*", + "databrew:List*", + "datasync:Get*", + "datasync:List*", + "detective:Get*", + "detective:List*", + "devopsguru:Get*", + "devopsguru:List*", + "devicefarm:Get*", + "devicefarm:List*", + "dynamodb:Get*", + "dynamodb:List*", + "ec2:Get*", + "ec2:List*", + "ecr:Get*", + "ecr:List*", + "ecs:Get*", + "ecs:List*", + "efs:Get*", + "efs:List*", + "eks:Get*", + "eks:List*", + "emr:Get*", + "emr:List*", + "emrcontainers:Get*", + "emrcontainers:List*", + "elasticache:Get*", + "elasticache:List*", + "elasticloadbalancingv2:Get*", + "elasticloadbalancingv2:List*", + "eventschemas:Get*", + "eventschemas:List*", + "events:Get*", + "events:List*", + "evidently:Get*", + "evidently:List*", + "fis:Get*", + "fis:List*", + "fms:Get*", + "fms:List*", + "finspace:Get*", + "finspace:List*", + "forecast:Get*", + "forecast:List*", + "frauddetector:Get*", + "frauddetector:List*", + "gamelift:Get*", + "gamelift:List*", + "globalaccelerator:Get*", + "globalaccelerator:List*", + "glue:Get*", + "glue:List*", + "greengrassv2:Get*", + "greengrassv2:List*", + "groundstation:Get*", + "groundstation:List*", + "healthlake:Get*", + "healthlake:List*", + "iam:Get*", + "iam:List*", + "ivs:Get*", + "ivs:List*", + "imagebuilder:Get*", + "imagebuilder:List*", + "inspector:Get*", + "inspector:List*", + "inspectorv2:Get*", + "inspectorv2:List*", + "iot:Get*", + "iot:List*", + "iotanalytics:Get*", + "iotanalytics:List*", + "iotcoredeviceadvisor:Get*", + "iotcoredeviceadvisor:List*", + "iotevents:Get*", + "iotevents:List*", + "iotfleethub:Get*", + "iotfleethub:List*", + "iotsitewise:Get*", + "iotsitewise:List*", + "iotwireless:Get*", + "iotwireless:List*", + "kms:Get*", + "kms:List*", + "kafkaconnect:Get*", + "kafkaconnect:List*", + "kendra:Get*", + "kendra:List*", + "kinesis:Get*", + "kinesis:List*", + "kinesisfirehose:Get*", + "kinesisfirehose:List*", + "kinesisvideo:Get*", + "kinesisvideo:List*", + "lambda:Get*", + "lambda:List*", + "lex:Get*", + "lex:List*", + "licensemanager:Get*", + "licensemanager:List*", + "lightsail:Get*", + "lightsail:List*", + "location:Get*", + "location:List*", + "logs:Get*", + "logs:List*", + "lookoutequipment:Get*", + "lookoutequipment:List*", + "lookoutmetrics:Get*", + "lookoutmetrics:List*", + "lookoutvision:Get*", + "lookoutvision:List*", + "msk:Get*", + "msk:List*", + "mwaa:Get*", + "mwaa:List*", + "macie:Get*", + "macie:List*", + "mediaconnect:Get*", + "mediaconnect:List*", + "mediapackage:Get*", + "mediapackage:List*", + "memorydb:Get*", + "memorydb:List*", + "networkfirewall:Get*", + "networkfirewall:List*", + "networkmanager:Get*", + "networkmanager:List*", + "nimblestudio:Get*", + "nimblestudio:List*", + "opensearchservice:Get*", + "opensearchservice:List*", + "opsworkscm:Get*", + "opsworkscm:List*", + "panorama:Get*", + "panorama:List*", + "personalize:Get*", + "personalize:List*", + "pinpoint:Get*", + "pinpoint:List*", + "qldb:Get*", + "qldb:List*", + "quicksight:Get*", + "quicksight:List*", + "rds:Get*", + "rds:List*", + "rum:Get*", + "rum:List*", + "redshift:Get*", + "redshift:List*", + "refactorspaces:Get*", + "refactorspaces:List*", + "rekognition:Get*", + "rekognition:List*", + "resiliencehub:Get*", + "resiliencehub:List*", + "resourcegroups:Get*", + "resourcegroups:List*", + "robomaker:Get*", + "robomaker:List*", + "route53:Get*", + "route53:List*", + "route53recoverycontrol:Get*", + "route53recoverycontrol:List*", + "route53recoveryreadiness:Get*", + "route53recoveryreadiness:List*", + "route53resolver:Get*", + "route53resolver:List*", + "s3:Get*", + "s3:List*", + "s3objectlambda:Get*", + "s3objectlambda:List*", + "s3outposts:Get*", + "s3outposts:List*", + "ses:Get*", + "ses:List*", + "sqs:Get*", + "sqs:List*", + "ssm:Get*", + "ssm:List*", + "ssmcontacts:Get*", + "ssmcontacts:List*", + "ssmincidents:Get*", + "ssmincidents:List*", + "sso:Get*", + "sso:List*", + "sagemaker:Get*", + "sagemaker:List*", + "servicecatalog:Get*", + "servicecatalog:List*", + "servicecatalogappregistry:Get*", + "servicecatalogappregistry:List*", + "signer:Get*", + "signer:List*", + "stepfunctions:Get*", + "stepfunctions:List*", + "synthetics:Get*", + "synthetics:List*", + "timestream:Get*", + "timestream:List*", + "transfer:Get*", + "transfer:List*", + "wafv2:Get*", + "wafv2:List*", + "wisdom:Get*", + "wisdom:List*", + "workspaces:Get*", + "workspaces:List*", + "xray:Get*", + "xray:List*", + ], + resources=["*"], + ) + ) + + # InputType CloudFormation - PaCFramework OPA - PythonSubprocess + + self.lambda_evaluate_cloudformation_by_opa = aws_lambda.Function( + self, + "EvaluateCloudFormationTemplateByOPAPythonSubprocess", + runtime=aws_lambda.Runtime.PYTHON_3_9, + handler="lambda_function.lambda_handler", + timeout=Duration.seconds(60), + memory_size=10240, # todo power-tune + code=aws_lambda.Code.from_asset( + "./supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess" + ), + ) + + self.lambda_evaluate_cloudformation_by_opa.role.add_to_policy( aws_iam.PolicyStatement( actions=[ "s3:HeadObject", @@ -208,6 +564,8 @@ def deploy_inner_sfn_lambdas(self): resources=[ self.bucket_opa_policies.bucket_arn, self.bucket_opa_policies.arn_for_objects("*"), + self.bucket_converted_inputs.bucket_arn, + self.bucket_converted_inputs.arn_for_objects("*"), ], ) ) @@ -308,7 +666,8 @@ def deploy_inner_sfn(self): aws_iam.PolicyStatement( actions=["lambda:InvokeFunction"], resources=[ - self.lambda_opa_eval_python_subprocess.function_arn, + self.lambda_pac_evaluation_router.function_arn, + self.lambda_evaluate_cloudformation_by_opa.function_arn, self.lambda_gather_infractions.function_arn, self.lambda_handle_infraction.function_arn, ], @@ -358,37 +717,52 @@ def deploy_inner_sfn(self): ), definition_string=json.dumps( { - "StartAt": "ParseInput", + "StartAt": "PaCEvaluationRouter", "States": { - "ParseInput": { - "Type": "Pass", - "Next": "OpaEval", + "PaCEvaluationRouter": { + "Type": "Task", + "Next": "ChoicePaCEvaluationRouting", + "ResultPath": "$.PaCEvaluationRouter", + "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { - "JsonInput": { - "Bucket.$": "$.Input.Bucket", - "Key.$": "$.Input.Key", - }, - "OuterEvalEngineSfnExecutionId.$": "$.OuterEvalEngineSfn.ExecutionId", - "ConsumerMetadata.$":"$.ConsumerMetadata", + "FunctionName": self.lambda_pac_evaluation_router.function_name, + "Payload.$": "$", }, - "ResultPath": "$", + "ResultSelector": {"Routing.$": "$.Payload.Routing"}, + }, + "ChoicePaCEvaluationRouting": { + "Type": "Choice", + "Default": "NoValidRoute", + "Choices": [ + { + "Variable": "$.PaCEvaluationRouter.Routing.InvokingSfnNextState", + "StringEquals": "EvaluateCloudFormationTemplateByOPA", + "Next": "EvaluateCloudFormationTemplateByOPA", + } + ], + }, + "NoValidRoute": { + "Type": "Fail", }, - "OpaEval": { + "EvaluateCloudFormationTemplateByOPA": { "Type": "Task", "Next": "GatherInfractions", - "ResultPath": "$.OpaEval", + "ResultPath": "$.EvaluateCloudFormationTemplateByOPA", "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { - "FunctionName": self.lambda_opa_eval_python_subprocess.function_name, + "FunctionName": self.lambda_evaluate_cloudformation_by_opa.function_name, "Payload": { - "JsonInput.$": "$.JsonInput", - "OpaPolicies": { - "Bucket": self.bucket_opa_policies.bucket_name + "JsonInput": { + "Bucket.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Bucket", + "Key.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Key", + }, + "PaC": { + "Bucket.$": "$.PaCEvaluationRouter.Routing.PaC.Bucket" }, }, }, "ResultSelector": { - "OpaEvalResults.$": "$.Payload.OpaEvalResults" + "Results.$": "$.Payload.EvaluateCloudFormationTemplateByOPAResults" }, }, "GatherInfractions": { @@ -398,7 +772,7 @@ def deploy_inner_sfn(self): "Resource": "arn:aws:states:::lambda:invoke", "Parameters": { "FunctionName": self.lambda_gather_infractions.function_name, - "Payload.$": "$.OpaEval.OpaEvalResults", + "Payload.$": "$.EvaluateCloudFormationTemplateByOPA.Results", }, "ResultSelector": { "Infractions.$": "$.Payload.Infractions" @@ -425,9 +799,12 @@ def deploy_inner_sfn(self): "ItemsPath": "$.GatherInfractions.Infractions", "Parameters": { "Infraction.$": "$$.Map.Item.Value", - "JsonInput.$": "$.JsonInput", + "JsonInput": { + "Bucket.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Bucket", + "Key.$": "$.PaCEvaluationRouter.Routing.ModifiedInput.Key", + }, "OuterEvalEngineSfnExecutionId.$": "$.OuterEvalEngineSfnExecutionId", - "ConsumerMetadata.$": "$.ConsumerMetadata", + "ConsumerMetadata.$": "$.ControlBrokerConsumerInputs.ConsumerMetadata", }, "Iterator": { "StartAt": "HandleInfraction", @@ -441,10 +818,10 @@ def deploy_inner_sfn(self): "FunctionName": self.lambda_handle_infraction.function_name, "Payload": { "Infraction.$": "$.Infraction", - "JsonInput.$": "$.JsonInput", + "JsonInput.$":"$.JsonInput", "OuterEvalEngineSfnExecutionId.$": "$.OuterEvalEngineSfnExecutionId", "ConsumerMetadata.$": "$.ConsumerMetadata", - } + }, }, "ResultSelector": {"Payload.$": "$.Payload"}, }, @@ -607,11 +984,8 @@ def deploy_outer_sfn(self): "ResultPath": "$.ForEachInput", "ItemsPath": "$.InvokedByApigw.ControlBrokerConsumerInputs.InputKeys", "Parameters": { - "Input": { - "Bucket.$": "$.InvokedByApigw.ControlBrokerConsumerInputs.Bucket", - "Key.$": "$$.Map.Item.Value", - }, - "ConsumerMetadata.$": "$.InvokedByApigw.ControlBrokerConsumerInputs.ConsumerMetadata", + "InvokedByApigw.$": "$.InvokedByApigw", + "ControlBrokerConsumerInputKey.$": "$$.Map.Item.Value", }, "Iterator": { "StartAt": "InvokeInnerEvalEngineSfn", @@ -624,11 +998,9 @@ def deploy_outer_sfn(self): "Parameters": { "StateMachineArn": self.sfn_inner_eval_engine.attr_arn, "Input": { - "Input.$": "$.Input", - "OuterEvalEngineSfn": { - "ExecutionId.$": "$$.Execution.Id" - }, - "ConsumerMetadata.$": "$.ConsumerMetadata", + "ControlBrokerConsumerInputKey.$": "$.ControlBrokerConsumerInputKey", + "ControlBrokerConsumerInputs.$": "$.InvokedByApigw.ControlBrokerConsumerInputs", + "OuterEvalEngineSfnExecutionId.$": "$$.Execution.Id", }, }, }, @@ -657,9 +1029,9 @@ def deploy_outer_sfn(self): "FunctionName": self.lambda_write_results_report.function_name, "Payload": { "OuterEvalEngineSfnExecutionId.$": "$$.Execution.Id", - "ResultsReportS3Uri.$":"$.ResultsReportS3Uri", - "ForEachInput.$":"$.ForEachInput", - } + "ResultsReportS3Uri.$": "$.ResultsReportS3Uri", + "ForEachInput.$": "$.ForEachInput", + }, }, "ResultSelector": {"Payload.$": "$.Payload"}, }, diff --git a/stacks/endpoint_stack.py b/stacks/endpoint_stack.py new file mode 100644 index 00000000..e430b349 --- /dev/null +++ b/stacks/endpoint_stack.py @@ -0,0 +1,135 @@ +import os +import json +from typing import List, Sequence +from os import path + +from aws_cdk import ( + Duration, + Stack, + CfnOutput, + aws_s3, + aws_lambda, + aws_stepfunctions, + aws_iam, + aws_apigatewayv2_alpha, # experimental as of 4.25.22 + aws_apigatewayv2_integrations_alpha, # experimental as of 4.25.22 + aws_apigatewayv2_authorizers_alpha, # experimental as of 4.25.22 +) +from constructs import Construct + + +class EndpointStack(Stack): + def __init__( + self, + *args, + control_broker_outer_state_machine: aws_stepfunctions.StateMachine, + control_broker_roles: List[aws_iam.Role], + control_broker_eval_results_bucket: aws_s3.Bucket, + **kwargs, + ): + """Create a EndpointStack. + + :param control_broker_outer_state_machine: The outer state machine to call when invoking the control broker during tests. + :type control_broker_outer_state_machine: aws_stepfunctions.StateMachine + :param control_broker_principals: The principals to which we need to give S3 access for our input bucket. + :type control_broker_principals: List[aws_iam.IPrincipal] + :param control_broker_eval_results_bucket: The bucket owned by ControlBroker to host Evaluation ResultsReports. + :type control_broker_eval_results_bucket: aws_s3.Bcuket + """ + super().__init__(*args, **kwargs) + + self.control_broker_outer_state_machine = control_broker_outer_state_machine + self.control_broker_eval_results_bucket = control_broker_eval_results_bucket + + self.endpoint() + + def endpoint(self): + + # auth - lambda + + lambda_authorizer = aws_lambda.Function( + self, + "ControlBrokerClientAuthorizer", + runtime=aws_lambda.Runtime.PYTHON_3_9, + handler="lambda_function.lambda_handler", + timeout=Duration.seconds(60), + memory_size=1024, + code=aws_lambda.Code.from_asset( + "./supplementary_files/lambdas/apigw_authorizer" + ), + ) + + authorizer_lambda = aws_apigatewayv2_authorizers_alpha.HttpLambdaAuthorizer( + "ControlBrokerClientAuthorizer", + lambda_authorizer, + response_types=[ + aws_apigatewayv2_authorizers_alpha.HttpLambdaResponseType.SIMPLE + ], + results_cache_ttl=Duration.seconds(0), + identity_source=[ + "$request.header.Authorization", # Authorization must be present in headers or 401, e.g. r = requests.post(url,auth = auth, ...) + ], + ) + + # auth - iam + + # authorizer_iam = aws_apigatewayv2_authorizers_alpha.HttpIamAuthorizer() + + # integration + + lambda_invoked_by_apigw = aws_lambda.Function( + self, + "InvokedByApigw", + runtime=aws_lambda.Runtime.PYTHON_3_9, + handler="lambda_function.lambda_handler", + timeout=Duration.seconds(60), + memory_size=1024, + code=aws_lambda.Code.from_asset( + "./supplementary_files/lambdas/invoked_by_apigw" + ), + environment={ + "ControlBrokerOuterSfnArn": self.control_broker_outer_state_machine.state_machine_arn, + "ControlBrokerEvalResultsReportsBucket": self.control_broker_eval_results_bucket.bucket_name, + }, + ) + + lambda_invoked_by_apigw.role.add_to_policy( + aws_iam.PolicyStatement( + actions=[ + "states:StartExecution", + ], + resources=[self.control_broker_outer_state_machine.state_machine_arn], + ) + ) + + integration = aws_apigatewayv2_integrations_alpha.HttpLambdaIntegration( + "ControlBrokerClient", lambda_invoked_by_apigw + ) + + # api + + self.http_api = aws_apigatewayv2_alpha.HttpApi( + self, + "ControlBrokerEndpoint", + # default_authorizer = authorizer + ) + + self.path = "/" + + routes = self.http_api.add_routes( + path=self.path, + methods=[aws_apigatewayv2_alpha.HttpMethod.POST], + integration=integration, + authorizer=authorizer_lambda + # authorizer=authorizer_iam + ) + + self.apigw_full_invoke_url = path.join( + self.http_api.url.rstrip("/"), self.path.strip("/") + ) + + CfnOutput(self, "ApigwInvokeUrl", value=self.apigw_full_invoke_url) + + open_api_definition = f'aws apigatewayv2 export-api --api-id {self.http_api.http_api_id} --output-type YAML --specification OAS30 --stage-name $default stage-definition.yaml' + + CfnOutput(self, "CBEndpointOpenApiDefinition", value=open_api_definition) diff --git a/stacks/pipeline_stack.py b/stacks/pipeline_stack.py index 3ed245ba..3c00db81 100644 --- a/stacks/pipeline_stack.py +++ b/stacks/pipeline_stack.py @@ -1,3 +1,4 @@ +from typing import List import boto3 from aws_cdk import ( ArnFormat, @@ -8,26 +9,46 @@ aws_iam as iam, pipelines as pipelines, ) - +from constructs import Construct class GitHubCDKPipelineStack(Stack): - """ - If you want to use an existing CodeStar connection for the source stage, specify its arn with - codestar_connection_arn + """Create a CDK Pipelines CodePipeline using a GitHub repo via a CodeStar Connection. - additional_synth_iam_statements are added to the synth stage role""" + Optionally allows using a pre-existing CodeStar Connection via a SecretsManager Secret containing + the ARN of the CodeStar Connection. + """ def __init__( self, - scope, - id, - github_repo_name, - github_repo_owner, - github_repo_branch, - codestar_connection_arn_secret_id=None, - additional_synth_iam_statements=None, + scope: Construct, + id: str, + github_repo_name: str, + github_repo_owner: str, + github_repo_branch: str, + codestar_connection_arn_secret_id: str = None, + additional_synth_iam_statements: List[iam.PolicyStatement] = None, **kwargs, ): + """Initialize a CDK Pipeline stack that uses a GitHub repo for its source. + + :param scope: Scope of this stack. + :type scope: Construct + :param id: Unique identifier. + :type id: str + :param github_repo_name: Name of the repo to use for the pipeline's source stage. + :type github_repo_name: str + :param github_repo_owner: Owner of the repo for the pipeline's source stage. + :type github_repo_owner: str + :param github_repo_branch: Branch of the github repo, defaults to None + :type github_repo_branch: str, optional + :param codestar_connection_arn_secret_id: ID of a SecretsManager Secret + that contains the ARN to a CodeStar Connection to use to access the + GitHub repo. Useful if you already have a Connection you want to reuse. + :type codestar_connection_arn_secret_id: str, optional + :param additional_synth_iam_statements: Statements to add to the CDK deployment role to allow CDK to get the + secret at codestar_connection_arn_secret_id, if specified. Defaults to None + :type additional_synth_iam_statements: List[iam.PolicyStatement], optional + """ super().__init__(scope, id, **kwargs) # Create codestar connection to connect pipeline to git. @@ -83,6 +104,7 @@ def __init__( "pip install -r requirements.txt", # Instructs Codebuild to install required packages "npx cdk synth", ], + env={"PIPELINE_SYNTH": "true"} ) self.pipeline = pipelines.CodePipeline( @@ -90,5 +112,5 @@ def __init__( "Pipeline", synth=pipeline_synth_action, publish_assets_in_parallel=False, - docker_enabled_for_synth=True + docker_enabled_for_synth=True, ) diff --git a/stacks/test_stack.py b/stacks/test_stack.py index 91c48cb8..2696fd61 100644 --- a/stacks/test_stack.py +++ b/stacks/test_stack.py @@ -38,7 +38,10 @@ def __init__( removal_policy=RemovalPolicy.DESTROY, ) for principal in control_broker_roles: - canary_bucket.grant_read(aws_iam.ArnPrincipal(principal.role_arn), f"{CANARY_TEST_TEMPLATE_DEST}/*") + canary_bucket.grant_read( + aws_iam.ArnPrincipal(principal.role_arn), + f"{CANARY_TEST_TEMPLATE_DEST}/*", + ) control_broker_consumer_policy = aws_iam.ManagedPolicy( self, "ControlBrokerConsumerPolicy", @@ -134,8 +137,6 @@ def __init__( ), ) - - """ TODO.1 @@ -163,4 +164,4 @@ def __init__( assert that for each non-read-only property, the API response is the same - """ \ No newline at end of file + """ diff --git a/supplementary_files/lambdas/conversion/lambda_function.py b/supplementary_files/lambdas/conversion/lambda_function.py deleted file mode 100644 index b358f5c4..00000000 --- a/supplementary_files/lambdas/conversion/lambda_function.py +++ /dev/null @@ -1,9 +0,0 @@ -def convert_config_to_cfn(): - pass - -convert: { - "ConfigEvent":convert_config_to_cfn , - # "SAM": convert_sam_to_cfn#TODO - # "HelmChart": #TODO - # "Terraform": #TODO -} diff --git a/supplementary_files/lambdas/handle_infraction/lambda_function.py b/supplementary_files/lambdas/handle_infraction/lambda_function.py index a63169c2..e643e83c 100644 --- a/supplementary_files/lambdas/handle_infraction/lambda_function.py +++ b/supplementary_files/lambdas/handle_infraction/lambda_function.py @@ -8,9 +8,9 @@ eb = boto3.client('events') def update_item(*, - table, - pk, - sk, + table:str, + pk:str, + sk:str, attributes:dict[str,str] ): @@ -56,8 +56,8 @@ def ddb_compatible_type(Item): return True def put_event_entry(*, - event_bus_name, - source, + event_bus_name:str, + source:str, detail:dict ): try: @@ -67,7 +67,7 @@ def put_event_entry(*, 'EventBusName':event_bus_name, 'Detail':json.dumps(detail), 'DetailType':os.environ.get('AWS_LAMBDA_FUNCTION_NAME'), - 'source':source, + 'Source':source, } ] ) @@ -98,7 +98,7 @@ def lambda_handler(event, context): # to ddb update = update_item( - table = os.environ['tableName'], + table = os.environ['TableName'], pk = outer_eval_enginge_sfn_execution_id, sk = sk, attributes = consumer_metadata @@ -107,7 +107,7 @@ def lambda_handler(event, context): # to eb put = put_event_entry( - event_bus_name = os.environ.get('event_bus_name'), + event_bus_name = os.environ['EventBusName'], source = outer_eval_enginge_sfn_execution_id, detail = { 'Infraction':event.get('Infraction'), diff --git a/supplementary_files/lambdas/invoked_by_apigw/lambda_function.py b/supplementary_files/lambdas/invoked_by_apigw/lambda_function.py index 53b9badf..1a88670c 100644 --- a/supplementary_files/lambdas/invoked_by_apigw/lambda_function.py +++ b/supplementary_files/lambdas/invoked_by_apigw/lambda_function.py @@ -49,7 +49,7 @@ def lambda_handler(event,context): post_request_json_body = json.loads(event['body']) - eval_engine_sfn_arn = os.environ.get('ControlBrokerOutersfn_arn') + eval_engine_sfn_arn = os.environ.get('ControlBrokerOuterSfnArn') print(f'eval_engine_sfn_arn:\n{eval_engine_sfn_arn}') eval_results_reports_bucket = os.environ.get('ControlBrokerEvalResultsReportsBucket') diff --git a/supplementary_files/lambdas/opa_eval/python_subprocess/lambda_function.py b/supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess/lambda_function.py similarity index 93% rename from supplementary_files/lambdas/opa_eval/python_subprocess/lambda_function.py rename to supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess/lambda_function.py index 37afda59..d87c5700 100644 --- a/supplementary_files/lambdas/opa_eval/python_subprocess/lambda_function.py +++ b/supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess/lambda_function.py @@ -19,7 +19,7 @@ def s3_download(*,bucket,key,local_path): local_path ) except ClientError as e: - print(f'ClientError:\nbucket: {bucket}\nkey: {key}\n{e}') + print(f'ClientError:\nbucket: {bucket}\nkey:\n{key}\n{e}') raise else: print(f'No ClientError download_file\nbucket:\n{bucket}\nkey:\n{key}') @@ -36,7 +36,7 @@ def s3_download_dir(*,bucket, prefix=None, local_path): for result in pagination: if result.get('CommonPrefixes') is not None: - for subdir in result.get('Commonprefixes'): + for subdir in result.get('CommonPrefixes'): s3_download_dir( prefix = subdir.get('Prefix'), local_path = local_path, @@ -81,12 +81,11 @@ def mkdir(dir_): p.mkdir(parents=True,exist_ok=True) return str(p) - def lambda_handler(event, context): print(event) - opa_policies_bucket = event['OpaPolicies']['Bucket'] + opa_policies_bucket = event['PaC']['Bucket'] json_input = event['JsonInput'] @@ -134,5 +133,5 @@ def lambda_handler(event, context): print(f'opa_eval_results:\n{opa_eval_results}\n{type(opa_eval_results)}') return { - "OpaEvalResults": opa_eval_results + "EvaluateCloudFormationTemplateByOPAResults": opa_eval_results } \ No newline at end of file diff --git a/supplementary_files/lambdas/opa_eval/python_subprocess/opa b/supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess/opa similarity index 100% rename from supplementary_files/lambdas/opa_eval/python_subprocess/opa rename to supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess/opa diff --git a/supplementary_files/lambdas/opa_eval/python_subprocess/opa-eval.sh b/supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess/opa-eval.sh similarity index 100% rename from supplementary_files/lambdas/opa_eval/python_subprocess/opa-eval.sh rename to supplementary_files/lambdas/pac_evaluation/input_type_cloudformation/pac_framework_opa/python_subprocess/opa-eval.sh diff --git a/supplementary_files/lambdas/pac_evaluation_router/lambda_function.py b/supplementary_files/lambdas/pac_evaluation_router/lambda_function.py new file mode 100644 index 00000000..21415fdf --- /dev/null +++ b/supplementary_files/lambdas/pac_evaluation_router/lambda_function.py @@ -0,0 +1,296 @@ +import json +import os + +import json +import boto3 +from botocore.exceptions import ClientError + +s3 = boto3.client('s3') +cfn = boto3.client('cloudformation') +cloudcontrol = boto3.client('cloudcontrol') + +def get_object(*,bucket,key): + + try: + r = s3.get_object( + Bucket = bucket, + Key = key + ) + except ClientError as e: + print(f'ClientError:\nbucket:\n{bucket}\nkey:\n{key}\n{e}') + raise + else: + print(f'no ClientError get_object:\nbucket:\n{bucket}\nkey:\n{key}') + body = r['Body'] + content = json.loads(body.read().decode('utf-8')) + return content + +def put_object(*,bucket,key,object_:dict): + + print(f'begin put_object\nbucket:\n{bucket}\nkey:\n{key}') + + try: + r = s3.put_object( + Bucket = bucket, + Key = key, + Body = json.dumps(object_) + ) + except ClientError as e: + print(f'ClientError:\nbucket:\n{bucket}\nkey:\n{key}\n{e}') + raise + else: + print(f'no ClientError put_object:\nbucket:\n{bucket}\nkey:\n{key}') + return True + +class CloudControl(): + + def __init__(self,type_name,identifier): + self.type_name = type_name + self.identifier = identifier + + def get_resource_schema(self,*,resource_type): + try: + r = cfn.describe_type( + Type = 'RESOURCE', + TypeName = resource_type, + ) + except cfn.exceptions.TypeNotFoundException: + print(f'TypeNotFoundException: {resource_type}') + return None + except ClientError as e: + raise + else: + schema = json.loads(r['Schema']) + print(schema) + return schema + + def cloudcontrol_get(self,*,type_name,identifier): + try: + r = cloudcontrol.get_resource( + TypeName = type_name, + Identifier = identifier + ) + except ClientError as e: + print(f'ClientError\n{e}') + raise + else: + properties = json.loads(r['ResourceDescription']['Properties']) + print(f'cloudcontrol.get_resource properties\ntype_name:\n{type_name}\nidentifier:\n{identifier}\nProperties:\n{properties}') + return properties + + def get_cfn(self): + + cloudcontrol_properties = self.cloudcontrol_get(type_name=self.type_name,identifier=self.identifier) + + resource_schema = self.get_resource_schema(resource_type=self.type_name) + print(f'resource_schema:\n{resource_schema}') + + schema_properties = resource_schema['properties'] + + read_only_properties = [i.split('/properties/')[1] for i in resource_schema['readOnlyProperties']] + for read_only_property in read_only_properties: + cloudcontrol_properties.pop(read_only_property,None) + + cfn = { + "Resources" : { + "ConfigEventResource" : { + "Type" : self.type_name, + "Properties" : cloudcontrol_properties, + } + } + } + print(f'cfn:\n{cfn}') + return cfn + + +class ConfigEventToCloudFormationConverter(): + + def __init__( + self, + event:dict + ): + + self.event = event + + self.config_event_s3_path = { + "Bucket":self.event['ControlBrokerConsumerInputs']['Bucket'], + "Key":self.event['ControlBrokerConsumerInputKey'] + } + + self.config_event = get_object( + bucket = self.config_event_s3_path['Bucket'], + key = self.config_event_s3_path['Key'] + ) + + def parse_config_event(self): + + print(f'config_event:\n{self.config_event}') + + invoking_event = json.loads(self.config_event["invokingEvent"]) + print(f'invoking_event:\n{invoking_event}') + + rule_parameters = self.config_event.get("ruleParameters") + if rule_parameters: + rule_parameters = json.loads(rule_parameters) + print(f'rule_parameters:\n{rule_parameters}') + + configuration_item = invoking_event["configurationItem"] + print(f'configuration_item:\n{configuration_item}') + + item_status = configuration_item["configurationItemStatus"] + print(f'item_status:\n{item_status}') + + self.resource_type = configuration_item['resourceType'] + print(f'resource_type:\n{self.resource_type}') + + resource_configuration = configuration_item['configuration'] + print(f'resource_configuration:\n{resource_configuration}') + + if resource_configuration: + resource_configuration_keys = list(resource_configuration.keys()) + print(f'resource_configuration_keys:\n{resource_configuration_keys}') + + self.resource_id = configuration_item['resourceId'] + print(f'resource_id:\n{self.resource_id}') + + def get_converted_cloudformation(self): + + c = CloudControl(type_name=self.resource_type,identifier=self.resource_id) + + self.cfn = c.get_cfn() + + return cfn + + def put_converted_cloudformation(self): + + self.converted_s3_path = { + 'Bucket' : os.environ['ConvertedInputsBucket'], + 'Key' : self.config_event_s3_path['Key'], + } + + put_object( + bucket = self.converted_s3_path['Bucket'], + key = self.converted_s3_path['Key'], + object_ = self.cfn + ) + + def get_converted_s3_path(self): + + self.parse_config_event() + self.get_converted_cloudformation() + self.put_converted_cloudformation() + + return self.converted_s3_path + +class PacEvaluationRouter(): + def __init__( + self, + event:dict, + ): + self.event = event + + + def convert_config_event_to_cfn(self,original_consumer_input_s3_path): + + c = ConfigEventToCloudFormationConverter(original_consumer_input_s3_path) + + modified_input_s3_path = c.get_converted_s3_path() + + return modified_input_s3_path + + def get_pac_bucket(self,*,pac_framework): + + pac_bucket_routing = json.loads(os.environ['PaCBucketRouting']) + + pac_bucket = pac_bucket_routing[pac_framework] + + return { + "Bucket": pac_bucket + } + + def get_modified_input_s3_path(self,*,input_conversion_object): + + original_consumer_input_s3_path = { + "Bucket":self.event['ControlBrokerConsumerInputs']['Bucket'], + "Key":self.event['ControlBrokerConsumerInputKey'] + } + + if not input_conversion_object: + + # pass-through original input - no modification needed + + return original_consumer_input_s3_path + + else: + + modified_input_s3_path = input_conversion_object(self.event) + + return modified_input_s3_path + + def get_invoking_sfn_next_state(self,*,input_type,pac_framework): + + return f'Evaluate{input_type}By{pac_framework}' + + def get_routing_decision(self): + + control_broker_consumer_inputs = self.event['ControlBrokerConsumerInputs'] + + control_broker_consumer_input_key = self.event['ControlBrokerConsumerInputKey'] + + input_type = control_broker_consumer_inputs['InputType'] + + if input_type == 'CloudFormationTemplate': + + pac_framework = "OPA" + + routing_decision = { + "PaC": self.get_pac_bucket( + pac_framework = pac_framework + ), + "ModifiedInput": self.get_modified_input_s3_path( + input_conversion_object = None + ), + "InvokingSfnNextState": self.get_invoking_sfn_next_state( + input_type = input_type, + pac_framework = pac_framework, + ) + } + + if input_type == 'ConfigEvent': + + pac_framework = "OPA" + + routing_decision = { + "PaC": self.get_pac_bucket( + pac_framework = pac_framework + ), + "ModifiedInput": self.get_modified_input_s3_path( + input_conversion_object = self.convert_config_event_to_cfn + ), + "InvokingSfnNextState": self.get_invoking_sfn_next_state( + input_type = "CloudFormationTemplate", # ConvertedTo + pac_framework = pac_framework, + ) + } + + + return routing_decision + + +def lambda_handler(event, context): + + print(event) + + p = PacEvaluationRouter( + event = event, + ) + + routing_decision = p.get_routing_decision() + + routing = { + "Routing": routing_decision, + } + + print(f"routing:\n{routing}") + + return routing \ No newline at end of file diff --git a/supplementary_files/lambdas/valid_cfn_canary/python/lambda_function.py b/supplementary_files/lambdas/valid_cfn_canary/python/lambda_function.py index 968f28c3..d2e52662 100644 --- a/supplementary_files/lambdas/valid_cfn_canary/python/lambda_function.py +++ b/supplementary_files/lambdas/valid_cfn_canary/python/lambda_function.py @@ -30,7 +30,9 @@ def lambda_handler(event=None, context=None): ) control_broker_input_object = json.dumps( { - "CFN": { + "CFN": { + # FIXME: see this issue for latest on API contract, required input schema + # https://github.com/VerticalRelevance/control-broker/issues/4 "Bucket": control_broker_readable_input_bucket, "Keys": [test_file_s3_key], } diff --git a/supplementary_files/lambdas/write_results_report/lambda_function.py b/supplementary_files/lambdas/write_results_report/lambda_function.py index 707f94eb..a0bed30e 100644 --- a/supplementary_files/lambdas/write_results_report/lambda_function.py +++ b/supplementary_files/lambdas/write_results_report/lambda_function.py @@ -27,7 +27,7 @@ def s3_uri_to_bucket_key(*,s3_uri): print(f'ClientError:\n{e}') raise else: - print(f'no ClientError s3.put_object()\s3_uri:\n{s3_uri}') + print(f'no ClientError s3.put_object()\ns3_uri:\n{s3_uri}') return True def simple_pk_query(*, diff --git a/utils/environment.py b/utils/environment.py new file mode 100644 index 00000000..5a12038e --- /dev/null +++ b/utils/environment.py @@ -0,0 +1,4 @@ +import os + +def is_pipeline_synth(): + return "PIPELINE_SYNTH" in os.environ