-
Notifications
You must be signed in to change notification settings - Fork 11
/
workflows.yml
432 lines (415 loc) · 15.1 KB
/
workflows.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
# Full set of parameters, commands, jobs, and workflows used for CI/CD
version: 2.1
# CircleCI Orbs to use for jobs
# The aws-cli orb is needed for ECR login
orbs:
aws-cli: circleci/aws-cli@3.1.4
# The first 4 parameters to control monorepo conditionals
# Each Prefect project or supplemental folder (.aws) will get its own changes boolean
# This will come from path filtering setup logic in config.yml
parameters:
aws_changes:
type: boolean
default: false
common_utils_changes:
type: boolean
default: false
data_products_changes:
type: boolean
default: false
# This is a parameter to help with consistent working directories
base_working_directory:
type: string
default: /tmp/data-flows-v2/repo
# Below are anchors that are used as aliases further down the config
# https://circleci.com/docs/introduction-to-yaml-configurations/#anchors-and-aliases
# Branch shortcuts for filtering on workflows
only_main_v2: &only_main_v2
filters:
branches:
only:
- main-v2
not_dev_main_stg_v2: ¬_dev_main_stg_v2
filters:
branches:
ignore:
- staging-v2
- dev-v2
- main-v2
only_dev_v2: &only_dev_v2
filters:
branches:
only:
- dev-v2
only_stg_v2: &only_stg_v2
filters:
branches:
only:
- staging-v2
# Shortcuts for conditionals based on boolean parameters
has_aws_changes: &has_aws_changes
condition: << pipeline.parameters.aws_changes >>
has_common_utils_changes: &has_common_utils_changes
condition: << pipeline.parameters.common_utils_changes >>
# Reusable commands for jobs
commands:
# used to make sure git checkout happens in common base directory
git_checkout:
steps:
- checkout:
path: << pipeline.parameters.base_working_directory >>
# this is used as a final command on all jobs
# we need this because of Github branch rules to require builds to pass
no_op:
steps:
- run:
name: Build Completed
command: |
echo "All build steps expected have completed."
# common set of commands to run for python projects
# currently we run poetry install, black check, and pytest validation with expected 100% coverage
# https://github.com/psf/black
# we also run a a project version check via the df-cli to make sure pyproject.toml is version bumped as needed
python_changes_build:
parameters:
dependency_groups:
default: "main,dev"
type: string
coverage_path:
default: src
type: string
tests_path:
default: tests
type: string
steps:
- run:
command: |
poetry install --no-ansi --only <<parameters.dependency_groups>>
poetry run black --check .
if [[ ${CIRCLE_BRANCH} != "main-v2" ]]
then
poetry run df-cli check-version
fi
poetry run python -m pytest -vvvs --cov=<<parameters.coverage_path>> --cov-report term-missing --cov-fail-under=100 <<parameters.tests_path>>
name: Python Testing and Validation
# shortcut for the common_utils_build.sh script
common_utils_wheel_file:
steps:
- run:
command: |
../common-utils/common_utils_build.sh
# steps for using df-cli to validate flow configs and build steps
prefect_project_worker_build:
steps:
- run:
command: |
poetry run df-cli process-docker-envs --build-only
poetry run df-cli process-flow-specs
name: Prefect Project Build
# steps for using the df-cli to push dockers envs and flows
prefect_project_worker_deploy:
steps:
- run:
command: |
export GIT_SHA=$CIRCLE_SHA1 && poetry run df-cli process-docker-envs
poetry run df-cli process-flow-specs
export PYTHONPATH=$(pwd)/src && poetry run prefect --no-prompt deploy --all
name: Prefect Project Deploy
# Jobs to be used by workflows
# Use of conditionals allow required checks to pass when build steps are not needed
# Conditionals will the trigger the appropriate steps for the appropriate folders/files
jobs:
# this will:
# - check code style and syntax
# - synthesize the typescript to a terraform manifest
# - plan or apply based on parameters
aws_changes_build:
parameters:
node_env:
default: "development"
type: string
runner_resource_class:
default: pocket/default-dev
type: string
stack_name:
type: string
apply_stack:
default: false
type: boolean
# this is so that we can run the cdktf process in a self hosted runner
machine: true
resource_class: << parameters.runner_resource_class >>
environment:
CDKTF_LOG_LEVEL: DEBUG
NODE_ENV: << parameters.node_env >>
APPLY_STACK: << parameters.apply_stack >>
steps:
- when: # using the changes shortcut to trigger a conditional check
<<: *has_aws_changes
steps:
- checkout
- run:
command: |
. /home/circleci/.codebuild_shims_wrapper.sh
cd .aws
nvm install
npm install
[[ ${NODE_ENV} == "development" ]] && npm run prettier-check
[[ ${NODE_ENV} == "development" ]] && npm run lint
npm run synth
name: install packages, lint, and create tf template
- run:
command: |
cd .aws/cdktf.out/stacks/<< parameters.stack_name >>
if [ ${NODE_ENV} == "development" ]
then
export TF_WORKSPACE=Dev
else
export TF_WORKSPACE=Prod
fi
terraform init
if [[ ${APPLY_STACK} == "false" ]]
then
terraform plan
else
terraform apply -auto-approve
fi
name: terraform plan/apply
- no_op # always run no_op so that builds still passes when no actual build steps run
# Steps to run to validate health of the common_utils library
common_utils_changes_build:
docker:
- image: cimg/python:3.10
steps:
- when:
<<: *has_common_utils_changes
steps:
- git_checkout
- python_changes_build
- no_op
# notice the use of the base_working_directory parameter
working_directory: << pipeline.parameters.base_working_directory >>/common-utils
# Collection of steps for building and deploying a data-flows Prefect subproject
# Notice we have configs to control whether we need to perform the df-cli steps
prefect_project_changes:
parameters:
deployment_type:
default: dev
type: string
validate_build_only:
default: true
type: boolean
use_deploy_cli:
default: true
type: boolean
# the project_folder and has_changes parameters help us re-use this job for other projects
project_folder:
type: string
has_changes:
default: false
type: boolean
environment:
POCKET_PREFECT_DEPLOYMENT_TYPE: << parameters.deployment_type >>
DF_CONFIG_DEPLOYMENT_TYPE: << parameters.deployment_type >>
DF_CONFIG_GH_REPO: https://github.com/Pocket/data-flows.git
docker:
- image: cimg/python:3.10
steps:
- when:
condition: << parameters.has_changes >>
steps:
- git_checkout
- common_utils_wheel_file # run the common_utils_build.sh script on behalf of developer
- python_changes_build
- when: # here is conditional to control when and how we run the df-cli
condition: << parameters.use_deploy_cli >>
steps:
- setup_remote_docker:
version: docker24
- when:
condition: << parameters.validate_build_only >>
steps:
- prefect_project_worker_build
- unless:
condition: << parameters.validate_build_only >>
steps:
- aws-cli/setup:
role-arn: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/data-flows-ci-oidc-role"
role-session-name: prefect-flow-deploy
- prefect_project_worker_deploy
- no_op
working_directory: << pipeline.parameters.base_working_directory >>/<< parameters.project_folder >>
# In order for the common_utils to work properly with the prefect agent...
# we need to temporarily leverage a custom version of Prefect in custom image
# This is located at https://github.com/Pocket/prefect-aws.git@test-prs
prefect_agent_docker:
parameters:
push_image:
default: false
type: boolean
docker:
- image: cimg/aws:2023.04
steps:
- when:
<<: *has_aws_changes
steps:
- git_checkout
- setup_remote_docker:
version: docker24
- run:
command: |
docker build --platform linux/amd64 -t prefect-agent -f agent/Dockerfile .
- when:
condition: << parameters.push_image >>
steps:
- aws-cli/setup:
role-arn: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/data-flows-ci-oidc-role"
role-session-name: prefect-agent-docker-image
- run:
command: |
aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com
IMAGE_TAG=${CIRCLE_SHA1:0:7}
docker tag prefect-agent:latest ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/data-flows-prefect-v2-envs:prefect-agent-${IMAGE_TAG}
docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/data-flows-prefect-v2-envs:prefect-agent-${IMAGE_TAG}
- no_op
working_directory: << pipeline.parameters.base_working_directory >>/.aws/.docker
# Final CI/CD Workflows
# The branch shortcuts are used to group jobs into specific workflows
# New contexts were created to make sure that jobs have the proper envars
# PR workflows are for validating PRs
# Dev workflows are for deploying to AWS dev based environments, this means the dev account for AWS changes and flows deployed will run on AWS Dev inf prefect flows
# Staging workflows to allow for deploying flows to the production platform in 'pre-release' state - still need to define how pre-release will work though.
# Main workflows are for releasing to a production environment, this means the production account for AWS changes and flows deployed will run on AWS Production inf prefect flows
workflows:
pr_workflow:
jobs:
- aws_changes_build:
<<: *not_dev_main_stg_v2
context: pocket
node_env: development
runner_resource_class: pocket/default-dev
stack_name: prefect-oidc
name: terraform plan oidc dev
- aws_changes_build:
<<: *not_dev_main_stg_v2
context: pocket
node_env: production
runner_resource_class: pocket/default-prod
stack_name: prefect-oidc
name: terraform plan oidc production
- prefect_agent_docker:
<<: *not_dev_main_stg_v2
name: prefect agent docker build
- aws_changes_build:
<<: *not_dev_main_stg_v2
context: pocket
node_env: development
runner_resource_class: pocket/default-dev
stack_name: prefect-v2
name: terraform plan prefect dev
requires:
- terraform plan oidc dev
- aws_changes_build:
<<: *not_dev_main_stg_v2
context: pocket
node_env: production
runner_resource_class: pocket/default-prod
stack_name: prefect-v2
name: terraform plan prefect production
requires:
- terraform plan oidc production
- common_utils_changes_build:
<<: *not_dev_main_stg_v2
- prefect_project_changes:
<<: *not_dev_main_stg_v2
context:
- pocket-prefect-dev
- pocket-oidc-dev
has_changes: << pipeline.parameters.data_products_changes >>
project_folder: data-products
requires:
- common_utils_changes_build
dev_workflow:
jobs:
- aws_changes_build:
<<: *only_dev_v2
context: pocket
node_env: development
runner_resource_class: pocket/default-dev
stack_name: prefect-oidc
apply_stack: true
name: terraform apply oidc dev
- prefect_agent_docker:
<<: *only_dev_v2
context: pocket-oidc-dev
push_image: true
name: prefect agent docker push
requires:
- terraform apply oidc dev
- aws_changes_build:
<<: *only_dev_v2
context: pocket
node_env: development
runner_resource_class: pocket/default-dev
stack_name: prefect-v2
apply_stack: true
name: terraform apply prefect dev
requires:
- prefect agent docker push
main_workflow:
jobs:
- aws_changes_build:
<<: *only_main_v2
context: pocket
node_env: production
runner_resource_class: pocket/default-prod
stack_name: prefect-oidc
apply_stack: true
name: terraform apply oidc production
- prefect_agent_docker:
<<: *only_main_v2
context: pocket-oidc-prod
push_image: true
name: prefect agent docker push
requires:
- terraform apply oidc production
- aws_changes_build:
<<: *only_main_v2
context: pocket
node_env: production
runner_resource_class: pocket/default-prod
stack_name: prefect-v2
apply_stack: true
name: terraform apply prefect production
requires:
- prefect agent docker push
- common_utils_changes_build:
<<: *only_main_v2
- prefect_project_changes:
<<: *only_main_v2
context:
- pocket-prefect-prod
- pocket-oidc-prod
has_changes: << pipeline.parameters.data_products_changes >>
project_folder: data-products
deployment_type: main
validate_build_only: false
requires:
- terraform apply prefect production
- common_utils_changes_build
staging_workflow:
jobs:
- common_utils_changes_build:
<<: *only_stg_v2
- prefect_project_changes:
<<: *only_stg_v2
context:
- pocket-prefect-prod
- pocket-oidc-prod
has_changes: << pipeline.parameters.data_products_changes >>
project_folder: data-products
deployment_type: staging
validate_build_only: false
requires:
- common_utils_changes_build
# VS Code Extension Version: 1.5.1