-
Notifications
You must be signed in to change notification settings - Fork 230
/
cloudbuild_master.yaml
230 lines (207 loc) · 7.62 KB
/
cloudbuild_master.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
#############################################################################
## The top section of this file is identical in the 3 cloudbuild.*yaml files.
## Make sure any edits you make here are copied over to the other files too
## if appropriate.
##
## TODO(al): consider if it's possible to merge these 3 files and control via
## substitutions.
#############################################################################
timeout: 1200s
options:
machineType: N1_HIGHCPU_32
volumes:
- name: go-modules
path: /go
env:
- GOPROXY=https://proxy.golang.org
- PROJECT_ROOT=github.com/google/certificate-transparency-go
- GOPATH=/go
substitutions:
_CLUSTER_NAME: trillian-opensource-ci
_MASTER_ZONE: us-central1-a
# Cloud Build logs sent to GCS bucket
logsBucket: 'gs://trillian-cloudbuild-logs'
steps:
# First build a "ct_testbase" docker image which contains most of the tools we need for the later steps:
- name: 'gcr.io/cloud-builders/docker'
entrypoint: 'bash'
args: ['-c', 'docker pull gcr.io/$PROJECT_ID/ct_testbase:latest || exit 0']
- name: 'gcr.io/cloud-builders/docker'
args: [
'build',
'-t', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
'--cache-from', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
'-f', './integration/Dockerfile',
'.'
]
# prepare spins up an ephemeral trillian instance for testing use.
- name: gcr.io/$PROJECT_ID/ct_testbase
entrypoint: 'bash'
id: 'prepare'
args:
- '-exc'
- |
# Use latest versions of Trillian docker images built by the Trillian CI cloudbuilders.
docker pull gcr.io/$PROJECT_ID/log_server:latest
docker tag gcr.io/$PROJECT_ID/log_server:latest deployment_trillian-log-server
docker pull gcr.io/$PROJECT_ID/log_signer:latest
docker tag gcr.io/$PROJECT_ID/log_signer:latest deployment_trillian-log-signer
# Bring up an ephemeral trillian instance using the docker-compose config in the Trillian repo:
export TRILLIAN_LOCATION="$$(go list -f '{{.Dir}}' github.com/google/trillian)"
# We need to fix up Trillian's docker-compose to connect to the CloudBuild network to that tests can use it:
echo -e "networks:\n default:\n external:\n name: cloudbuild" >> $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml
docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml pull mysql trillian-log-server trillian-log-signer
docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml up -d mysql trillian-log-server trillian-log-signer
# Install proto related bits and block on Trillian being ready
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'ci-ready'
entrypoint: 'bash'
args:
- '-ec'
- |
go install \
github.com/golang/protobuf/proto \
github.com/golang/protobuf/protoc-gen-go \
github.com/golang/mock/mockgen \
go.etcd.io/etcd/v3 go.etcd.io/etcd/etcdctl/v3 \
github.com/fullstorydev/grpcurl/cmd/grpcurl
# Generate all protoc and mockgen files
go generate -run="protoc" ./...
go generate -run="mockgen" ./...
# Cache all the modules we'll need too
go mod download
go test ./...
# Wait for trillian logserver to be up
until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done
# Reset the CT test database
export CT_GO_PATH="$$(go list -f '{{.Dir}}' github.com/google/certificate-transparency-go)"
export MYSQL_HOST="mysql"
export MYSQL_ROOT_PASSWORD="zaphod"
export MYSQL_USER_HOST="%"
yes | bash "$${CT_GO_PATH}/scripts/resetctdb.sh" --verbose
waitFor: ['prepare']
# Run the presubmit tests
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'default_test'
env:
- 'GOFLAGS='
- 'PRESUBMIT_OPTS=--no-linters --no-generate'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'race_detection'
env:
- 'GOFLAGS=-race'
- 'PRESUBMIT_OPTS=--no-linters --no-generate'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'etcd_with_coverage'
env:
- 'GOFLAGS='
- 'PRESUBMIT_OPTS=--no-linters --no-generate --coverage'
- 'WITH_ETCD=true'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'etcd_with_race'
env:
- 'GOFLAGS=-race'
- 'PRESUBMIT_OPTS=--no-linters --no-generate'
- 'WITH_ETCD=true'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'with_pkcs11_and_race'
env:
- 'GOFLAGS=-race --tags=pkcs11'
- 'PRESUBMIT_OPTS=--no-linters --no-generate'
- 'WITH_PKCS11=true'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
# Collect and submit codecoverage reports
- name: 'gcr.io/cloud-builders/curl'
id: 'codecov.io'
entrypoint: bash
args: ['-c', 'bash <(curl -s https://codecov.io/bash)']
env:
- 'VCS_COMMIT_ID=$COMMIT_SHA'
- 'VCS_BRANCH_NAME=$BRANCH_NAME'
- 'VCS_PULL_REQUEST=$_PR_NUMBER'
- 'CI_BUILD_ID=$BUILD_ID'
- 'CODECOV_TOKEN=$_CODECOV_TOKEN' # _CODECOV_TOKEN is specified in the cloud build trigger
waitFor: ['etcd_with_coverage']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'ci_complete'
entrypoint: /bin/true
waitFor: ['codecov.io', 'default_test', 'race_detection', 'etcd_with_coverage', 'etcd_with_race', 'with_pkcs11_and_race']
############################################################################
## End of replicated section.
## Below are deployment specific steps for the CD env.
############################################################################
- id: build_ctfe
name: gcr.io/cloud-builders/docker
args:
- build
- --file=trillian/examples/deployment/docker/ctfe/Dockerfile
- --tag=gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
- --cache-from=gcr.io/${PROJECT_ID}/ctfe
- .
waitFor: ["-"]
- id: push_ctfe
name: gcr.io/cloud-builders/docker
args:
- push
- gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
waitFor:
- build_ctfe
- id: tag_latest_ctfe
name: gcr.io/cloud-builders/gcloud
args:
- container
- images
- add-tag
- gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
- gcr.io/${PROJECT_ID}/ctfe:latest
waitFor:
- push_ctfe
- id: build_envsubst
name: gcr.io/cloud-builders/docker
args:
- build
- trillian/examples/deployment/docker/envsubst
- -t
- envsubst
waitFor: ["-"]
- id: envsubst_kubernetes_configs
name: envsubst
args:
- trillian/examples/deployment/kubernetes/ctfe-deployment.yaml
- trillian/examples/deployment/kubernetes/ctfe-service.yaml
- trillian/examples/deployment/kubernetes/ctfe-ingress.yaml
env:
- PROJECT_ID=${PROJECT_ID}
- IMAGE_TAG=${COMMIT_SHA}
waitFor:
- build_envsubst
- id: update_kubernetes_configs
name: gcr.io/cloud-builders/kubectl
args:
- apply
- -f=trillian/examples/deployment/kubernetes/ctfe-deployment.yaml
- -f=trillian/examples/deployment/kubernetes/ctfe-service.yaml
- -f=trillian/examples/deployment/kubernetes/ctfe-ingress.yaml
env:
- CLOUDSDK_COMPUTE_ZONE=${_MASTER_ZONE}
- CLOUDSDK_CONTAINER_CLUSTER=${_CLUSTER_NAME}
waitFor:
- envsubst_kubernetes_configs
- push_ctfe
images:
- gcr.io/${PROJECT_ID}/ctfe:${COMMIT_SHA}
- gcr.io/${PROJECT_ID}/ct_testbase:latest