ci: improve linting speed (elastic#22103)

* ci: improve linting speed * fix: use multiple agents for the matrix * fix: add python-env * fix: set make folder parameter * feat: move k8s check and dev-tools to their own stages * fix: lint in a single stage is faster * fix: suggested changes * chore:change triggers * chore: update for packetbeat # Conflicts: # Jenkinsfile # Makefile # auditbeat/Jenkinsfile.yml # filebeat/Jenkinsfile.yml # heartbeat/Jenkinsfile.yml # journalbeat/Jenkinsfile.yml # libbeat/Jenkinsfile.yml # metricbeat/Jenkinsfile.yml # packetbeat/Jenkinsfile.yml # winlogbeat/Jenkinsfile.yml # x-pack/auditbeat/Jenkinsfile.yml # x-pack/dockerlogbeat/Jenkinsfile.yml # x-pack/elastic-agent/Jenkinsfile.yml # x-pack/filebeat/Jenkinsfile.yml # x-pack/functionbeat/Jenkinsfile.yml # x-pack/libbeat/Jenkinsfile.yml # x-pack/packetbeat/Jenkinsfile.yml # x-pack/winlogbeat/Jenkinsfile.yml
kuisathaverat · Nov 3, 2020 · adba400 · adba400
1 parent 62f1115
commit adba400
Show file tree

Hide file tree

Showing 19 changed files with 110 additions and 9 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -68,10 +68,12 @@ pipeline {
         GOFLAGS = '-mod=readonly'
       }
       steps {
-        withGithubNotify(context: 'Lint') {
-          withBeatsEnv(archive: false, id: 'lint') {
+        withGithubNotify(context: "Lint") {
+          withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
-            cmd(label: 'make check', script: 'make check')
+            cmd(label: "make check-python", script: "make check-python")
+            cmd(label: "make check-go", script: "make check-go")
+            cmd(label: "Check for changes", script: "make check-no-changes")
           }
         }
       }

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
@@ -15,6 +15,7 @@ projects:
     - "x-pack/libbeat"
     - "x-pack/metricbeat"
     - "x-pack/winlogbeat"
+    - "dev-tools"
     ##- "x-pack/heartbeat"    It's not yet in the 1.0 pipeline.
     ##- "x-pack/journalbeat"  It's not yet in the 1.0 pipeline.
     ##- "x-pack/packetbeat"   It's not yet in the 1.0 pipeline.
@@ -41,4 +42,4 @@ disabled:
         labels:      ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR.
             - skip-ci
         ## TODO: This will allow to configure what to do based on the PR configuration
-        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.
+        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.
diff --git a/Makefile b/Makefile
@@ -81,17 +81,35 @@ clean-vendor:
 	@sh script/clean_vendor.sh
 
 .PHONY: check
-check: python-env
+check:
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
-	@# Checks also python files which are not part of the beats
-	@$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
-	@# Validate that all updates were committed
+	$(MAKE) check-python
+	# check if vendor folder does not exists
+	[ ! -d vendor ]
+	# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers
+	@$(MAKE) check-go
+	@$(MAKE) check-no-changes
+
+## ccheck-go : Check there is no changes in Go modules.
+.PHONY: check-go
+check-go:
+	@go mod tidy
+
+## ccheck-no-changes : Check there is no local changes.
+.PHONY: check-no-changes
+check-no-changes:
 	@git diff | cat
 	@git update-index --refresh
 	@git diff-index --exit-code HEAD --
 
+## check-python : Python Linting.
+.PHONY: check-python
+check-python: python-env
+	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+
+## check-headers : Check the license headers.
 .PHONY: check-headers
 check-headers:
 	@go get github.com/elastic/go-licenser

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C auditbeat check;
+          make -C auditbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C auditbeat check testsuite integration-tests-environment"
     crosscompile:

diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml
@@ -2,6 +2,7 @@ when:
     branches: true             ## for all the branches
     changeset:                 ## when PR contains any of those entries in the changeset
         - "^deploy/kubernetes/.*"
+        - "^libbeat/docs/version.asciidoc"
     comments:                  ## when PR comment contains any of those entries
         - "/test deploy/kubernetes"
     labels:                    ## when PR labels matches any of those entries
@@ -11,5 +12,7 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    lint:
+       make: "make -C deploy/kubernetes all"
     k8sTest:
         k8sTest: "v1.10.0,v1.9.4,v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10"
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C filebeat check;
+          make -C filebeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C filebeat check testsuite integration-tests-environment"
     windows:

diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C heartbeat check;
+          make -C heartbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C heartbeat check testsuite integration-tests-environment"
     windows:

diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml
@@ -13,9 +13,14 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C journalbeat check;
+          make -C journalbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C journalbeat check testsuite"
     windows:
         mage: "mage fields build goTestUnit"
         platforms:             ## override default labels in this specific stage.
-            - "windows-2008-r2"
+            - "windows-2008-r2"
diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml
@@ -12,6 +12,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C libbeat check;
+          make -C libbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C libbeat check testsuite integration-tests-environment"
         when:

diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C metricbeat check;
+          make -C metricbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C metricbeat check testsuite integration-tests-environment"
         when:

diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C packetbeat check;
+          make -C packetbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C packetbeat check testsuite"    ## See https://github.com/elastic/beats/issues/22266
     windows:

diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C winlogbeat check;
+          make -C winlogbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C winlogbeat check testsuite"
     crosscompile:

diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/auditbeat check;
+          make -C x-pack/auditbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C auditbeat check testsuite integration-tests-environment"
     windows:

diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/filebeat check;
+          make -C x-pack/filebeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C x-pack/filebeat check testsuite"
     windows:

diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/functionbeat check;
+          make -C x-pack/functionbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C x-pack/functionbeat check testsuite integration-tests-environment"
     windows:

diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/libbeat check;
+          make -C x-pack/libbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C x-pack/libbeat check testsuite integration-tests-environment"
     windows:

diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/metricbeat check;
+          make -C x-pack/metricbeat update;
+          make check-no-changes;
     build:
         make: "RACE_DETECTOR=1 make -C x-pack/metricbeat check testsuite integration-tests-environment"
         when:

diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml
@@ -13,6 +13,11 @@ when:
     tags: true                 ## for all the tags
 platform: "linux && ubuntu-16" ## default label for all the stages
 stages:
+    Lint:
+        mage: |
+          mage check;
+          mage update;
+        make: "make check-no-changes"
     build:
         make: "RACE_DETECTOR=1 make -C x-pack/packetbeat check testsuite integration-tests-environment"
     windows:

diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml
@@ -13,6 +13,13 @@ when:
     tags: true                 ## for all the tags
 platform: "windows-2019"       ## default label for all the stages
 stages:
+    Lint:
+        make: |
+          make -C x-pack/winlogbeat check;
+          make -C x-pack/winlogbeat update;
+          make check-no-changes;
+        platforms:             ## override default labels in this specific stage.
+            - "linux && ubuntu-18"
     windows:
         mage: "mage fields build goTestUnit"
         platforms:             ## override default labels in this specific stage.