Merge pull request #21 from aufi/add-test-stages

Introduce test tiers

.github/workflows/main.yml → .github/workflows/main-tier0.yml

@@ -1,4 +1,4 @@
-name: Test execution
+name: Test TIER0
 
 on:
   pull_request:
@@ -11,22 +11,17 @@ on:
       - '**.md'
 
 jobs:
-  tests:
+  test-tier0:
     runs-on: ubuntu-latest
-    #runs-on: self-hosted
     steps:
       - uses: actions/checkout@v3
       - name: start-minikube
         uses: konveyor/tackle2-operator/.github/actions/start-minikube@main
         with:
           memory: 6500M
-          #container-runtime: containerd
-          #driver: podman
-          #cpus: 4
-          #memory: 10000M
       - name: install-tackle
         uses: konveyor/tackle2-operator/.github/actions/install-tackle@main
       - uses: actions/setup-go@v3
         with:
           go-version: '1.19'
-      - run: make test-analysis
+      - run: make test-tier0

.github/workflows/main-tier1.yml

@@ -0,0 +1,27 @@
+name: Test TIER1
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**.md'
+  push:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**.md'
+
+jobs:
+  test-tier1:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: start-minikube
+        uses: konveyor/tackle2-operator/.github/actions/start-minikube@main
+        with:
+          memory: 6500M
+      - name: install-tackle
+        uses: konveyor/tackle2-operator/.github/actions/install-tackle@main
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.19'
+      - run: make test-tier1

.github/workflows/main-tier2.yml

@@ -0,0 +1,27 @@
+name: Test TIER2
+
+on:
+  pull_request:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**.md'
+  push:
+    branches: [ "main" ]
+    paths-ignore:
+      - '**.md'
+
+jobs:
+  test-tier2:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: start-minikube
+        uses: konveyor/tackle2-operator/.github/actions/start-minikube@main
+        with:
+          memory: 6500M
+      - name: install-tackle
+        uses: konveyor/tackle2-operator/.github/actions/install-tackle@main
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.19'
+      - run: make test-tier2

.github/workflows/nightly-tier0.yml

@@ -0,0 +1,21 @@
+name: Test nightly TIER0
+
+on:
+  schedule:
+    - cron: '14 1,13 * * *'
+
+jobs:
+  test-tier0:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: start-minikube
+        uses: konveyor/tackle2-operator/.github/actions/start-minikube@main
+        with:
+          memory: 6500M
+      - name: install-tackle
+        uses: konveyor/tackle2-operator/.github/actions/install-tackle@main
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.19'
+      - run: make test-tier0

.github/workflows/nightly-tier1.yml

@@ -0,0 +1,21 @@
+name: Test nightly TIER1
+
+on:
+  schedule:
+    - cron: '14 1,13 * * *'
+
+jobs:
+  test-tier1:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: start-minikube
+        uses: konveyor/tackle2-operator/.github/actions/start-minikube@main
+        with:
+          memory: 6500M
+      - name: install-tackle
+        uses: konveyor/tackle2-operator/.github/actions/install-tackle@main
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.19'
+      - run: make test-tier1

.github/workflows/nightly-tier2.yml

@@ -0,0 +1,21 @@
+name: Test nightly TIER2
+
+on:
+  schedule:
+    - cron: '14 1,13 * * *'
+
+jobs:
+  test-tier2:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: start-minikube
+        uses: konveyor/tackle2-operator/.github/actions/start-minikube@main
+        with:
+          memory: 6500M
+      - name: install-tackle
+        uses: konveyor/tackle2-operator/.github/actions/install-tackle@main
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.19'
+      - run: make test-tier2

Makefile

@@ -17,13 +17,40 @@ setup:
 clean:
 	minikube delete || true
 
-# Execute application analysis tests.
+# Update Hub dependency with latest binding and api.
+update-hub:
+	go get -u github.com/konveyor/tackle2-hub@main
+
+#
+# Test tiers.
+#
+
+# TIER0 - a core functionality, should never fail, Konveyor would be fully broken.
+test-tier0:
+	$(MAKE) test-analysis
+
+# TIER1 - all normal features expected to work.
+test-tier1:
+	$(MAKE) test-metrics
+	TIER1=1 $(MAKE) test-analysis
+
+# TIER2 - advanced features and nice-to-haves.
+test-tier2:
+	TIER2=1 $(MAKE) test-analysis
+
+#
+# Feature tests.
+#
+
+# Application analysis tests.
 test-analysis:
 	HUB_BASE_URL="http://${MINIKUBE_IP}/hub" go test -count=1 -timeout 7200s -v ./analysis/...
 
-# Execute all tests.
-test-all: test-analysis
+# Metrics.
+test-metrics:
+	HUB_BASE_URL="http://${MINIKUBE_IP}/hub" ginkgo -v ./e2e/metrics/...
 
-# Update Hub dependency with latest binding and api.
-update-hub:
-	go get -u github.com/konveyor/tackle2-hub@main
+# Add next features tests here and call the target from appropriate tier.
+
+# Execute all tests.
+test-all: test-tier0 test-tier1 test-tier2

README.md

@@ -1,10 +1,14 @@
 # Golang API test suite for Konveyor
 
-[![API tests on Quay](https://quay.io/repository/konveyor/go-konveyor-tests/status "API tests Repository on Quay")](https://quay.io/repository/konveyor/go-konveyor-tests)
-[![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/konveyor/go-konveyor-tests/pulls)
+[![License](https://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/konveyor/go-konveyor-tests/pulls)
 
-[![API CI Test Suite](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main.yml/badge.svg?branch=main)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main.yml)
-[![End-To-End Test Suite](https://github.com/konveyor/go-konveyor-tests/actions/workflows/test-nightly.yml/badge.svg?branch=main)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/test-nightly.yml)
+[![Test TIER0](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main-tier0.yml/badge.svg)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main-tier0.yml)
+[![Test TIER1](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main-tier1.yml/badge.svg)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main-tier1.yml)
+[![Test TIER2](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main-tier2.yml/badge.svg)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/main-tier2.yml)
+
+[![Test nightly TIER0](https://github.com/konveyor/go-konveyor-tests/actions/workflows/nightly-tier0.yml/badge.svg)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/nightly-tier0.yml)
+[![Test nightly TIER1](https://github.com/konveyor/go-konveyor-tests/actions/workflows/nightly-tier1.yml/badge.svg)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/nightly-tier1.yml)
+[![Test nightly TIER2](https://github.com/konveyor/go-konveyor-tests/actions/workflows/nightly-tier2.yml/badge.svg)](https://github.com/konveyor/go-konveyor-tests/actions/workflows/nightly-tier2.yml)
 
 This repository contains application-level tests for Konveyor. That means test focusing on integration of multiple components and real-world Koveyor use-cases. Basic components tests should be placed and executed in their own repositories.
 
@@ -47,7 +51,35 @@ $ export HUB_BASE_URL="http://`minikube ip`/hub"
 $ go test -count=1 -v ./analysis/
 ```
 
-### Test execution options
+## Test tiers
+
+To provide maximum information about the project functionality, tests were separated into three tiers. From core functionality to nice to haves.
+
+### Tier 0
+
+Very basic and core functionality. A bug here would lead to mostly useless project. This tier should never fail. Examples: Hub API or a basic application analysis flow.
+
+```
+$ make test-tier0
+```
+
+### Tier 1
+
+Features of the project expected to work to satifly most of end-users expectations. Examples: real-world use cases of application analysis, Jira integrations or metrics.
+
+```
+$ make test-tier1
+```
+
+### Tier 2
+
+More advanced features like complex application analysis or some edge cases. This tier should be kept green, but a failure should not affect most of users.
+
+```
+$ make test-tier2
+```
+
+## Test execution options
 
 #### PARALLEL
 

analysis/analysis_test.go

@@ -16,15 +16,15 @@ import (
 
 // Test application analysis
 func TestApplicationAnalysis(t *testing.T) {
-	// Find right test cases for given stage (include Stage0 always).
-	testCases := Stage0TestCases
-	_, stage1 := os.LookupEnv("STAGE1")
-	if stage1 {
-		testCases = append(testCases, Stage1TestCases...)
+	// Find right test cases for given Tier (include Tier0 always).
+	testCases := Tier0TestCases
+	_, tier1 := os.LookupEnv("TIER1")
+	if tier1 {
+		testCases = append(testCases, Tier1TestCases...)
 	}
-	_, stage2 := os.LookupEnv("STAGE2")
-	if stage2 {
-		testCases = append(testCases, Stage2TestCases...)
+	_, tier2 := os.LookupEnv("TIER2")
+	if tier2 {
+		testCases = append(testCases, Tier2TestCases...)
 	}
 	// Run test cases.
 	for _, testcase := range testCases {
@@ -110,7 +110,7 @@ func TestApplicationAnalysis(t *testing.T) {
 			}
 			for i, got := range gotAnalysis.Issues {
 				expected := tc.Analysis.Issues[i]
-				if got.Category != expected.Category || got.RuleSet != expected.RuleSet || got.Rule != expected.Rule || got.Effort != expected.Effort || !strings.HasPrefix(got.Description, expected.Description) {	
+				if got.Category != expected.Category || got.RuleSet != expected.RuleSet || got.Rule != expected.Rule || got.Effort != expected.Effort || !strings.HasPrefix(got.Description, expected.Description) {
 					t.Errorf("\nDifferent issue error. Got %+v, expected %+v.\n\n", got, expected)
 				}
 
@@ -124,7 +124,7 @@ func TestApplicationAnalysis(t *testing.T) {
 				}
 				for j, gotInc := range got.Incidents {
 					expectedInc := expected.Incidents[j]
-					if gotInc.File != expectedInc.File || gotInc.Line != expectedInc.Line || !strings.HasPrefix(gotInc.Message, expectedInc.Message) {	
+					if gotInc.File != expectedInc.File || gotInc.Line != expectedInc.Line || !strings.HasPrefix(gotInc.Message, expectedInc.Message) {
 						t.Errorf("\nDifferent incident error. Got %+v, expected %+v.\n\n", gotInc, expectedInc)
 					}
 				}

analysis/test_cases.go

@@ -1,28 +1,28 @@
 package analysis
 
 //
-// Stage 0 Analysis test cases - should never fail.
+// Tier 0 Analysis test cases - should never fail.
 // List of applications with expected analysis outputs.
-var Stage0TestCases = []TC{
+var Tier0TestCases = []TC{
 	Tomcat,
 	PathfinderSample,
 }
 
 //
-// Stage 1 Analysis test cases - should work.
+// Tier 1 Analysis test cases - should work.
 // List of applications with expected analysis outputs.
-var Stage1TestCases = []TC{
+var Tier1TestCases = []TC{
 	PetclinicMain,
 	PetclinicHazelcast,
-	Daytrader,
 	ApacheWicket,
 	SeamBooking,
 }
 
 //
-// Stage 2 Analysis test cases - great if works.
+// Tier 2 Analysis test cases - great if works.
 // List of applications with expected analysis outputs.
-var Stage2TestCases = []TC{
+var Tier2TestCases = []TC{
+	Daytrader,
 }
 
 //