Add dummy backend

.vscode/settings.json

@@ -9,6 +9,9 @@
   "go.lintTool": "golangci-lint",
   "go.lintFlags": ["--fast"],
   "go.buildTags": "test",
+  "gopls": {
+    "buildFlags": ["-tags=test"]
+  },
   "eslint.workingDirectories": ["./web"],
   "prettier.ignorePath": "./web/.prettierignore",
   // Enable the ESlint flat config support

cli/exec/exec.go

@@ -32,6 +32,7 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/docker"
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/dummy"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local"
 	backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
@@ -228,6 +229,7 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error
 		kubernetes.New(),
 		docker.New(),
 		local.New(),
+		dummy.New(),
 	}
 	backendEngine, err := backend.FindBackend(backendCtx, backends, c.String("backend-engine"))
 	if err != nil {

cmd/agent/main.go

@@ -17,6 +17,7 @@ package main
 import (
 	"go.woodpecker-ci.org/woodpecker/v2/cmd/agent/core"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/docker"
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/dummy"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local"
 	backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
@@ -27,5 +28,6 @@ func main() {
 		kubernetes.New(),
 		docker.New(),
 		local.New(),
+		dummy.New(),
 	})
 }

docs/docs/92-development/09-testing.md

@@ -0,0 +1,81 @@
+# Testing
+
+## Backend
+
+### Unit Tests
+
+[We use default golang unit tests](https://go.dev/doc/tutorial/add-a-test)
+with [`"github.com/stretchr/testify/assert"`](https://pkg.go.dev/github.com/stretchr/testify@v1.9.0/assert) to simplify testing.
+
+### Integration Tests
+
+### Dummy backend
+
+There is a special backend called **`dummy`** which does not execute any commands, but emulates how a typical backend should behave.
+To enable it you need to build the agent or cli with the `test` build tag.
+
+An example pipeline config would be:
+
+```yaml
+when:
+  event: manual
+
+steps:
+  - name: echo
+    image: dummy
+    commands: echo "hello woodpecker"
+    environment:
+      SLEEP: '1s'
+
+services:
+  echo:
+    image: dummy
+    commands: echo "i am a sevice"
+```
+
+This could be executed via `woodpecker-cli --log-level trace exec --backend-engine dummy example.yaml`:
+
+```none
+9:18PM DBG pipeline/pipeline.go:94 > executing 2 stages, in order of: CLI=exec
+9:18PM DBG pipeline/pipeline.go:104 > stage CLI=exec StagePos=0 Steps=echo
+9:18PM DBG pipeline/pipeline.go:104 > stage CLI=exec StagePos=1 Steps=echo
+9:18PM TRC pipeline/backend/dummy/dummy.go:75 > create workflow environment taskUUID=01J10P578JQE6E25VV1EQF0745
+9:18PM DBG pipeline/pipeline.go:176 > prepare CLI=exec step=echo
+9:18PM DBG pipeline/pipeline.go:203 > executing CLI=exec step=echo
+9:18PM TRC pipeline/backend/dummy/dummy.go:81 > start step echo taskUUID=01J10P578JQE6E25VV1EQF0745
+9:18PM TRC pipeline/backend/dummy/dummy.go:167 > tail logs of step echo taskUUID=01J10P578JQE6E25VV1EQF0745
+9:18PM DBG pipeline/pipeline.go:209 > complete CLI=exec step=echo
+[echo:L0:0s] StepName: echo
+[echo:L1:0s] StepType: service
+[echo:L2:0s] StepUUID: 01J10P578JQE6E25VV1A2DNQN9
+[echo:L3:0s] StepCommands:
+[echo:L4:0s] ------------------
+[echo:L5:0s] echo ja
+[echo:L6:0s] ------------------
+[echo:L7:0s] 9:18PM DBG pipeline/pipeline.go:176 > prepare CLI=exec step=echo
+9:18PM DBG pipeline/pipeline.go:203 > executing CLI=exec step=echo
+9:18PM TRC pipeline/backend/dummy/dummy.go:81 > start step echo taskUUID=01J10P578JQE6E25VV1EQF0745
+9:18PM TRC pipeline/backend/dummy/dummy.go:167 > tail logs of step echo taskUUID=01J10P578JQE6E25VV1EQF0745
+[echo:L0:0s] StepName: echo
+[echo:L1:0s] StepType: commands
+[echo:L2:0s] StepUUID: 01J10P578JQE6E25VV1DFSXX1Y
+[echo:L3:0s] StepCommands:
+[echo:L4:0s] ------------------
+[echo:L5:0s] echo ja
+[echo:L6:0s] ------------------
+[echo:L7:0s] 9:18PM TRC pipeline/backend/dummy/dummy.go:108 > wait for step echo taskUUID=01J10P578JQE6E25VV1EQF0745
+9:18PM TRC pipeline/backend/dummy/dummy.go:187 > stop step echo taskUUID=01J10P578JQE6E25VV1EQF0745
+9:18PM DBG pipeline/pipeline.go:209 > complete CLI=exec step=echo
+9:18PM TRC pipeline/backend/dummy/dummy.go:208 > delete workflow environment taskUUID=01J10P578JQE6E25VV1EQF0745
+```
+
+There are also environment variables to alter step behaviour:
+
+- `SLEEP: 10` will let the step wait 10 seconds
+- `EXPECT_TYPE` allows to check if a step is a `clone`, `service`, `plugin` or `commands`
+- `STEP_START_FAIL: true` if set will simulate a step to fail before actually started (e.g. happens when the container image can not be pulled)
+- `STEP_TAIL_FAIL: true` if set will error when we simulate to read from stdout for logs
+- `STEP_EXIT_CODE: 2` if set will be used as exit code, default is 0
+- `STEP_OOM_KILLED: true` simulate a step who got killed because of memory constrains
+
+Also for the whole workflow if you set it's UUID ot `WorkflowSetupShouldFail`, you can fail a workflow setup fail.

pipeline/backend/dummy/dummy.go

@@ -0,0 +1,240 @@
+// Copyright 2024 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build test
+// +build test
+
+package dummy
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+
+	backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
+)
+
+type dummy struct {
+	kv sync.Map
+}
+
+const (
+	// Step names to control behavior of dummy backend.
+	WorkflowSetupFailUUID = "WorkflowSetupShouldFail"
+	EnvKeyStepSleep       = "SLEEP"
+	EnvKeyStepType        = "EXPECT_TYPE"
+	EnvKeyStepStartFail   = "STEP_START_FAIL"
+	EnvKeyStepExitCode    = "STEP_EXIT_CODE"
+	EnvKeyStepTailFail    = "STEP_TAIL_FAIL"
+	EnvKeyStepOOMKilled   = "STEP_OOM_KILLED"
+
+	// Internal const.
+	stepStateStarted   = "started"
+	stepStateDone      = "done"
+	testServiceTimeout = 1 * time.Second
+)
+
+// New returns a dummy backend.
+func New() backend.Backend {
+	return &dummy{
+		kv: sync.Map{},
+	}
+}
+
+func (e *dummy) Name() string {
+	return "dummy"
+}
+
+func (e *dummy) IsAvailable(_ context.Context) bool {
+	return true
+}
+
+func (e *dummy) Flags() []cli.Flag {
+	return nil
+}
+
+// Load new client for Docker Backend using environment variables.
+func (e *dummy) Load(_ context.Context) (*backend.BackendInfo, error) {
+	return &backend.BackendInfo{
+		Platform: "dummy",
+	}, nil
+}
+
+func (e *dummy) SetupWorkflow(_ context.Context, _ *backend.Config, taskUUID string) error {
+	if taskUUID == WorkflowSetupFailUUID {
+		return fmt.Errorf("expected fail to setup workflow")
+	}
+	log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment")
+	e.kv.Store("task_"+taskUUID, "setup")
+	return nil
+}
+
+func (e *dummy) StartStep(_ context.Context, step *backend.Step, taskUUID string) error {
+	log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
+
+	// internal state checks
+	_, exist := e.kv.Load("task_" + taskUUID)
+	if !exist {
+		return fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID)
+	}
+	stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID))
+	if stepExist {
+		// Detect issues like https://github.com/woodpecker-ci/woodpecker/issues/3494
+		return fmt.Errorf("StartStep detected already started step '%s' (%s) in state: %s", step.Name, step.UUID, stepState)
+	}
+
+	if stepStartFail, _ := strconv.ParseBool(step.Environment[EnvKeyStepStartFail]); stepStartFail {
+		return fmt.Errorf("expected fail to start step")
+	}
+
+	expectStepType, testStepType := step.Environment[EnvKeyStepType]
+	if testStepType && string(step.Type) != expectStepType {
+		return fmt.Errorf("expected step type '%s' but got '%s'", expectStepType, step.Type)
+	}
+
+	e.kv.Store(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID), stepStateStarted)
+	return nil
+}
+
+func (e *dummy) WaitStep(ctx context.Context, step *backend.Step, taskUUID string) (*backend.State, error) {
+	log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name)
+
+	_, exist := e.kv.Load("task_" + taskUUID)
+	if !exist {
+		err := fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID)
+		return &backend.State{Error: err}, err
+	}
+
+	// check state
+	stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID))
+	if !stepExist {
+		err := fmt.Errorf("WaitStep expect step '%s' (%s) to be created but found none", step.Name, step.UUID)
+		return &backend.State{Error: err}, err
+	}
+	if stepState != stepStateStarted {
+		err := fmt.Errorf("WaitStep expect step '%s' (%s) to be '%s' but it is: %s", step.Name, step.UUID, stepStateStarted, stepState)
+		return &backend.State{Error: err}, err
+	}
+
+	// extend wait time logic
+	if sleep, sleepExist := step.Environment[EnvKeyStepSleep]; sleepExist {
+		toSleep, err := time.ParseDuration(sleep)
+		if err != nil {
+			err = fmt.Errorf("WaitStep fail to parse sleep duration: %w", err)
+			return &backend.State{Error: err}, err
+		}
+		time.Sleep(toSleep)
+	} else {
+		if step.Type == backend.StepTypeService {
+			select {
+			case <-time.NewTimer(testServiceTimeout).C:
+				err := fmt.Errorf("WaitStep fail due to timeout of service after 1 second")
+				return &backend.State{Error: err}, err
+			case <-ctx.Done():
+				// context for service closed ... we can move forward
+			}
+		} else {
+			time.Sleep(time.Nanosecond)
+		}
+	}
+
+	e.kv.Store(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID), stepStateDone)
+
+	oomKilled, _ := strconv.ParseBool(step.Environment[EnvKeyStepOOMKilled])
+	exitCode := 0
+
+	if code, exist := step.Environment[EnvKeyStepExitCode]; exist {
+		exitCode, _ = strconv.Atoi(strings.TrimSpace(code))
+	}
+
+	return &backend.State{
+		ExitCode:  exitCode,
+		Exited:    true,
+		OOMKilled: oomKilled,
+	}, nil
+}
+
+func (e *dummy) TailStep(_ context.Context, step *backend.Step, taskUUID string) (io.ReadCloser, error) {
+	log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name)
+
+	_, exist := e.kv.Load("task_" + taskUUID)
+	if !exist {
+		return nil, fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID)
+	}
+
+	// check state
+	stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID))
+	if !stepExist {
+		return nil, fmt.Errorf("WaitStep expect step '%s' (%s) to be created but found none", step.Name, step.UUID)
+	}
+	if stepState != stepStateStarted {
+		return nil, fmt.Errorf("WaitStep expect step '%s' (%s) to be '%s' but it is: %s", step.Name, step.UUID, stepStateStarted, stepState)
+	}
+
+	if tailShouldFail, _ := strconv.ParseBool(step.Environment[EnvKeyStepTailFail]); tailShouldFail {
+		return nil, fmt.Errorf("expected fail to read stdout of step")
+	}
+
+	return io.NopCloser(strings.NewReader(dummyExecStepOutput(step))), nil
+}
+
+func (e *dummy) DestroyStep(_ context.Context, step *backend.Step, taskUUID string) error {
+	log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name)
+
+	_, exist := e.kv.Load("task_" + taskUUID)
+	if !exist {
+		return fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID)
+	}
+
+	// check state
+	stepState, stepExist := e.kv.Load(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID))
+	if !stepExist {
+		return fmt.Errorf("WaitStep expect step '%s' (%s) to be created but found none", step.Name, step.UUID)
+	}
+	if stepState != stepStateDone {
+		return fmt.Errorf("WaitStep expect step '%s' (%s) to be '%s' but it is: %s", step.Name, step.UUID, stepStateDone, stepState)
+	}
+
+	e.kv.Delete(fmt.Sprintf("task_%s_step_%s", taskUUID, step.UUID))
+	return nil
+}
+
+func (e *dummy) DestroyWorkflow(_ context.Context, _ *backend.Config, taskUUID string) error {
+	log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment")
+
+	_, exist := e.kv.Load("task_" + taskUUID)
+	if !exist {
+		return fmt.Errorf("expect env of workflow %s to exist but found none to destroy", taskUUID)
+	}
+	e.kv.Delete("task_" + taskUUID)
+	return nil
+}
+
+func dummyExecStepOutput(step *backend.Step) string {
+	return fmt.Sprintf(`StepName: %s
+StepType: %s
+StepUUID: %s
+StepCommands:
+------------------
+%s
+------------------
+`, step.Name, step.Type, step.UUID, strings.Join(step.Commands, "\n"))
+}