diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index ffd3425fa..f02bf09bd 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -18,10 +18,10 @@ inputs: markdown: description: "The path where the Markdown report will be saved" required: false - subdomain-gateway-spec: - description: "Whether the gateway implements the subdomain gateway spec" + specs: + description: "A comma-separated list of specs to test" required: false - default: "true" + default: "" args: description: "The arguments to pass to the test command" required: false @@ -35,13 +35,13 @@ runs: env: URL: ${{ inputs.gateway-url }} JSON: ${{ inputs.json }} - SUBDOMAIN: ${{ inputs.subdomain-gateway-spec }} + SPECS: ${{ inputs.specs }} with: repository: ${{ steps.github.outputs.action_repository }} ref: ${{ steps.github.outputs.action_ref }} dockerfile: Dockerfile opts: --network=host - args: test --url="$URL" --json="$JSON" --subdomain="$SUBDOMAIN" -- ${{ inputs.args }} + args: test --url="$URL" --json="$JSON" --specs="$SPECS" -- ${{ inputs.args }} - name: Create the XML if: (inputs.xml || inputs.html || inputs.markdown) && (failure() || success()) uses: pl-strflt/gotest-json-to-junit-xml@v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1824af752..e26e87e8f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: xml: output.xml html: output.html markdown: output.md - subdomain-gateway-spec: false + specs: -subdomain-gateway args: -skip TestGatewayCar - name: Set summary if: (failure() || success()) diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index a3bbcf776..58c3a2b1b 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -58,7 +58,7 @@ func copyFiles(inputPaths []string, outputDirectoryPath string) error { func main() { var gatewayURL string var jsonOutput string - var subdomainGatewaySpec bool + var specs string var directory string var merged bool @@ -67,9 +67,9 @@ func main() { Usage: "Tooling for the gateway test suite", Commands: []*cli.Command{ { - Name: "test", + Name: "test", Aliases: []string{"t"}, - Usage: "Run the conformance test suite against your gateway", + Usage: "Run the conformance test suite against your gateway", Flags: []cli.Flag{ &cli.StringFlag{ Name: "gateway-url", @@ -85,23 +85,18 @@ func main() { Value: "", Destination: &jsonOutput, }, - &cli.BoolFlag{ - Name: "subdomain-gateway-spec", - Aliases: []string{"subdomain-gateway", "subdomain"}, - Usage: "Whether the gateway implements the subdomain gateway spec", - Value: true, - Destination: &subdomainGatewaySpec, + &cli.StringFlag{ + Name: "specs", + Usage: "A comma-separated list of specs to test", + Value: "", + Destination: &specs, }, }, Action: func(cCtx *cli.Context) error { args := []string{"test", "./tests", "-test.v=test2json"} - tags := []string{} - if !subdomainGatewaySpec { - tags = append(tags, "no_subdomain_gateway_spec") - } - if len(tags) > 0 { - args = append(args, "-tags", strings.Join(tags, ",")) + if specs != "" { + args = append(args, fmt.Sprintf("-specs=%s", specs)) } args = append(args, cCtx.Args().Slice()...) @@ -142,9 +137,9 @@ func main() { }, }, { - Name: "extract-fixtures", + Name: "extract-fixtures", Aliases: []string{"e"}, - Usage: "Extract gateway testing fixtures that are used by the conformance test suite", + Usage: "Extract gateway testing fixtures that are used by the conformance test suite", Flags: []cli.Flag{ &cli.StringFlag{ Name: "directory", diff --git a/tests/init.go b/tests/init.go new file mode 100644 index 000000000..fc20b16a5 --- /dev/null +++ b/tests/init.go @@ -0,0 +1,60 @@ +package tests + +import ( + "flag" + "regexp" + "strings" + + "github.com/ipfs/gateway-conformance/tooling/specs" +) + +type specsFlag string + +func (s *specsFlag) String() string { + return string(*s) +} + +func (s *specsFlag) Set(value string) error { + names := strings.Split(value, ",") + var only, enable, disable = []specs.Spec{}, []specs.Spec{}, []specs.Spec{} + for _, name := range names { + spec, err := specs.FromString(regexp.MustCompile(`^[-+]`).ReplaceAllString(name, "")) + if err != nil { + return err + } + if strings.HasPrefix(name, "+") { + enable = append(enable, spec) + } else if strings.HasPrefix(name, "-") { + disable = append(disable, spec) + } else { + only = append(only, spec) + } + } + if len(only) > 0 { + // disable all specs + for _, spec := range specs.All() { + spec.Disable() + } + // enable only the specified specs + for _, spec := range only { + spec.Enable() + } + } else { + // enable the specified specs + for _, spec := range enable { + spec.Enable() + } + // disable the specified specs + for _, spec := range disable { + spec.Disable() + } + } + *s = specsFlag(value) + return nil +} + +var specsFlagValue specsFlag + +func init() { + flag.Var(&specsFlagValue, "specs", "comma-separated list of specs to test") +} diff --git a/tests/t0114_gateway_subdomains_test.go b/tests/t0114_gateway_subdomains_test.go index 1b30173c6..86ac705b0 100644 --- a/tests/t0114_gateway_subdomains_test.go +++ b/tests/t0114_gateway_subdomains_test.go @@ -1,6 +1,3 @@ -//go:build !no_subdomain_gateway_spec -// +build !no_subdomain_gateway_spec - package tests import ( @@ -8,6 +5,7 @@ import ( "testing" . "github.com/ipfs/gateway-conformance/tooling/check" + . "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) @@ -58,5 +56,7 @@ func TestGatewaySubdomains(t *testing.T) { }, } - Run(t, tests) + if SubdomainGateway.IsEnabled() { + Run(t, tests) + } } diff --git a/tooling/specs/specs.go b/tooling/specs/specs.go new file mode 100644 index 000000000..1d68ea735 --- /dev/null +++ b/tooling/specs/specs.go @@ -0,0 +1,72 @@ +package specs + +import "fmt" + +type maturity string + +const ( + wip maturity = "wip" + draft maturity = "draft" + reliable maturity = "reliable" + stable maturity = "stable" + permanent maturity = "permanent" + deprecated maturity = "deprecated" +) + +func (m maturity) isMature() bool { + switch m { + case reliable, stable, permanent: + return true + default: + return false + } +} + +type Spec string + +const ( + SubdomainGateway Spec = "subdomain-gateway" +) + +var specMaturity = map[Spec]maturity{ + SubdomainGateway: stable, +} + +func (s Spec) IsMature() bool { + return specMaturity[s].isMature() +} + +var specEnabled = map[Spec]bool{} + +func (s Spec) IsEnabled() bool { + if enabled, ok := specEnabled[s]; ok { + return enabled + } else { + return s.IsMature() + } +} + +func (s Spec) Enable() { + specEnabled[s] = true +} + +func (s Spec) Disable() { + specEnabled[s] = false +} + +func All() []Spec { + specs := []Spec{} + for spec := range specMaturity { + specs = append(specs, spec) + } + return specs +} + +func FromString(name string) (Spec, error) { + for _, spec := range All() { + if string(spec) == name { + return spec, nil + } + } + return "", fmt.Errorf("unknown spec: %s", name) +}