feat: implement --specs=([+-]?(?P<name>[^,]+)[,|$])+

.github/actions/test/action.yml

@@ -18,10 +18,10 @@ inputs:
   markdown:
     description: "The path where the Markdown report will be saved"
     required: false
-  subdomain-gateway-spec:
-    description: "Whether the gateway implements the subdomain gateway spec"
+  specs:
+    description: "A comma-separated list of specs to test"
     required: false
-    default: "true"
+    default: ""
   args:
     description: "The arguments to pass to the test command"
     required: false
@@ -35,13 +35,13 @@ runs:
       env:
         URL: ${{ inputs.gateway-url }}
         JSON: ${{ inputs.json }}
-        SUBDOMAIN: ${{ inputs.subdomain-gateway-spec }}
+        SPECS: ${{ inputs.specs }}
       with:
         repository: ${{ steps.github.outputs.action_repository }}
         ref: ${{ steps.github.outputs.action_ref }}
         dockerfile: Dockerfile
         opts: --network=host
-        args: test --url="$URL" --json="$JSON" --subdomain="$SUBDOMAIN" -- ${{ inputs.args }}
+        args: test --url="$URL" --json="$JSON" --specs="$SPECS" -- ${{ inputs.args }}
     - name: Create the XML
       if: (inputs.xml || inputs.html || inputs.markdown) && (failure() || success())
       uses: pl-strflt/gotest-json-to-junit-xml@v1

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
           xml: output.xml
           html: output.html
           markdown: output.md
-          subdomain-gateway-spec: false
+          specs: -subdomain-gateway
           args: -skip TestGatewayCar
       - name: Set summary
         if: (failure() || success())

cmd/gateway-conformance/main.go

@@ -58,7 +58,7 @@ func copyFiles(inputPaths []string, outputDirectoryPath string) error {
 func main() {
 	var gatewayURL string
 	var jsonOutput string
-	var subdomainGatewaySpec bool
+	var specs string
 	var directory string
 	var merged bool
 
@@ -67,9 +67,9 @@ func main() {
 		Usage: "Tooling for the gateway test suite",
 		Commands: []*cli.Command{
 			{
-				Name:  "test",
+				Name:    "test",
 				Aliases: []string{"t"},
-				Usage: "Run the conformance test suite against your gateway",
+				Usage:   "Run the conformance test suite against your gateway",
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:        "gateway-url",
@@ -85,23 +85,18 @@ func main() {
 						Value:       "",
 						Destination: &jsonOutput,
 					},
-					&cli.BoolFlag{
-						Name:        "subdomain-gateway-spec",
-						Aliases:     []string{"subdomain-gateway", "subdomain"},
-						Usage:       "Whether the gateway implements the subdomain gateway spec",
-						Value:       true,
-						Destination: &subdomainGatewaySpec,
+					&cli.StringFlag{
+						Name:        "specs",
+						Usage:       "A comma-separated list of specs to test",
+						Value:       "",
+						Destination: &specs,
 					},
 				},
 				Action: func(cCtx *cli.Context) error {
 					args := []string{"test", "./tests", "-test.v=test2json"}
 
-					tags := []string{}
-					if !subdomainGatewaySpec {
-						tags = append(tags, "no_subdomain_gateway_spec")
-					}
-					if len(tags) > 0 {
-						args = append(args, "-tags", strings.Join(tags, ","))
+					if specs != "" {
+						args = append(args, fmt.Sprintf("-specs=%s", specs))
 					}
 
 					args = append(args, cCtx.Args().Slice()...)
@@ -142,9 +137,9 @@ func main() {
 				},
 			},
 			{
-				Name:  "extract-fixtures",
+				Name:    "extract-fixtures",
 				Aliases: []string{"e"},
-				Usage: "Extract gateway testing fixtures that are used by the conformance test suite",
+				Usage:   "Extract gateway testing fixtures that are used by the conformance test suite",
 				Flags: []cli.Flag{
 					&cli.StringFlag{
 						Name:        "directory",

tests/init.go

@@ -0,0 +1,60 @@
+package tests
+
+import (
+	"flag"
+	"regexp"
+	"strings"
+
+	"github.com/ipfs/gateway-conformance/tooling/specs"
+)
+
+type specsFlag string
+
+func (s *specsFlag) String() string {
+	return string(*s)
+}
+
+func (s *specsFlag) Set(value string) error {
+	names := strings.Split(value, ",")
+	var only, enable, disable = []specs.Spec{}, []specs.Spec{}, []specs.Spec{}
+	for _, name := range names {
+		spec, err := specs.FromString(regexp.MustCompile(`^[-+]`).ReplaceAllString(name, ""))
+		if err != nil {
+			return err
+		}
+		if strings.HasPrefix(name, "+") {
+			enable = append(enable, spec)
+		} else if strings.HasPrefix(name, "-") {
+			disable = append(disable, spec)
+		} else {
+			only = append(only, spec)
+		}
+	}
+	if len(only) > 0 {
+		// disable all specs
+		for _, spec := range specs.All() {
+			spec.Disable()
+		}
+		// enable only the specified specs
+		for _, spec := range only {
+			spec.Enable()
+		}
+	} else {
+		// enable the specified specs
+		for _, spec := range enable {
+			spec.Enable()
+		}
+		// disable the specified specs
+		for _, spec := range disable {
+			spec.Disable()
+		}
+	}
+	*s = specsFlag(value)
+	return nil
+}
+
+var specsFlagValue specsFlag
+
+func init() {
+	flag.Var(&specsFlagValue, "specs", "comma-separated list of specs to test")
+}

tests/t0114_gateway_subdomains_test.go

@@ -1,13 +1,11 @@
-//go:build !no_subdomain_gateway_spec
-// +build !no_subdomain_gateway_spec
-
 package tests
 
 import (
 	"fmt"
 	"testing"
 
 	. "github.com/ipfs/gateway-conformance/tooling/check"
+	. "github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
@@ -58,5 +56,7 @@ func TestGatewaySubdomains(t *testing.T) {
 		},
 	}
 
-	Run(t, tests)
+	if SubdomainGateway.IsEnabled() {
+		Run(t, tests)
+	}
 }

tooling/specs/specs.go

@@ -0,0 +1,72 @@
+package specs
+
+import "fmt"
+
+type maturity string
+
+const (
+	wip        maturity = "wip"
+	draft      maturity = "draft"
+	reliable   maturity = "reliable"
+	stable     maturity = "stable"
+	permanent  maturity = "permanent"
+	deprecated maturity = "deprecated"
+)
+
+func (m maturity) isMature() bool {
+	switch m {
+	case reliable, stable, permanent:
+		return true
+	default:
+		return false
+	}
+}
+
+type Spec string
+
+const (
+	SubdomainGateway Spec = "subdomain-gateway"
+)
+
+var specMaturity = map[Spec]maturity{
+	SubdomainGateway: stable,
+}
+
+func (s Spec) IsMature() bool {
+	return specMaturity[s].isMature()
+}
+
+var specEnabled = map[Spec]bool{}
+
+func (s Spec) IsEnabled() bool {
+	if enabled, ok := specEnabled[s]; ok {
+		return enabled
+	} else {
+		return s.IsMature()
+	}
+}
+
+func (s Spec) Enable() {
+	specEnabled[s] = true
+}
+
+func (s Spec) Disable() {
+	specEnabled[s] = false
+}
+
+func All() []Spec {
+	specs := []Spec{}
+	for spec := range specMaturity {
+		specs = append(specs, spec)
+	}
+	return specs
+}
+
+func FromString(name string) (Spec, error) {
+	for _, spec := range All() {
+		if string(spec) == name {
+			return spec, nil
+		}
+	}
+	return "", fmt.Errorf("unknown spec: %s", name)
+}