test: port remote pinning tests to Go

This also means that rb-pinning-service-api is no longer required for
running remote pinning tests. This alone saves at least 3 minutes in
test runtime in CI because we don't need to checkout the repo, build
the Docker image, run it, etc.

Instead this implements a simple pinning service in Go that the test
runs in-process, with a callback that can be used to control the async
behavior of the pinning service (e.g. simulate work happening
asynchronously like transitioning from "queued" -> "pinning" ->
"pinned").

This also adds an environment variable to Kubo to control the MFS
remote pin polling interval, so that we don't have to wait 30 seconds
in the test for MFS changes to be repinned. This is purely for tests
so I don't think we should document this.

This entire test suite runs in around 2.5 sec on my laptop, compared to
the existing 3+ minutes in CI.

.circleci/main.yml

@@ -165,28 +165,11 @@ jobs:
     - run: sudo apt install socat net-tools fish libxml2-utils
     - checkout
 
-    - run:
-        mkdir rb-pinning-service-api &&
-        cd rb-pinning-service-api &&
-        git init &&
-        git remote add origin  https://github.com/ipfs-shipyard/rb-pinning-service-api.git &&
-        git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 &&
-        git checkout FETCH_HEAD
-    - run:
-        cd rb-pinning-service-api &&
-        (for i in {1..3}; do docker-compose pull && break || sleep 5; done) &&
-        docker-compose up -d
-
     - *make_out_dirs
     - *restore_gomod
 
     - run:
-        name: Setup Environment Variables
-        # we need the docker host IP; all ports exported by child containers can be accessed there.
-        command: echo "export TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV
-    - run:
-        echo TEST_DOCKER_HOST=$TEST_DOCKER_HOST &&
-        make -O -j << pipeline.parameters.make_jobs >> test_sharness coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml CONTINUE_ON_S_FAILURE=1 TEST_DOCKER_HOST=$TEST_DOCKER_HOST
+        make -O -j << pipeline.parameters.make_jobs >> test_sharness coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml CONTINUE_ON_S_FAILURE=1
     - run:
         when: always
         command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile

.github/workflows/sharness.yml

@@ -29,24 +29,10 @@ jobs:
           path: kubo
       - name: Install missing tools
         run: sudo apt install -y socat net-tools fish libxml2-utils
-      - name: Checkout IPFS Pinning Service API
-        uses: actions/checkout@v3
-        with:
-          repository: ipfs-shipyard/rb-pinning-service-api
-          ref: 773c3adbb421c551d2d89288abac3e01e1f7c3a8
-          path: rb-pinning-service-api
-      # TODO: check if docker compose (not docker-compose) is available on default gh runners
-      - name: Start IPFS Pinning Service API
-        run: |
-          (for i in {1..3}; do docker compose pull && break || sleep 5; done) &&
-            docker compose up -d
-        working-directory: rb-pinning-service-api
       - name: Restore Go Cache
         uses: protocol/cache-go-action@v1
         with:
           name: ${{ github.job }}
-      - name: Find IPFS Pinning Service API address
-        run: echo "TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $GITHUB_ENV
       - uses: actions/cache@v3
         with:
           path: test/sharness/lib/dependencies

cmd/ipfs/pinmfs.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/libp2p/go-libp2p/core/host"
@@ -31,7 +32,18 @@ func (x lastPin) IsValid() bool {
 	return x != lastPin{}
 }
 
-const daemonConfigPollInterval = time.Minute / 2
+var daemonConfigPollInterval = time.Minute / 2
+
+func init() {
+	if pollDurStr := os.Getenv("MFS_PIN_POLL_INTERVAL"); pollDurStr != "" {
+		d, err := time.ParseDuration(pollDurStr)
+		if err != nil {
+			mfslog.Error("error parsing MFS_PIN_POLL_INTERVAL, using default:", err)
+		}
+		daemonConfigPollInterval = d
+	}
+}
+
 const defaultRepinInterval = 5 * time.Minute
 
 type pinMFSContext interface {

go.mod

@@ -66,6 +66,7 @@ require (
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jbenet/go-temp-err-catcher v0.1.0
 	github.com/jbenet/goprocess v0.1.4
+	github.com/julienschmidt/httprouter v1.3.0
 	github.com/libp2p/go-doh-resolver v0.4.0
 	github.com/libp2p/go-libp2p v0.26.2
 	github.com/libp2p/go-libp2p-http v0.4.0
@@ -90,6 +91,8 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	github.com/stretchr/testify v1.8.1
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/tidwall/gjson v1.14.4
+	github.com/tidwall/sjson v1.2.5
 	github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	go.opencensus.io v0.24.0
@@ -215,6 +218,8 @@ require (
 	github.com/samber/lo v1.36.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect

go.sum

@@ -700,6 +700,7 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -1331,6 +1332,15 @@ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cb
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
 github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=

test/cli/harness/node.go

@@ -76,6 +76,18 @@ func (n *Node) WriteBytes(filename string, b []byte) {
 	}
 }
 
+func (n *Node) ReadFile(filename string) string {
+	b, err := os.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+func (n *Node) ConfigFile() string {
+	return filepath.Join(n.Dir, "config")
+}
+
 func (n *Node) ReadConfig() *config.Config {
 	cfg, err := serial.Load(filepath.Join(n.Dir, "config"))
 	if err != nil {

test/cli/must.go

@@ -0,0 +1,8 @@
+package cli
+
+func MustVal[V any](val V, err error) V {
+	if err != nil {
+		panic(err)
+	}
+	return val
+}