From 92f20d9f45cf799294fd1cd3acb245105706ac79 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 5 Jun 2023 14:14:23 +0200 Subject: [PATCH] feat(routing/http): delegated IPNS client and server implementation --- CHANGELOG.md | 1 + routing/http/client/client.go | 73 +++++++++++++++- routing/http/client/client_test.go | 104 ++++++++++++++++++++++ routing/http/server/server.go | 122 ++++++++++++++++++++++++-- routing/http/server/server_test.go | 134 +++++++++++++++++++++++++++++ 5 files changed, 427 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efc2c6c12..579431789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The following emojis are used to highlight certain changes: ## [Unreleased] +- The `routing/http` client and server now support Delegated IPNS as per [IPIP-379](https://github.com/ipfs/specs/pull/379). - ✨ The gateway templates were updated to provide better features for users and gateway implementers: - New human-friendly error messages. - Updated, higher-definition icons in directory listings. diff --git a/routing/http/client/client.go b/routing/http/client/client.go index b3a74150c..f9d73096e 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -6,13 +6,16 @@ import ( "encoding/json" "errors" "fmt" + "io" "mime" "net/http" "strings" "time" "github.com/benbjohnson/clock" + "github.com/gogo/protobuf/proto" ipns "github.com/ipfs/boxo/ipns" + ipns_pb "github.com/ipfs/boxo/ipns/pb" "github.com/ipfs/boxo/routing/http/contentrouter" "github.com/ipfs/boxo/routing/http/internal/drjson" "github.com/ipfs/boxo/routing/http/server" @@ -41,8 +44,9 @@ var ( ) const ( - mediaTypeJSON = "application/json" - mediaTypeNDJSON = "application/x-ndjson" + mediaTypeJSON = "application/json" + mediaTypeNDJSON = "application/x-ndjson" + mediaTypeIPNSRecord = "application/vnd.ipfs.ipns-record" ) type client struct { @@ -324,3 +328,68 @@ func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.Wri return 0, nil } + +func (c *client) GetIPNSRecord(ctx context.Context, pid peer.ID) (*ipns_pb.IpnsEntry, error) { + url := c.baseURL + "/routing/v1/ipns/" + peer.ToCid(pid).String() + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + httpReq.Header.Set("Accept", mediaTypeIPNSRecord) + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("making HTTP req to get IPNS record: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, httpError(resp.StatusCode, resp.Body) + } + + // Limit the reader to the maximum record size. + rawRecord, err := io.ReadAll(io.LimitReader(resp.Body, int64(ipns.MaxRecordSize))) + if err != nil { + return nil, fmt.Errorf("making HTTP req to get IPNS record: %w", err) + } + + record, err := ipns.UnmarshalIpnsEntry(rawRecord) + if err != nil { + return nil, fmt.Errorf("IPNS record from remote endpoint is not valid: %w", err) + } + + err = ipns.ValidateWithPeerID(pid, record) + if err != nil { + return nil, fmt.Errorf("IPNS record from remote endpoint is not valid: %w", err) + } + + return record, nil +} + +func (c *client) PutIPNSRecord(ctx context.Context, pid peer.ID, record *ipns_pb.IpnsEntry) error { + url := c.baseURL + "/routing/v1/ipns/" + peer.ToCid(pid).String() + + rawRecord, err := proto.Marshal(record) + if err != nil { + return err + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(rawRecord)) + if err != nil { + return err + } + httpReq.Header.Set("Content-Type", mediaTypeIPNSRecord) + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return fmt.Errorf("making HTTP req to get IPNS record: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return httpError(resp.StatusCode, resp.Body) + } + + return nil +} diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 880fa33e1..0a3ace20a 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -3,6 +3,7 @@ package client import ( "context" "crypto/rand" + "errors" "net/http" "net/http/httptest" "runtime" @@ -10,6 +11,10 @@ import ( "time" "github.com/benbjohnson/clock" + "github.com/gogo/protobuf/proto" + "github.com/ipfs/boxo/coreiface/path" + ipns "github.com/ipfs/boxo/ipns" + ipns_pb "github.com/ipfs/boxo/ipns/pb" "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" @@ -31,6 +36,7 @@ func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limi args := m.Called(ctx, key, limit) return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) } + func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) { args := m.Called(ctx, req) return args.Get(0).(time.Duration), args.Error(1) @@ -41,6 +47,16 @@ func (m *mockContentRouter) Provide(ctx context.Context, req *server.WriteProvid return args.Get(0).(types.ProviderResponse), args.Error(1) } +func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, pid peer.ID) (*ipns_pb.IpnsEntry, error) { + args := m.Called(ctx, pid) + return args.Get(0).(*ipns_pb.IpnsEntry), args.Error(1) +} + +func (m *mockContentRouter) PutIPNSRecord(ctx context.Context, pid peer.ID, record *ipns_pb.IpnsEntry) error { + args := m.Called(ctx, pid, record) + return args.Error(0) +} + type testDeps struct { // recordingHandler records requests received on the server side recordingHandler *recordingHandler @@ -441,3 +457,91 @@ func TestClient_Provide(t *testing.T) { }) } } + +func makePeerID(t *testing.T) (crypto.PrivKey, peer.ID) { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPrivateKey(sk) + require.NoError(t, err) + + return sk, pid +} + +func makeIPNSRecord(t *testing.T, sk crypto.PrivKey) (*ipns_pb.IpnsEntry, []byte) { + cid, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + + path := path.IpfsPath(cid) + eol := time.Now().Add(time.Hour * 48) + ttl := time.Second * 20 + + record, err := ipns.Create(sk, []byte(path.String()), 1, eol, ttl) + require.NoError(t, err) + + rawRecord, err := proto.Marshal(record) + require.NoError(t, err) + + return record, rawRecord +} + +func TestClient_IPNS(t *testing.T) { + t.Run("Get IPNS Record", func(t *testing.T) { + sk, pid := makePeerID(t) + record, _ := makeIPNSRecord(t, sk) + + deps := makeTestDeps(t, nil, nil) + client := deps.client + router := deps.router + + router.On("GetIPNSRecord", mock.Anything, pid).Return(record, nil) + + receivedRecord, err := client.GetIPNSRecord(context.Background(), pid) + require.NoError(t, err) + require.Equal(t, record, receivedRecord) + }) + + t.Run("Get IPNS Record returns error if server sends bad data", func(t *testing.T) { + sk, _ := makePeerID(t) + record, _ := makeIPNSRecord(t, sk) + _, pid2 := makePeerID(t) + + deps := makeTestDeps(t, nil, nil) + client := deps.client + router := deps.router + + router.On("GetIPNSRecord", mock.Anything, pid2).Return(record, nil) + + receivedRecord, err := client.GetIPNSRecord(context.Background(), pid2) + require.Error(t, err) + require.Nil(t, receivedRecord) + }) + + t.Run("Get IPNS Record returns error if server errors", func(t *testing.T) { + _, pid := makePeerID(t) + + deps := makeTestDeps(t, nil, nil) + client := deps.client + router := deps.router + + router.On("GetIPNSRecord", mock.Anything, pid).Return(nil, errors.New("something wrong happened")) + + receivedRecord, err := client.GetIPNSRecord(context.Background(), pid) + require.Error(t, err) + require.Nil(t, receivedRecord) + }) + + t.Run("Put IPNS Record", func(t *testing.T) { + sk, pid := makePeerID(t) + record, _ := makeIPNSRecord(t, sk) + + deps := makeTestDeps(t, nil, nil) + client := deps.client + router := deps.router + + router.On("PutIPNSRecord", mock.Anything, pid, record).Return(nil) + + err := client.PutIPNSRecord(context.Background(), pid, record) + require.NoError(t, err) + }) +} diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 47c075f0a..2a768f949 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -9,10 +9,15 @@ import ( "io" "mime" "net/http" + "strconv" "strings" "time" + "github.com/cespare/xxhash/v2" + "github.com/gogo/protobuf/proto" "github.com/gorilla/mux" + "github.com/ipfs/boxo/ipns" + ipns_pb "github.com/ipfs/boxo/ipns/pb" "github.com/ipfs/boxo/routing/http/internal/drjson" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" @@ -25,9 +30,10 @@ import ( ) const ( - mediaTypeJSON = "application/json" - mediaTypeNDJSON = "application/x-ndjson" - mediaTypeWildcard = "*/*" + mediaTypeJSON = "application/json" + mediaTypeNDJSON = "application/x-ndjson" + mediaTypeWildcard = "*/*" + mediaTypeIPNSRecord = "application/vnd.ipfs.ipns-record" DefaultRecordsLimit = 20 DefaultStreamingRecordsLimit = 0 @@ -35,8 +41,11 @@ const ( var logger = logging.Logger("service/server/delegatedrouting") -const ProvidePath = "/routing/v1/providers/" -const FindProvidersPath = "/routing/v1/providers/{cid}" +const ( + ProvidePath = "/routing/v1/providers/" + FindProvidersPath = "/routing/v1/providers/{cid}" + IPNSPath = "/routing/v1/ipns/{cid}" +) type FindProvidersAsyncResponse struct { ProviderResponse types.ProviderResponse @@ -49,6 +58,13 @@ type ContentRouter interface { FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) Provide(ctx context.Context, req *WriteProvideRequest) (types.ProviderResponse, error) + + // GetIPNSRecord searches for an IPNS Record for the given peer ID. + GetIPNSRecord(ctx context.Context, pid peer.ID) (*ipns_pb.IpnsEntry, error) + + // PutIPNSRecord stores the provided IPNS Record for the given peer ID. It is + // guaranteed that the record matches the provided peer ID. + PutIPNSRecord(ctx context.Context, pid peer.ID, record *ipns_pb.IpnsEntry) error } type BitswapWriteProvideRequest struct { @@ -105,6 +121,9 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { r.HandleFunc(ProvidePath, server.provide).Methods(http.MethodPut) r.HandleFunc(FindProvidersPath, server.findProviders).Methods(http.MethodGet) + r.HandleFunc(IPNSPath, server.getIPNSRecord).Methods(http.MethodGet) + r.HandleFunc(IPNSPath, server.putIPNSRecord).Methods(http.MethodPut) + return r } @@ -296,6 +315,99 @@ func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.Result } } +func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Accept"), mediaTypeIPNSRecord) { + writeErr(w, "GetIPNSRecord", http.StatusNotAcceptable, errors.New("content type in 'Accept' header is missing or not supported")) + return + } + + vars := mux.Vars(r) + cidStr := vars["cid"] + cid, err := cid.Decode(cidStr) + if err != nil { + writeErr(w, "GetIPNSRecord", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + return + } + + pid, err := peer.FromCid(cid) + if err != nil { + writeErr(w, "GetIPNSRecord", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + return + } + + record, err := s.svc.GetIPNSRecord(r.Context(), pid) + if err != nil { + writeErr(w, "GetIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return + } + + rawRecord, err := proto.Marshal(record) + if err != nil { + writeErr(w, "GetIPNSRecord", http.StatusInternalServerError, err) + return + } + + if record.Ttl != nil { + seconds := int(time.Duration(*record.Ttl).Seconds()) + w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", seconds)) + } else { + w.Header().Set("Cache-Control", "max-age=60") + } + + recordEtag := strconv.FormatUint(xxhash.Sum64(rawRecord), 32) + w.Header().Set("Etag", recordEtag) + w.Header().Set("Content-Type", mediaTypeIPNSRecord) + w.Write(rawRecord) +} + +func (s *server) putIPNSRecord(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Content-Type"), mediaTypeIPNSRecord) { + writeErr(w, "PutIPNSRecord", http.StatusNotAcceptable, errors.New("content type in 'Content-Type' header is missing or not supported")) + return + } + + vars := mux.Vars(r) + cidStr := vars["cid"] + cid, err := cid.Decode(cidStr) + if err != nil { + writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + return + } + + pid, err := peer.FromCid(cid) + if err != nil { + writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + return + } + + // Limit the reader to the maximum record size. + rawRecord, err := io.ReadAll(io.LimitReader(r.Body, int64(ipns.MaxRecordSize))) + if err != nil { + writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is too long: %w", err)) + return + } + + record, err := ipns.UnmarshalIpnsEntry(rawRecord) + if err != nil { + writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + return + } + + err = ipns.ValidateWithPeerID(pid, record) + if err != nil { + writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + return + } + + err = s.svc.PutIPNSRecord(r.Context(), pid, record) + if err != nil { + writeErr(w, "PutIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return + } + + w.WriteHeader(http.StatusOK) +} + func writeJSONResult(w http.ResponseWriter, method string, val any) { w.Header().Add("Content-Type", mediaTypeJSON) diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index 69db7d556..5c3825a1e 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -1,16 +1,23 @@ package server import ( + "bytes" "context" + "crypto/rand" "io" "net/http" "net/http/httptest" "testing" "time" + "github.com/gogo/protobuf/proto" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipns" + ipns_pb "github.com/ipfs/boxo/ipns/pb" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -117,6 +124,123 @@ func TestResponse(t *testing.T) { }) } +func makePeerID(t *testing.T) (crypto.PrivKey, peer.ID) { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPrivateKey(sk) + require.NoError(t, err) + + return sk, pid +} + +func makeIPNSRecord(t *testing.T, cid cid.Cid, sk crypto.PrivKey) (*ipns_pb.IpnsEntry, []byte) { + path := path.IpfsPath(cid) + eol := time.Now().Add(time.Hour * 48) + ttl := time.Second * 20 + + record, err := ipns.Create(sk, []byte(path.String()), 1, eol, ttl) + require.NoError(t, err) + + rawRecord, err := proto.Marshal(record) + require.NoError(t, err) + + return record, rawRecord +} + +func TestIPNS(t *testing.T) { + cid1, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + + sk, pid1 := makePeerID(t) + record1, rawRecord1 := makeIPNSRecord(t, cid1, sk) + + _, pid2 := makePeerID(t) + + makeRequest := func(t *testing.T, router *mockContentRouter, path string) *http.Response { + server := httptest.NewServer(Handler(router)) + t.Cleanup(server.Close) + serverAddr := "http://" + server.Listener.Addr().String() + urlStr := serverAddr + path + req, err := http.NewRequest(http.MethodGet, urlStr, nil) + require.NoError(t, err) + req.Header.Set("Accept", mediaTypeIPNSRecord) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp + } + + t.Run("GET /routing/v1/ipns/{cid-peer-id} returns 200", func(t *testing.T) { + t.Parallel() + + router := &mockContentRouter{} + router.On("GetIPNSRecord", mock.Anything, pid1).Return(record1, nil) + + resp := makeRequest(t, router, "/routing/v1/ipns/"+peer.ToCid(pid1).String()) + require.Equal(t, 200, resp.StatusCode) + require.Equal(t, mediaTypeIPNSRecord, resp.Header.Get("Content-Type")) + require.NotEmpty(t, resp.Header.Get("Etag")) + require.Equal(t, "max-age=20", resp.Header.Get("Cache-Control")) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, body, rawRecord1) + }) + + t.Run("GET /routing/v1/ipns/{non-peer-cid} returns 400", func(t *testing.T) { + t.Parallel() + router := &mockContentRouter{} + resp := makeRequest(t, router, "/routing/v1/ipns/"+cid1.String()) + require.Equal(t, 400, resp.StatusCode) + }) + + t.Run("GET /routing/v1/ipns/{peer-id} returns 400", func(t *testing.T) { + t.Parallel() + router := &mockContentRouter{} + resp := makeRequest(t, router, "/routing/v1/ipns/"+pid1.String()) + require.Equal(t, 400, resp.StatusCode) + }) + + t.Run("PUT /routing/v1/ipns/{cid-peer-id} returns 200", func(t *testing.T) { + t.Parallel() + + router := &mockContentRouter{} + router.On("PutIPNSRecord", mock.Anything, pid1, record1).Return(nil) + + server := httptest.NewServer(Handler(router)) + t.Cleanup(server.Close) + serverAddr := "http://" + server.Listener.Addr().String() + urlStr := serverAddr + "/routing/v1/ipns/" + peer.ToCid(pid1).String() + + req, err := http.NewRequest(http.MethodPut, urlStr, bytes.NewReader(rawRecord1)) + require.NoError(t, err) + req.Header.Set("Content-Type", mediaTypeIPNSRecord) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + }) + + t.Run("PUT /routing/v1/ipns/{cid-peer-id} returns 400 for wrong record", func(t *testing.T) { + t.Parallel() + + router := &mockContentRouter{} + + server := httptest.NewServer(Handler(router)) + t.Cleanup(server.Close) + serverAddr := "http://" + server.Listener.Addr().String() + urlStr := serverAddr + "/routing/v1/ipns/" + peer.ToCid(pid2).String() + + req, err := http.NewRequest(http.MethodPut, urlStr, bytes.NewReader(rawRecord1)) + require.NoError(t, err) + req.Header.Set("Content-Type", mediaTypeIPNSRecord) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.Equal(t, 400, resp.StatusCode) + }) +} + type mockContentRouter struct{ mock.Mock } func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) { @@ -132,3 +256,13 @@ func (m *mockContentRouter) Provide(ctx context.Context, req *WriteProvideReques args := m.Called(ctx, req) return args.Get(0).(types.ProviderResponse), args.Error(1) } + +func (m *mockContentRouter) GetIPNSRecord(ctx context.Context, pid peer.ID) (*ipns_pb.IpnsEntry, error) { + args := m.Called(ctx, pid) + return args.Get(0).(*ipns_pb.IpnsEntry), args.Error(1) +} + +func (m *mockContentRouter) PutIPNSRecord(ctx context.Context, pid peer.ID, record *ipns_pb.IpnsEntry) error { + args := m.Called(ctx, pid, record) + return args.Error(0) +}