Add memifproxy chain element

If a client receives an fd for a unix file socket via grpcfd,
then it is possible that vpp will be unable to open that file in its
/proc/${pid}/fd/${fd} form.  This is because while the receiving
client process gets the rights to that open file, vpp doesn't necessarily
have them.

In order to resolve this issue, the receiving client proxies the memif
control socket communications bilaterally.

Signed-off-by: Ed Warnicke <hagbard@gmail.com>

go.mod

@@ -6,6 +6,7 @@ require (
 	git.fd.io/govpp.git v0.3.6-0.20200903151113-c94a96227985
 	github.com/edwarnicke/govpp v0.0.0-20201111163523-106f68b6ba26
 	github.com/golang/protobuf v1.4.3
+	github.com/hashicorp/go-multierror v1.0.0
 	github.com/networkservicemesh/api v0.0.0-20210202152048-ec956057eb3a
 	github.com/networkservicemesh/sdk v0.0.0-20210208092844-64f6aa269f63
 	github.com/pkg/errors v0.9.1

pkg/networkservice/mechanisms/memif/client.go

@@ -14,6 +14,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// +build !windows
+
 package memif
 
 import (
@@ -23,10 +25,13 @@ import (
 	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/core/chain"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
 	"google.golang.org/grpc"
 
 	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
+
+	"github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/memif/memifproxy"
 )
 
 type memifClient struct {
@@ -35,9 +40,12 @@ type memifClient struct {
 
 // NewClient provides a NetworkServiceClient chain elements that support the memif Mechanism
 func NewClient(vppConn api.Connection) networkservice.NetworkServiceClient {
-	return &memifClient{
-		vppConn: vppConn,
-	}
+	return chain.NewNetworkServiceClient(
+		&memifClient{
+			vppConn: vppConn,
+		},
+		memifproxy.New(),
+	)
 }
 
 func (m *memifClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) {

pkg/networkservice/mechanisms/memif/memifproxy/client.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2020 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package memifproxy
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	memifMech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/memif"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+)
+
+const (
+	memifNetwork = "unixpacket"
+	maxFDCount   = 1
+	bufferSize   = 128
+)
+
+type memifProxyClient struct{}
+
+// New - create a new memifProxy client chain element
+func New() networkservice.NetworkServiceClient {
+	return &memifProxyClient{}
+}
+
+func (m *memifProxyClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) {
+	conn, err := next.Client(ctx).Request(ctx, request, opts...)
+	if err != nil {
+		return nil, err
+	}
+	mechanism := memifMech.ToMechanism(conn.GetMechanism())
+	if mechanism == nil {
+		return conn, nil
+	}
+	// If we are already running a proxy... just keep running it
+	if _, ok := load(ctx, true); ok {
+		return conn, nil
+	}
+	err = os.MkdirAll(filepath.Dir(listenSocketFilename(conn)), 0700)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to mkdir %s", filepath.Dir(listenSocketFilename(conn)))
+	}
+	listener, err := newProxyListener(mechanism, listenSocketFilename(conn))
+	if err != nil {
+		return nil, err
+	}
+	store(ctx, metadata.IsClient(m), listener)
+	return conn, nil
+}
+
+func (m *memifProxyClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {
+	rv, err := next.Client(ctx).Close(ctx, conn)
+	if listener, ok := loadAndDelete(ctx, metadata.IsClient(m)); ok {
+		_ = listener.Close()
+		_ = os.RemoveAll(filepath.Dir(listenSocketFilename(conn)))
+	}
+	return rv, err
+}
+
+func listenSocketFilename(conn *networkservice.Connection) string {
+	return filepath.Join(os.TempDir(), "memifproxy", conn.GetId(), "memif.socket")
+}

pkg/networkservice/mechanisms/memif/memifproxy/doc.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2020 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package memifproxy provides a NetworkServiceClient chain element to 'proxy' to the memif control socket
+// This is done in case the vpp instance can't open the memif socketfile
+package memifproxy

pkg/networkservice/mechanisms/memif/memifproxy/metadata.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2020 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package memifproxy
+
+import (
+	"context"
+
+	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
+)
+
+type key struct{}
+
+// store sets the *proxyListener stored in per Connection.Id metadata.
+func store(ctx context.Context, isClient bool, listener *proxyListener) {
+	metadata.Map(ctx, isClient).Store(key{}, listener)
+}
+
+// load returns the *proxyListener stored in per Connection.Id metadata, or nil if no
+// value is present.
+// The ok result indicates whether value was found in the per Connection.Id metadata.
+func load(ctx context.Context, isClient bool) (value *proxyListener, ok bool) {
+	rawValue, ok := metadata.Map(ctx, isClient).Load(key{})
+	if !ok {
+		return
+	}
+	value, ok = rawValue.(*proxyListener)
+	return value, ok
+}
+
+// loadAndDelete deletes the *proxyListener stored in per Connection.Id metadata,
+// returning the previous value if any. The loaded result reports whether the key was present.
+func loadAndDelete(ctx context.Context, isClient bool) (value *proxyListener, ok bool) {
+	rawValue, ok := metadata.Map(ctx, isClient).LoadAndDelete(key{})
+	if !ok {
+		return
+	}
+	value, ok = rawValue.(*proxyListener)
+	return value, ok
+}

pkg/networkservice/mechanisms/memif/memifproxy/proxy_connection.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2020 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package memifproxy
+
+import (
+	"net"
+	"syscall"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+)
+
+type proxyConnection struct {
+	in  net.Conn
+	out net.Conn
+}
+
+func newProxyConnection(in, out net.Conn) (*proxyConnection, error) {
+	p := &proxyConnection{
+		in:  in,
+		out: out,
+	}
+	if err := p.copy(in, out); err != nil {
+		return nil, err
+	}
+	if err := p.copy(out, in); err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+func (p *proxyConnection) Close() error {
+	inErr := p.in.Close()
+	outErr := p.out.Close()
+	if inErr != nil {
+		return multierror.Append(inErr, outErr)
+	}
+	return outErr
+}
+
+func (p *proxyConnection) copy(dst, src net.Conn) error {
+	b := make([]byte, bufferSize)
+	unixsrc, unixSrcOK := src.(interface {
+		ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *net.UnixAddr, err error)
+	})
+	if !unixSrcOK {
+		return errors.Errorf("%s does not implement ReadMsgUnix", src.LocalAddr())
+	}
+
+	unixdst, unixdstOK := dst.(interface {
+		WriteMsgUnix(b, oob []byte, addr *net.UnixAddr) (n, oobn int, err error)
+	})
+
+	if !unixdstOK {
+		return errors.Errorf("%s does not implement ReadMsgUnix", dst.LocalAddr())
+	}
+
+	go func() {
+		oob := make([]byte, syscall.CmsgSpace(4*maxFDCount))
+		for {
+			var writeN, writeoob int
+			readn, readoobn, _, _, err := unixsrc.ReadMsgUnix(b, oob)
+			if err != nil {
+				return
+			}
+			for writeN < readn {
+				writeN, writeoob, err = unixdst.WriteMsgUnix(b[writeN:readn], oob[writeoob:readoobn], nil)
+				if err != nil {
+					return
+				}
+			}
+		}
+	}()
+	return nil
+}