diff --git a/go.mod b/go.mod index 45df40bb..3e406a75 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( git.fd.io/govpp.git v0.3.6-0.20200903151113-c94a96227985 github.com/edwarnicke/govpp v0.0.0-20201111163523-106f68b6ba26 github.com/golang/protobuf v1.4.3 + github.com/hashicorp/go-multierror v1.0.0 github.com/networkservicemesh/api v0.0.0-20210202152048-ec956057eb3a github.com/networkservicemesh/sdk v0.0.0-20210208092844-64f6aa269f63 github.com/pkg/errors v0.9.1 diff --git a/pkg/networkservice/mechanisms/memif/client.go b/pkg/networkservice/mechanisms/memif/client.go index 37376a09..c5e73b9a 100644 --- a/pkg/networkservice/mechanisms/memif/client.go +++ b/pkg/networkservice/mechanisms/memif/client.go @@ -14,6 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !windows + package memif import ( @@ -23,10 +25,13 @@ import ( "github.com/golang/protobuf/ptypes/empty" "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" "google.golang.org/grpc" "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" + + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/memif/memifproxy" ) type memifClient struct { @@ -35,9 +40,12 @@ type memifClient struct { // NewClient provides a NetworkServiceClient chain elements that support the memif Mechanism func NewClient(vppConn api.Connection) networkservice.NetworkServiceClient { - return &memifClient{ - vppConn: vppConn, - } + return chain.NewNetworkServiceClient( + &memifClient{ + vppConn: vppConn, + }, + memifproxy.New(), + ) } func (m *memifClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { diff --git a/pkg/networkservice/mechanisms/memif/memifproxy/client.go b/pkg/networkservice/mechanisms/memif/memifproxy/client.go new file mode 100644 index 00000000..00ff8816 --- /dev/null +++ b/pkg/networkservice/mechanisms/memif/memifproxy/client.go @@ -0,0 +1,84 @@ +// Copyright (c) 2020 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package memifproxy + +import ( + "context" + "os" + "path/filepath" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/networkservice" + memifMech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/memif" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +const ( + memifNetwork = "unixpacket" + maxFDCount = 1 + bufferSize = 128 +) + +type memifProxyClient struct{} + +// New - create a new memifProxy client chain element +func New() networkservice.NetworkServiceClient { + return &memifProxyClient{} +} + +func (m *memifProxyClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + return nil, err + } + mechanism := memifMech.ToMechanism(conn.GetMechanism()) + if mechanism == nil { + return conn, nil + } + // If we are already running a proxy... just keep running it + if _, ok := load(ctx, true); ok { + return conn, nil + } + err = os.MkdirAll(filepath.Dir(listenSocketFilename(conn)), 0700) + if err != nil { + return nil, errors.Wrapf(err, "unable to mkdir %s", filepath.Dir(listenSocketFilename(conn))) + } + listener, err := newProxyListener(mechanism, listenSocketFilename(conn)) + if err != nil { + return nil, err + } + store(ctx, metadata.IsClient(m), listener) + return conn, nil +} + +func (m *memifProxyClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + rv, err := next.Client(ctx).Close(ctx, conn) + if listener, ok := loadAndDelete(ctx, metadata.IsClient(m)); ok { + _ = listener.Close() + _ = os.RemoveAll(filepath.Dir(listenSocketFilename(conn))) + } + return rv, err +} + +func listenSocketFilename(conn *networkservice.Connection) string { + return filepath.Join(os.TempDir(), "memifproxy", conn.GetId(), "memif.socket") +} diff --git a/pkg/networkservice/mechanisms/memif/memifproxy/doc.go b/pkg/networkservice/mechanisms/memif/memifproxy/doc.go new file mode 100644 index 00000000..6eaff315 --- /dev/null +++ b/pkg/networkservice/mechanisms/memif/memifproxy/doc.go @@ -0,0 +1,19 @@ +// Copyright (c) 2020 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memifproxy provides a NetworkServiceClient chain element to 'proxy' to the memif control socket +// This is done in case the vpp instance can't open the memif socketfile +package memifproxy diff --git a/pkg/networkservice/mechanisms/memif/memifproxy/metadata.go b/pkg/networkservice/mechanisms/memif/memifproxy/metadata.go new file mode 100644 index 00000000..826e59e6 --- /dev/null +++ b/pkg/networkservice/mechanisms/memif/memifproxy/metadata.go @@ -0,0 +1,55 @@ +// Copyright (c) 2020 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package memifproxy + +import ( + "context" + + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" +) + +type key struct{} + +// store sets the *proxyListener stored in per Connection.Id metadata. +func store(ctx context.Context, isClient bool, listener *proxyListener) { + metadata.Map(ctx, isClient).Store(key{}, listener) +} + +// load returns the *proxyListener stored in per Connection.Id metadata, or nil if no +// value is present. +// The ok result indicates whether value was found in the per Connection.Id metadata. +func load(ctx context.Context, isClient bool) (value *proxyListener, ok bool) { + rawValue, ok := metadata.Map(ctx, isClient).Load(key{}) + if !ok { + return + } + value, ok = rawValue.(*proxyListener) + return value, ok +} + +// loadAndDelete deletes the *proxyListener stored in per Connection.Id metadata, +// returning the previous value if any. The loaded result reports whether the key was present. +func loadAndDelete(ctx context.Context, isClient bool) (value *proxyListener, ok bool) { + rawValue, ok := metadata.Map(ctx, isClient).LoadAndDelete(key{}) + if !ok { + return + } + value, ok = rawValue.(*proxyListener) + return value, ok +} diff --git a/pkg/networkservice/mechanisms/memif/memifproxy/proxy_connection.go b/pkg/networkservice/mechanisms/memif/memifproxy/proxy_connection.go new file mode 100644 index 00000000..e6612c0a --- /dev/null +++ b/pkg/networkservice/mechanisms/memif/memifproxy/proxy_connection.go @@ -0,0 +1,91 @@ +// Copyright (c) 2020 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package memifproxy + +import ( + "net" + "syscall" + + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" +) + +type proxyConnection struct { + in net.Conn + out net.Conn +} + +func newProxyConnection(in, out net.Conn) (*proxyConnection, error) { + p := &proxyConnection{ + in: in, + out: out, + } + if err := p.copy(in, out); err != nil { + return nil, err + } + if err := p.copy(out, in); err != nil { + return nil, err + } + return p, nil +} + +func (p *proxyConnection) Close() error { + inErr := p.in.Close() + outErr := p.out.Close() + if inErr != nil { + return multierror.Append(inErr, outErr) + } + return outErr +} + +func (p *proxyConnection) copy(dst, src net.Conn) error { + b := make([]byte, bufferSize) + unixsrc, unixSrcOK := src.(interface { + ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *net.UnixAddr, err error) + }) + if !unixSrcOK { + return errors.Errorf("%s does not implement ReadMsgUnix", src.LocalAddr()) + } + + unixdst, unixdstOK := dst.(interface { + WriteMsgUnix(b, oob []byte, addr *net.UnixAddr) (n, oobn int, err error) + }) + + if !unixdstOK { + return errors.Errorf("%s does not implement ReadMsgUnix", dst.LocalAddr()) + } + + go func() { + oob := make([]byte, syscall.CmsgSpace(4*maxFDCount)) + for { + var writeN, writeoob int + readn, readoobn, _, _, err := unixsrc.ReadMsgUnix(b, oob) + if err != nil { + return + } + for writeN < readn { + writeN, writeoob, err = unixdst.WriteMsgUnix(b[writeN:readn], oob[writeoob:readoobn], nil) + if err != nil { + return + } + } + } + }() + return nil +} diff --git a/pkg/networkservice/mechanisms/memif/memifproxy/proxy_listener.go b/pkg/networkservice/mechanisms/memif/memifproxy/proxy_listener.go new file mode 100644 index 00000000..0f039a74 --- /dev/null +++ b/pkg/networkservice/mechanisms/memif/memifproxy/proxy_listener.go @@ -0,0 +1,101 @@ +// Copyright (c) 2020 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package memifproxy + +import ( + "net" + "net/url" + + "github.com/hashicorp/go-multierror" + memifMech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/memif" + "github.com/pkg/errors" +) + +type proxyListener struct { + listener net.Listener + socketFilename string + proxyConnections []*proxyConnection +} + +func newProxyListener(mechanism *memifMech.Mechanism, listenSocketFilename string) (*proxyListener, error) { + // Extract the socket filename + u, err := url.Parse(mechanism.GetSocketFileURL()) + if err != nil { + return nil, errors.Wrapf(err, "not a valid url %q", mechanism.GetSocketFileURL()) + } + if u.Scheme != memifMech.SocketFileScheme { + return nil, errors.Errorf("socket file url must have scheme %q, actual %q", memifMech.SocketFileScheme, u.Scheme) + } + p := &proxyListener{ + socketFilename: u.Path, + } + + // Do a trial dial to ensure we can actually proxy + trialConn, err := net.Dial(memifNetwork, u.Path) + if err != nil { + return nil, errors.Wrapf(err, "proxyListener unable to dial %s", p.socketFilename) + } + _ = trialConn.Close() + + p.listener, err = net.Listen(memifNetwork, listenSocketFilename) + if err != nil { + return nil, errors.Wrapf(err, "proxyListener unable to listen on %s", listenSocketFilename) + } + go p.accept() + mechanism.SetSocketFileURL((&url.URL{Scheme: memifMech.SocketFileScheme, Path: listenSocketFilename}).String()) + return p, nil +} + +func (p *proxyListener) accept() { + defer func() { _ = p.Close() }() + for { + in, err := p.listener.Accept() + if optErr, ok := err.(*net.OpError); ok && !optErr.Temporary() { + // TODO - perhaps log this? + return + } + out, err := net.Dial(memifNetwork, p.socketFilename) + if optErr, ok := err.(*net.OpError); ok && !optErr.Temporary() { + _ = in.Close() + // TODO - perhaps log this? + return + } + proxyConn, err := newProxyConnection(in, out) + if err != nil { + _ = in.Close() + _ = out.Close() + // TODO - perhaps log this? + return + } + // TODO - clean up - while 99% of the time this won't be an issue because we will have exactly one thing + // in this list... in principle it could leak memory + p.proxyConnections = append(p.proxyConnections, proxyConn) + } +} + +func (p *proxyListener) Close() error { + if p == nil { + return nil + } + err := p.listener.Close() + for _, proxyConn := range p.proxyConnections { + err = multierror.Append(err, proxyConn.Close()) + } + return err +}