From 9c6939dd504f05c3087df5fd20996a647ee1fee1 Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Sun, 1 Sep 2024 17:32:19 +0300 Subject: [PATCH] feat: support static routes Support optional static routes in the CNI CmdAdd result. Signed-off-by: Fred Rolland --- README.md | 14 ++ api/grpc/nvidia/ipam/node/v1/node.pb.go | 178 +++++++++++++----- api/grpc/proto/nvidia/ipam/node/v1/node.proto | 7 + api/v1alpha1/cidrpool_test.go | 79 ++++++++ api/v1alpha1/cidrpool_type.go | 4 + api/v1alpha1/cidrpool_validate.go | 14 ++ api/v1alpha1/common_type.go | 6 + api/v1alpha1/ippool_test.go | 125 ++++++++++++ api/v1alpha1/ippool_type.go | 4 + api/v1alpha1/ippool_validate.go | 16 ++ api/v1alpha1/validate_routes.go | 61 ++++++ api/v1alpha1/zz_generated.deepcopy.go | 25 +++ deploy/crds/nv-ipam.nvidia.com_cidrpools.yaml | 17 ++ deploy/crds/nv-ipam.nvidia.com_ippools.yaml | 17 ++ pkg/cni/plugin/plugin.go | 9 + pkg/cni/plugin/plugin_test.go | 35 ++++ pkg/ipam-node/allocator/mocks/IPAllocator.go | 11 +- .../controllers/cidrpool/cidrpool.go | 10 +- pkg/ipam-node/controllers/ippool/ippool.go | 10 +- pkg/ipam-node/handlers/allocate.go | 104 ++++++++-- pkg/ipam-node/handlers/handlers_test.go | 86 ++++++++- pkg/pool/reader.go | 19 +- 22 files changed, 764 insertions(+), 87 deletions(-) create mode 100644 api/v1alpha1/validate_routes.go diff --git a/README.md b/README.md index b498336..0ff9211 100644 --- a/README.md +++ b/README.md @@ -350,6 +350,9 @@ spec: - matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists + defautGateway: true # optional + routes: # optional + - dst: 5.5.0.0/24 ``` ##### IPv6 example @@ -383,6 +386,10 @@ spec: * `endIP`: end IP of the exclude range (inclusive). * `nodeSelector` (optional): A list of node selector terms. The terms are ORed. Each term can have a list of matchExpressions that are ANDed. Only the nodes that match the provided labels will get assigned IP Blocks for the defined pool. + * `defautGateway` (optional): Add the pool gateway as default gateway in the pod static routes. + * `routes` (optional, list): contains CIDR to be added in the pod static routes via the pool gateway. + + * `dst`: The destination of the static route, in CIDR notation. > __Notes:__ > @@ -422,6 +429,9 @@ spec: - matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists + defautGateway: true # optional + routes: # optional + - dst: 5.5.0.0/24 ``` ##### IPv6 example @@ -475,6 +485,10 @@ spec: * `prefix`: statically allocated prefix. * `nodeSelector`(optional): A list of node selector terms. The terms are ORed. Each term can have a list of matchExpressions that are ANDed. Only the nodes that match the provided labels will get assigned IP Blocks for the defined pool. + * `defautGateway` (optional): Add the pool gateway as default gateway in the pod static routes. + * `routes` (optional, list): contains CIDR to be added in the pod static routes via the pool gateway. + + * `dst`: The destination of the static route, in CIDR notation. > __Notes:__ > diff --git a/api/grpc/nvidia/ipam/node/v1/node.pb.go b/api/grpc/nvidia/ipam/node/v1/node.pb.go index 0ebada8..aa626ef 100644 --- a/api/grpc/nvidia/ipam/node/v1/node.pb.go +++ b/api/grpc/nvidia/ipam/node/v1/node.pb.go @@ -520,6 +520,8 @@ type AllocationInfo struct { Gateway string `protobuf:"bytes,3,opt,name=gateway,proto3" json:"gateway,omitempty"` // type of the pool which is refered by the name in the pools field PoolType PoolType `protobuf:"varint,4,opt,name=pool_type,json=poolType,proto3,enum=nvidia.ipam.node.v1.PoolType" json:"pool_type,omitempty"` + // list of static routes + Routes []*Route `protobuf:"bytes,5,rep,name=routes,proto3" json:"routes,omitempty"` } func (x *AllocationInfo) Reset() { @@ -582,6 +584,61 @@ func (x *AllocationInfo) GetPoolType() PoolType { return PoolType_POOL_TYPE_UNSPECIFIED } +func (x *AllocationInfo) GetRoutes() []*Route { + if x != nil { + return x.Routes + } + return nil +} + +type Route struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Static route destination in CIDR format + Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"` +} + +func (x *Route) Reset() { + *x = Route{} + if protoimpl.UnsafeEnabled { + mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Route) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Route) ProtoMessage() {} + +func (x *Route) ProtoReflect() protoreflect.Message { + mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Route.ProtoReflect.Descriptor instead. +func (*Route) Descriptor() ([]byte, []int) { + return file_nvidia_ipam_node_v1_node_proto_rawDescGZIP(), []int{8} +} + +func (x *Route) GetDest() string { + if x != nil { + return x.Dest + } + return "" +} + // IsAllocatedReply contains reply for IsAllocated rpc call type IsAllocatedResponse struct { state protoimpl.MessageState @@ -592,7 +649,7 @@ type IsAllocatedResponse struct { func (x *IsAllocatedResponse) Reset() { *x = IsAllocatedResponse{} if protoimpl.UnsafeEnabled { - mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[8] + mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -605,7 +662,7 @@ func (x *IsAllocatedResponse) String() string { func (*IsAllocatedResponse) ProtoMessage() {} func (x *IsAllocatedResponse) ProtoReflect() protoreflect.Message { - mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[8] + mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -618,7 +675,7 @@ func (x *IsAllocatedResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use IsAllocatedResponse.ProtoReflect.Descriptor instead. func (*IsAllocatedResponse) Descriptor() ([]byte, []int) { - return file_nvidia_ipam_node_v1_node_proto_rawDescGZIP(), []int{8} + return file_nvidia_ipam_node_v1_node_proto_rawDescGZIP(), []int{9} } // DeallocateReply contains reply for Deallocate rpc call @@ -631,7 +688,7 @@ type DeallocateResponse struct { func (x *DeallocateResponse) Reset() { *x = DeallocateResponse{} if protoimpl.UnsafeEnabled { - mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[9] + mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -644,7 +701,7 @@ func (x *DeallocateResponse) String() string { func (*DeallocateResponse) ProtoMessage() {} func (x *DeallocateResponse) ProtoReflect() protoreflect.Message { - mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[9] + mi := &file_nvidia_ipam_node_v1_node_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -657,7 +714,7 @@ func (x *DeallocateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use DeallocateResponse.ProtoReflect.Descriptor instead. func (*DeallocateResponse) Descriptor() ([]byte, []int) { - return file_nvidia_ipam_node_v1_node_proto_rawDescGZIP(), []int{9} + return file_nvidia_ipam_node_v1_node_proto_rawDescGZIP(), []int{10} } var File_nvidia_ipam_node_v1_node_proto protoreflect.FileDescriptor @@ -724,7 +781,7 @@ var file_nvidia_ipam_node_v1_node_proto_rawDesc = []byte{ 0x0b, 0x32, 0x23, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0b, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x22, 0x8a, 0x01, 0x0a, 0x0e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x0e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x6f, 0x6c, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x12, 0x18, 0x0a, 0x07, 0x67, 0x61, @@ -733,34 +790,39 @@ var file_nvidia_ipam_node_v1_node_proto_rawDesc = []byte{ 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, - 0x22, 0x15, 0x0a, 0x13, 0x49, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x61, 0x6c, 0x6c, - 0x6f, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x53, 0x0a, - 0x08, 0x50, 0x6f, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x4f, 0x4f, - 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, - 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x49, 0x50, 0x50, 0x4f, 0x4f, 0x4c, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x50, 0x4f, - 0x4f, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x44, 0x52, 0x50, 0x4f, 0x4f, 0x4c, - 0x10, 0x02, 0x32, 0xad, 0x02, 0x0a, 0x0b, 0x49, 0x50, 0x41, 0x4d, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x12, 0x59, 0x0a, 0x08, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x12, 0x24, - 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, - 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, - 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, - 0x0b, 0x49, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x27, 0x2e, 0x6e, - 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x49, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, - 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x73, 0x41, 0x6c, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x5f, 0x0a, 0x0a, 0x44, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x12, - 0x26, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, - 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, - 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, - 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x32, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, + 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x22, 0x1b, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x64, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x65, 0x73, + 0x74, 0x22, 0x15, 0x0a, 0x13, 0x49, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x14, 0x0a, 0x12, 0x44, 0x65, 0x61, 0x6c, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x53, + 0x0a, 0x08, 0x50, 0x6f, 0x6f, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x19, 0x0a, 0x15, 0x50, 0x4f, + 0x4f, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x50, 0x4f, 0x4f, 0x4c, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x49, 0x50, 0x50, 0x4f, 0x4f, 0x4c, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x50, + 0x4f, 0x4f, 0x4c, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x44, 0x52, 0x50, 0x4f, 0x4f, + 0x4c, 0x10, 0x02, 0x32, 0xad, 0x02, 0x0a, 0x0b, 0x49, 0x50, 0x41, 0x4d, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x12, 0x59, 0x0a, 0x08, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x12, + 0x24, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, + 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, + 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x6c, 0x6c, 0x6f, + 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, + 0x0a, 0x0b, 0x49, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x27, 0x2e, + 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x73, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, + 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x73, 0x41, + 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x5f, 0x0a, 0x0a, 0x44, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, + 0x12, 0x26, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, + 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x6e, 0x76, 0x69, 0x64, 0x69, + 0x61, 0x2e, 0x69, 0x70, 0x61, 0x6d, 0x2e, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, + 0x65, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -776,7 +838,7 @@ func file_nvidia_ipam_node_v1_node_proto_rawDescGZIP() []byte { } var file_nvidia_ipam_node_v1_node_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_nvidia_ipam_node_v1_node_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_nvidia_ipam_node_v1_node_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_nvidia_ipam_node_v1_node_proto_goTypes = []interface{}{ (PoolType)(0), // 0: nvidia.ipam.node.v1.PoolType (*AllocateRequest)(nil), // 1: nvidia.ipam.node.v1.AllocateRequest @@ -787,8 +849,9 @@ var file_nvidia_ipam_node_v1_node_proto_goTypes = []interface{}{ (*DeallocateRequest)(nil), // 6: nvidia.ipam.node.v1.DeallocateRequest (*AllocateResponse)(nil), // 7: nvidia.ipam.node.v1.AllocateResponse (*AllocationInfo)(nil), // 8: nvidia.ipam.node.v1.AllocationInfo - (*IsAllocatedResponse)(nil), // 9: nvidia.ipam.node.v1.IsAllocatedResponse - (*DeallocateResponse)(nil), // 10: nvidia.ipam.node.v1.DeallocateResponse + (*Route)(nil), // 9: nvidia.ipam.node.v1.Route + (*IsAllocatedResponse)(nil), // 10: nvidia.ipam.node.v1.IsAllocatedResponse + (*DeallocateResponse)(nil), // 11: nvidia.ipam.node.v1.DeallocateResponse } var file_nvidia_ipam_node_v1_node_proto_depIdxs = []int32{ 2, // 0: nvidia.ipam.node.v1.AllocateRequest.parameters:type_name -> nvidia.ipam.node.v1.IPAMParameters @@ -799,17 +862,18 @@ var file_nvidia_ipam_node_v1_node_proto_depIdxs = []int32{ 2, // 5: nvidia.ipam.node.v1.DeallocateRequest.parameters:type_name -> nvidia.ipam.node.v1.IPAMParameters 8, // 6: nvidia.ipam.node.v1.AllocateResponse.allocations:type_name -> nvidia.ipam.node.v1.AllocationInfo 0, // 7: nvidia.ipam.node.v1.AllocationInfo.pool_type:type_name -> nvidia.ipam.node.v1.PoolType - 1, // 8: nvidia.ipam.node.v1.IPAMService.Allocate:input_type -> nvidia.ipam.node.v1.AllocateRequest - 5, // 9: nvidia.ipam.node.v1.IPAMService.IsAllocated:input_type -> nvidia.ipam.node.v1.IsAllocatedRequest - 6, // 10: nvidia.ipam.node.v1.IPAMService.Deallocate:input_type -> nvidia.ipam.node.v1.DeallocateRequest - 7, // 11: nvidia.ipam.node.v1.IPAMService.Allocate:output_type -> nvidia.ipam.node.v1.AllocateResponse - 9, // 12: nvidia.ipam.node.v1.IPAMService.IsAllocated:output_type -> nvidia.ipam.node.v1.IsAllocatedResponse - 10, // 13: nvidia.ipam.node.v1.IPAMService.Deallocate:output_type -> nvidia.ipam.node.v1.DeallocateResponse - 11, // [11:14] is the sub-list for method output_type - 8, // [8:11] is the sub-list for method input_type - 8, // [8:8] is the sub-list for extension type_name - 8, // [8:8] is the sub-list for extension extendee - 0, // [0:8] is the sub-list for field type_name + 9, // 8: nvidia.ipam.node.v1.AllocationInfo.routes:type_name -> nvidia.ipam.node.v1.Route + 1, // 9: nvidia.ipam.node.v1.IPAMService.Allocate:input_type -> nvidia.ipam.node.v1.AllocateRequest + 5, // 10: nvidia.ipam.node.v1.IPAMService.IsAllocated:input_type -> nvidia.ipam.node.v1.IsAllocatedRequest + 6, // 11: nvidia.ipam.node.v1.IPAMService.Deallocate:input_type -> nvidia.ipam.node.v1.DeallocateRequest + 7, // 12: nvidia.ipam.node.v1.IPAMService.Allocate:output_type -> nvidia.ipam.node.v1.AllocateResponse + 10, // 13: nvidia.ipam.node.v1.IPAMService.IsAllocated:output_type -> nvidia.ipam.node.v1.IsAllocatedResponse + 11, // 14: nvidia.ipam.node.v1.IPAMService.Deallocate:output_type -> nvidia.ipam.node.v1.DeallocateResponse + 12, // [12:15] is the sub-list for method output_type + 9, // [9:12] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_nvidia_ipam_node_v1_node_proto_init() } @@ -915,7 +979,7 @@ func file_nvidia_ipam_node_v1_node_proto_init() { } } file_nvidia_ipam_node_v1_node_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IsAllocatedResponse); i { + switch v := v.(*Route); i { case 0: return &v.state case 1: @@ -927,6 +991,18 @@ func file_nvidia_ipam_node_v1_node_proto_init() { } } file_nvidia_ipam_node_v1_node_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IsAllocatedResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_nvidia_ipam_node_v1_node_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DeallocateResponse); i { case 0: return &v.state @@ -945,7 +1021,7 @@ func file_nvidia_ipam_node_v1_node_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_nvidia_ipam_node_v1_node_proto_rawDesc, NumEnums: 1, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 1, }, diff --git a/api/grpc/proto/nvidia/ipam/node/v1/node.proto b/api/grpc/proto/nvidia/ipam/node/v1/node.proto index 1c27a2a..e6229ee 100644 --- a/api/grpc/proto/nvidia/ipam/node/v1/node.proto +++ b/api/grpc/proto/nvidia/ipam/node/v1/node.proto @@ -125,6 +125,13 @@ message AllocationInfo { string gateway = 3; // type of the pool which is refered by the name in the pools field PoolType pool_type = 4; + // list of static routes + repeated Route routes = 5; +} + +message Route { + // Static route destination in CIDR format + string dest = 1; } // IsAllocatedReply contains reply for IsAllocated rpc call diff --git a/api/v1alpha1/cidrpool_test.go b/api/v1alpha1/cidrpool_test.go index a80fecc..7a7284f 100644 --- a/api/v1alpha1/cidrpool_test.go +++ b/api/v1alpha1/cidrpool_test.go @@ -94,6 +94,85 @@ var _ = Describe("CIDRPool", func() { } validatePoolAndCheckErr(&cidrPool, true) }) + It("Invalid - no gatewayIndex, defaultGateway true", func() { + cidrPool := v1alpha1.CIDRPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.CIDRPoolSpec{ + CIDR: "fdf8:6aef:d1fe::/48", + PerNodeNetworkPrefix: 120, + DefaultGateway: true, + }, + } + validatePoolAndCheckErr(&cidrPool, false, ContainSubstring("spec.defaultGateway")) + }) + It("Valid - routes", func() { + cidrPool := v1alpha1.CIDRPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.CIDRPoolSpec{ + CIDR: "192.168.0.0/16", + PerNodeNetworkPrefix: 24, + GatewayIndex: ptr.To[int32](100), + Routes: []v1alpha1.Route{ + { + Dst: "5.5.0.0/16", + }, + { + Dst: "10.7.1.0/24", + }, + }, + }, + } + validatePoolAndCheckErr(&cidrPool, true) + }) + It("Invalid - routes not CIDR", func() { + cidrPool := v1alpha1.CIDRPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.CIDRPoolSpec{ + CIDR: "192.168.0.0/16", + PerNodeNetworkPrefix: 24, + GatewayIndex: ptr.To[int32](100), + Routes: []v1alpha1.Route{ + { + Dst: "5.5.0.0", + }, + }, + }, + } + validatePoolAndCheckErr(&cidrPool, false, ContainSubstring("spec.routes")) + }) + It("Invalid - routes without GatewayIndex", func() { + cidrPool := v1alpha1.CIDRPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.CIDRPoolSpec{ + CIDR: "192.168.0.0/16", + PerNodeNetworkPrefix: 24, + Routes: []v1alpha1.Route{ + { + Dst: "5.5.0.0", + }, + }, + }, + } + validatePoolAndCheckErr(&cidrPool, false, ContainSubstring("spec.routes")) + }) + It("Invalid - routes not same address family", func() { + ipPool := v1alpha1.CIDRPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.CIDRPoolSpec{ + CIDR: "192.168.0.0/16", + PerNodeNetworkPrefix: 24, + Routes: []v1alpha1.Route{ + { + Dst: "2001:db8:3333:4444::0/64", + }, + }, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.routes"), + ) + }) DescribeTable("CIDR", func(cidr string, prefix int32, isValid bool) { cidrPool := v1alpha1.CIDRPool{ diff --git a/api/v1alpha1/cidrpool_type.go b/api/v1alpha1/cidrpool_type.go index 84abebc..169e767 100644 --- a/api/v1alpha1/cidrpool_type.go +++ b/api/v1alpha1/cidrpool_type.go @@ -48,6 +48,10 @@ type CIDRPoolSpec struct { StaticAllocations []CIDRPoolStaticAllocation `json:"staticAllocations,omitempty"` // selector for nodes, if empty match all nodes NodeSelector *corev1.NodeSelector `json:"nodeSelector,omitempty"` + // if true, add gateway as default gateway in the routes list + DefaultGateway bool `json:"defautGateway,omitempty"` + // static routes list. The gateway used will according to the node allocation. + Routes []Route `json:"routes,omitempty"` } // CIDRPoolStatus contains the IP prefixes allocated to nodes diff --git a/api/v1alpha1/cidrpool_validate.go b/api/v1alpha1/cidrpool_validate.go index 27143c4..aa4a895 100644 --- a/api/v1alpha1/cidrpool_validate.go +++ b/api/v1alpha1/cidrpool_validate.go @@ -33,6 +33,20 @@ func (r *CIDRPool) Validate() field.ErrorList { if r.Spec.NodeSelector != nil { errList = append(errList, validateNodeSelector(r.Spec.NodeSelector, field.NewPath("spec"))...) } + if r.Spec.GatewayIndex == nil { + if r.Spec.DefaultGateway { + errList = append(errList, field.Invalid( + field.NewPath("spec", "defaultGateway"), r.Spec.DefaultGateway, + "cannot be true if spec.gatewayIndex is not set")) + } + if len(r.Spec.Routes) > 0 { + errList = append(errList, field.Invalid( + field.NewPath("spec", "routes"), r.Spec.Routes, + "cannot be set if spec.gatewayIndex is not set")) + } + } + _, network, _ := net.ParseCIDR(r.Spec.CIDR) + errList = append(errList, validateRoutes(r.Spec.Routes, network, r.Spec.DefaultGateway, field.NewPath("spec"))...) return errList } diff --git a/api/v1alpha1/common_type.go b/api/v1alpha1/common_type.go index 5b9fb96..4e78a65 100644 --- a/api/v1alpha1/common_type.go +++ b/api/v1alpha1/common_type.go @@ -19,3 +19,9 @@ type ExcludeRange struct { StartIP string `json:"startIP"` EndIP string `json:"endIP"` } + +// Route contains static route parameters +type Route struct { + // The destination of the route, in CIDR notation + Dst string `json:"dst"` +} diff --git a/api/v1alpha1/ippool_test.go b/api/v1alpha1/ippool_test.go index c5ecf94..7061c6c 100644 --- a/api/v1alpha1/ippool_test.go +++ b/api/v1alpha1/ippool_test.go @@ -156,4 +156,129 @@ var _ = Describe("Validate", func() { ContainSubstring("spec.nodeSelector"), ) }) + It("Invalid - no Gateway, defaultGateway true", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + DefaultGateway: true, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.defaultGateway"), + ) + }) + It("Valid - routes", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Gateway: "192.168.0.1", + Routes: []v1alpha1.Route{ + { + Dst: "5.5.0.0/16", + }, + { + Dst: "10.7.1.0/24", + }, + }, + }, + } + Expect(ipPool.Validate()).To(BeEmpty()) + }) + It("Invalid - routes without Gateway", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Routes: []v1alpha1.Route{ + { + Dst: "5.5.0.0/16", + }, + }, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.routes"), + ) + }) + It("Invalid - routes not CIDR", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Routes: []v1alpha1.Route{ + { + Dst: "5.5.0.0", + }, + }, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.routes"), + ) + }) + It("Invalid - routes not same address family", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + Routes: []v1alpha1.Route{ + { + Dst: "2001:db8:3333:4444::0/64", + }, + }, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.routes"), + ) + }) + It("Invalid - default routes with defaultGateway true - IPv6", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "2001:db8:3333:4444::0/64", + PerNodeBlockSize: 128, + DefaultGateway: true, + Routes: []v1alpha1.Route{ + { + Dst: "::/0", + }, + }, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.routes"), + ) + }) + It("Invalid - default routes with defaultGateway true - IPv4", func() { + ipPool := v1alpha1.IPPool{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: v1alpha1.IPPoolSpec{ + Subnet: "192.168.0.0/16", + PerNodeBlockSize: 128, + DefaultGateway: true, + Routes: []v1alpha1.Route{ + { + Dst: "0.0.0.0/0", + }, + }, + }, + } + Expect(ipPool.Validate().ToAggregate().Error()). + To( + ContainSubstring("spec.routes"), + ) + }) }) diff --git a/api/v1alpha1/ippool_type.go b/api/v1alpha1/ippool_type.go index 4a07738..8fb088b 100644 --- a/api/v1alpha1/ippool_type.go +++ b/api/v1alpha1/ippool_type.go @@ -45,6 +45,10 @@ type IPPoolSpec struct { Gateway string `json:"gateway,omitempty"` // selector for nodes, if empty match all nodes NodeSelector *corev1.NodeSelector `json:"nodeSelector,omitempty"` + // if true, add gateway as default gateway in the routes list + DefaultGateway bool `json:"defautGateway,omitempty"` + // static routes list using the gateway specified in the spec. + Routes []Route `json:"routes,omitempty"` } // IPPoolStatus contains the IP ranges allocated to nodes diff --git a/api/v1alpha1/ippool_validate.go b/api/v1alpha1/ippool_validate.go index 86baaf2..8d27866 100644 --- a/api/v1alpha1/ippool_validate.go +++ b/api/v1alpha1/ippool_validate.go @@ -71,5 +71,21 @@ func (r *IPPool) Validate() field.ErrorList { if r.Spec.NodeSelector != nil { errList = append(errList, validateNodeSelector(r.Spec.NodeSelector, field.NewPath("spec"))...) } + + if r.Spec.Gateway == "" { + if r.Spec.DefaultGateway { + errList = append(errList, field.Invalid( + field.NewPath("spec", "defaultGateway"), r.Spec.DefaultGateway, + "cannot be true if spec.gateway is not set")) + } + if len(r.Spec.Routes) > 0 { + errList = append(errList, field.Invalid( + field.NewPath("spec", "routes"), r.Spec.Routes, + "cannot be set if spec.gateway is not set")) + } + } + + errList = append(errList, validateRoutes(r.Spec.Routes, network, r.Spec.DefaultGateway, field.NewPath("spec"))...) + return errList } diff --git a/api/v1alpha1/validate_routes.go b/api/v1alpha1/validate_routes.go new file mode 100644 index 0000000..8f15860 --- /dev/null +++ b/api/v1alpha1/validate_routes.go @@ -0,0 +1,61 @@ +/* + Copyright 2024, NVIDIA CORPORATION & AFFILIATES + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package v1alpha1 + +import ( + "net" + + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// validateRoutes validate routes: +// - dst is a valid CIDR +func validateRoutes(routes []Route, network *net.IPNet, defaultGateway bool, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + for i, r := range routes { + _, routeNet, err := net.ParseCIDR(r.Dst) + if err != nil { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("routes").Index(i).Child("dst"), r.Dst, + "is invalid subnet")) + } + if routeNet != nil && network != nil { + if (routeNet.IP.To4() != nil) != (network.IP.To4() != nil) { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("routes").Index(i).Child("dst"), r.Dst, + "is not same address family IPv4/IPv6")) + } + } + if routeNet != nil && defaultGateway { + if isDefaultRoute(routeNet) { + allErrs = append(allErrs, field.Invalid( + fldPath.Child("routes").Index(i).Child("dst"), r.Dst, + "default route is not allowed if 'defaultGateway' is true")) + } + } + } + return allErrs +} + +func isDefaultRoute(ipNet *net.IPNet) bool { + // Check if it's IPv4 and matches 0.0.0.0/0 + if ipNet.IP.To4() != nil && ipNet.String() == "0.0.0.0/0" { + return true + } + // Check if it's IPv6 and matches ::/0 + if ipNet.IP.To4() == nil && ipNet.String() == "::/0" { + return true + } + return false +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1e15c94..7428a8d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -134,6 +134,11 @@ func (in *CIDRPoolSpec) DeepCopyInto(out *CIDRPoolSpec) { *out = new(v1.NodeSelector) (*in).DeepCopyInto(*out) } + if in.Routes != nil { + in, out := &in.Routes, &out.Routes + *out = make([]Route, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CIDRPoolSpec. @@ -268,6 +273,11 @@ func (in *IPPoolSpec) DeepCopyInto(out *IPPoolSpec) { *out = new(v1.NodeSelector) (*in).DeepCopyInto(*out) } + if in.Routes != nil { + in, out := &in.Routes, &out.Routes + *out = make([]Route, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IPPoolSpec. @@ -299,3 +309,18 @@ func (in *IPPoolStatus) DeepCopy() *IPPoolStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Route) DeepCopyInto(out *Route) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Route. +func (in *Route) DeepCopy() *Route { + if in == nil { + return nil + } + out := new(Route) + in.DeepCopyInto(out) + return out +} diff --git a/deploy/crds/nv-ipam.nvidia.com_cidrpools.yaml b/deploy/crds/nv-ipam.nvidia.com_cidrpools.yaml index 5cdd79d..4da6532 100644 --- a/deploy/crds/nv-ipam.nvidia.com_cidrpools.yaml +++ b/deploy/crds/nv-ipam.nvidia.com_cidrpools.yaml @@ -49,6 +49,10 @@ spec: is define in perNodeNetworkPrefix) and distributed between matching nodes type: string + defautGateway: + description: if true, add gateway as default gateway in the routes + list + type: boolean exclusions: description: contains reserved IP addresses that should not be allocated by nv-ipam @@ -160,6 +164,19 @@ spec: this size. format: int32 type: integer + routes: + description: static routes list. The gateway used will according to + the node allocation. + items: + description: Route contains static route parameters + properties: + dst: + description: The destination of the route, in CIDR notation + type: string + required: + - dst + type: object + type: array staticAllocations: description: static allocations for the pool items: diff --git a/deploy/crds/nv-ipam.nvidia.com_ippools.yaml b/deploy/crds/nv-ipam.nvidia.com_ippools.yaml index 8f73890..f0b7854 100644 --- a/deploy/crds/nv-ipam.nvidia.com_ippools.yaml +++ b/deploy/crds/nv-ipam.nvidia.com_ippools.yaml @@ -44,6 +44,10 @@ spec: spec: description: IPPoolSpec contains configuration for IP pool properties: + defautGateway: + description: if true, add gateway as default gateway in the routes + list + type: boolean exclusions: description: contains reserved IP addresses that should not be allocated by nv-ipam @@ -151,6 +155,19 @@ spec: description: amount of IPs to allocate for each node, must be less than amount of available IPs in the subnet type: integer + routes: + description: static routes list using the gateway specified in the + spec. + items: + description: Route contains static route parameters + properties: + dst: + description: The destination of the route, in CIDR notation + type: string + required: + - dst + type: object + type: array subnet: description: subnet of the pool type: string diff --git a/pkg/cni/plugin/plugin.go b/pkg/cni/plugin/plugin.go index 01b7c8e..c8b9f99 100644 --- a/pkg/cni/plugin/plugin.go +++ b/pkg/cni/plugin/plugin.go @@ -182,6 +182,15 @@ func grpcRespToResult(resp *nodev1.AllocateResponse) (*current.Result, error) { return nil, logErr(fmt.Sprintf("unexpected Gateway address format, received value: %s", alloc.Gateway)) } ipConfig.Gateway = gwIP + for _, r := range alloc.Routes { + _, ipNet, err := net.ParseCIDR(r.Dest) + if err != nil { + return nil, logErr(fmt.Sprintf("unexpected Route destination format, received value: %s, error: %s", + r.Dest, err.Error())) + } + route := &cnitypes.Route{Dst: *ipNet, GW: ipConfig.Gateway} + result.Routes = append(result.Routes, route) + } } result.IPs = append(result.IPs, ipConfig) diff --git a/pkg/cni/plugin/plugin_test.go b/pkg/cni/plugin/plugin_test.go index 6dc1091..c83a7f7 100644 --- a/pkg/cni/plugin/plugin_test.go +++ b/pkg/cni/plugin/plugin_test.go @@ -107,11 +107,46 @@ var _ = Describe("plugin tests", func() { Ip: "192.168.1.10/24", Gateway: "192.168.1.1", PoolType: nodev1.PoolType_POOL_TYPE_IPPOOL, + Routes: []*nodev1.Route{ + { + Dest: "5.5.0.0/16", + }, + }, }}, }, nil) err := p.CmdAdd(args) Expect(err).ToNot(HaveOccurred()) }) + It("executes failed - non cidr route", func() { + mockConfLoader.On("LoadConf", args).Return(testConf, nil) + mockDaemonClient.On("Allocate", mock.Anything, &nodev1.AllocateRequest{ + Parameters: &nodev1.IPAMParameters{ + Pools: []string{"my-pool"}, + PoolType: nodev1.PoolType_POOL_TYPE_IPPOOL, + CniIfname: "net1", + CniContainerid: "1234", + Metadata: &nodev1.IPAMMetadata{ + K8SPodName: "test", + K8SPodNamespace: "test", + }, + RequestedIps: []string{}, + Features: &nodev1.IPAMFeatures{}, + }}).Return(&nodev1.AllocateResponse{ + Allocations: []*nodev1.AllocationInfo{{ + Pool: "my-pool", + Ip: "192.168.1.10/24", + Gateway: "192.168.1.1", + PoolType: nodev1.PoolType_POOL_TYPE_IPPOOL, + Routes: []*nodev1.Route{ + { + Dest: "5.5.0.0", + }, + }, + }}, + }, nil) + err := p.CmdAdd(args) + Expect(err).To(HaveOccurred()) + }) }) Context("CmdDel()", func() { diff --git a/pkg/ipam-node/allocator/mocks/IPAllocator.go b/pkg/ipam-node/allocator/mocks/IPAllocator.go index a55d0f1..10df300 100644 --- a/pkg/ipam-node/allocator/mocks/IPAllocator.go +++ b/pkg/ipam-node/allocator/mocks/IPAllocator.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.37.1. DO NOT EDIT. +// Code generated by mockery v2.27.1. DO NOT EDIT. package mocks @@ -82,12 +82,13 @@ func (_c *IPAllocator_Allocate_Call) RunAndReturn(run func(string, string, types return _c } -// NewIPAllocator creates a new instance of IPAllocator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewIPAllocator(t interface { +type mockConstructorTestingTNewIPAllocator interface { mock.TestingT Cleanup(func()) -}) *IPAllocator { +} + +// NewIPAllocator creates a new instance of IPAllocator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewIPAllocator(t mockConstructorTestingTNewIPAllocator) *IPAllocator { mock := &IPAllocator{} mock.Mock.Test(t) diff --git a/pkg/ipam-node/controllers/cidrpool/cidrpool.go b/pkg/ipam-node/controllers/cidrpool/cidrpool.go index de825b6..cb5c92f 100644 --- a/pkg/ipam-node/controllers/cidrpool/cidrpool.go +++ b/pkg/ipam-node/controllers/cidrpool/cidrpool.go @@ -76,16 +76,22 @@ func (r *CIDRPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c startIP = nodeSubnet.IP } endIP := ip.LastIP(nodeSubnet) - cidrPool := &pool.Pool{ + routes := make([]pool.Route, 0, len(cidrPool.Spec.Routes)) + for _, r := range cidrPool.Spec.Routes { + routes = append(routes, pool.Route{Dst: r.Dst}) + } + pool := &pool.Pool{ Name: cidrPool.Name, Subnet: alloc.Prefix, Gateway: alloc.Gateway, StartIP: startIP.String(), EndIP: endIP.String(), Exclusions: buildExclusions(cidrPool.Spec.Exclusions, nodeSubnet, startIP, endIP), + Routes: routes, } + pool.DefaultGateway = cidrPool.Spec.DefaultGateway reqLog.Info("CIDRPool config updated", "name", cidrPool.Name) - r.PoolManager.UpdatePool(poolKey, cidrPool) + r.PoolManager.UpdatePool(poolKey, pool) found = true break } diff --git a/pkg/ipam-node/controllers/ippool/ippool.go b/pkg/ipam-node/controllers/ippool/ippool.go index de5ce03..d505d13 100644 --- a/pkg/ipam-node/controllers/ippool/ippool.go +++ b/pkg/ipam-node/controllers/ippool/ippool.go @@ -58,15 +58,21 @@ func (r *IPPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr for _, e := range ipPool.Spec.Exclusions { exclusions = append(exclusions, pool.ExclusionRange{StartIP: e.StartIP, EndIP: e.EndIP}) } - ipPool := &pool.Pool{ + routes := make([]pool.Route, 0, len(ipPool.Spec.Routes)) + for _, r := range ipPool.Spec.Routes { + routes = append(routes, pool.Route{Dst: r.Dst}) + } + pool := &pool.Pool{ Name: ipPool.Name, Subnet: ipPool.Spec.Subnet, Gateway: ipPool.Spec.Gateway, StartIP: alloc.StartIP, EndIP: alloc.EndIP, Exclusions: exclusions, + Routes: routes, } - r.PoolManager.UpdatePool(poolKey, ipPool) + pool.DefaultGateway = ipPool.Spec.DefaultGateway + r.PoolManager.UpdatePool(poolKey, pool) found = true break } diff --git a/pkg/ipam-node/handlers/allocate.go b/pkg/ipam-node/handlers/allocate.go index 19570ee..9a7f712 100644 --- a/pkg/ipam-node/handlers/allocate.go +++ b/pkg/ipam-node/handlers/allocate.go @@ -14,10 +14,12 @@ package handlers import ( + "bytes" "context" "errors" "fmt" "net" + "slices" "time" cniTypes "github.com/containernetworking/cni/pkg/types" @@ -31,6 +33,7 @@ import ( "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/allocator" storePkg "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/store" "github.com/Mellanox/nvidia-k8s-ipam/pkg/ipam-node/types" + "github.com/Mellanox/nvidia-k8s-ipam/pkg/pool" ) // Allocate is the handler for Allocate GRPC endpoint @@ -62,6 +65,11 @@ func (h *Handlers) Allocate(ctx context.Context, req *nodev1.AllocateRequest) (* } if r.Gateway != nil { allocationInfo.Gateway = r.Gateway.String() + routes := make([]*nodev1.Route, 0) + for _, route := range r.Routes { + routes = append(routes, &nodev1.Route{Dest: route.String()}) + } + allocationInfo.Routes = routes } resp.Allocations = append(resp.Allocations, allocationInfo) } @@ -72,6 +80,7 @@ func (h *Handlers) Allocate(ctx context.Context, req *nodev1.AllocateRequest) (* type PoolAlloc struct { Pool string *current.IPConfig + Routes []net.IPNet } func (h *Handlers) allocate(reqLog logr.Logger, @@ -191,38 +200,99 @@ func (h *Handlers) allocateInPool(poolName string, reqLog logr.Logger, result, err := alloc.Allocate(params.CniContainerid, params.CniIfname, allocMeta, selectedStaticIP) if err != nil { poolLog.Error(err, "failed to allocate IP address") - if errors.Is(err, storePkg.ErrReservationAlreadyExist) { - return PoolAlloc{}, status.Errorf(codes.AlreadyExists, - "allocation already exist in the pool \"%s\", poolType \"%s\"", poolName, poolType) - } - if errors.Is(err, allocator.ErrNoFreeAddresses) { - return PoolAlloc{}, status.Errorf(codes.ResourceExhausted, - "no free addresses in the pool \"%s\", poolType \"%s\"", - poolName, poolType) - } - if errors.Is(err, storePkg.ErrIPAlreadyReserved) { - return PoolAlloc{}, status.Errorf(codes.ResourceExhausted, - "requested IP is already reserved in the pool \"%s\", poolType \"%s\"", - poolName, poolType) - } - return PoolAlloc{}, status.Errorf(codes.Internal, - "failed to allocate IP address in pool \"%s\", poolType \"%s\"", poolName, poolType) + return PoolAlloc{}, specificError(err, poolName, poolType) } if params.Features != nil && params.Features.AllocateDefaultGateway { // TODO (ykulazhenkov): do we want to keep gateway in this case? // if we will return gateway here, the container will have same IP as interface address and as gateway result.Gateway = nil } - + routes, err := getRoutes(result.Gateway, poolCfg.Routes, poolCfg.DefaultGateway) + if err != nil { + return PoolAlloc{}, err + } poolLog.Info("IP address allocated", "allocation", result.String()) return PoolAlloc{ Pool: poolName, IPConfig: result, + Routes: routes, }, nil } +func specificError(err error, poolName string, poolType string) error { + if errors.Is(err, storePkg.ErrReservationAlreadyExist) { + return status.Errorf(codes.AlreadyExists, + "allocation already exist in the pool \"%s\", poolType \"%s\"", poolName, poolType) + } + if errors.Is(err, allocator.ErrNoFreeAddresses) { + return status.Errorf(codes.ResourceExhausted, + "no free addresses in the pool \"%s\", poolType \"%s\"", + poolName, poolType) + } + if errors.Is(err, storePkg.ErrIPAlreadyReserved) { + return status.Errorf(codes.ResourceExhausted, + "requested IP is already reserved in the pool \"%s\", poolType \"%s\"", + poolName, poolType) + } + return status.Errorf(codes.Internal, + "failed to allocate IP address in pool \"%s\", poolType \"%s\"", poolName, poolType) +} + func poolCfgError(reqLog logr.Logger, pool, poolType, reason string) error { reqLog.Error(nil, "invalid pool config", "pool", pool, "poolType", poolType, "reason", reason) return status.Errorf(codes.Internal, "invalid config for pool \"%s\", poolType \"%s\"", pool, poolType) } + +func getRoutes(gateway net.IP, routesCfg []pool.Route, defaultGateway bool) ([]net.IPNet, error) { + routes := make([]net.IPNet, 0) + if gateway == nil { + return routes, nil + } + for _, r := range routesCfg { + _, ipNet, err := net.ParseCIDR(r.Dst) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, + "unexpected Route destination format, received value: %s, error: %s", + r.Dst, err.Error()) + } + routes = append(routes, *ipNet) + } + if defaultGateway { + routes = append(routes, createDefaultRoute(gateway)) + } + + return sortAndDedupIPNets(routes), nil +} + +func createDefaultRoute(gateway net.IP) net.IPNet { + if gateway.To4() != nil { + // IPv4 default route + return net.IPNet{ + IP: net.IPv4(0, 0, 0, 0), + Mask: net.CIDRMask(0, 32), // /0 for IPv4 + } + } + return net.IPNet{ + IP: net.IPv6zero, // :: (IPv6 default) + Mask: net.CIDRMask(0, 128), // /0 for IPv6 + } +} + +// Function to sort and remove duplicates from a slice of net.IPNet +func sortAndDedupIPNets(slice []net.IPNet) []net.IPNet { + if len(slice) == 0 { + return slice + } + + slices.SortFunc(slice, func(a, b net.IPNet) int { + if !a.IP.Equal(b.IP) { + return bytes.Compare(a.IP, b.IP) // Compare IP addresses + } + return bytes.Compare(a.Mask, b.Mask) // Compare subnet masks if IPs are equal + }) + + return slices.CompactFunc(slice, func(a, b net.IPNet) bool { + return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask) + }) +} diff --git a/pkg/ipam-node/handlers/handlers_test.go b/pkg/ipam-node/handlers/handlers_test.go index 6ace033..e1397a9 100644 --- a/pkg/ipam-node/handlers/handlers_test.go +++ b/pkg/ipam-node/handlers/handlers_test.go @@ -55,10 +55,16 @@ func getPoolConfigs() map[string]*pool.Pool { Gateway: "192.168.0.1", }, testPoolName2: {Name: testPoolName2, - Subnet: "10.100.0.0/16", - StartIP: "10.100.0.2", - EndIP: "10.100.0.254", - Gateway: "10.100.0.1", + Subnet: "10.100.0.0/16", + StartIP: "10.100.0.2", + EndIP: "10.100.0.254", + Gateway: "10.100.0.1", + DefaultGateway: true, + Routes: []pool.Route{ + { + Dst: "5.5.0.0/16", + }, + }, }, } } @@ -126,13 +132,27 @@ var _ = Describe("Handlers", func() { HaveField("Pool", testPoolName1), HaveField("Ip", "192.168.0.2/16"), HaveField("Gateway", "192.168.0.1"), + HaveField("Routes", HaveLen(0)), ))) Expect(resp.Allocations).To(ContainElement( And( HaveField("Pool", testPoolName2), HaveField("Ip", "10.100.0.2/16"), HaveField("Gateway", "10.100.0.1"), + HaveField("Routes", HaveLen(2)), ))) + for _, alloc := range resp.Allocations { + if len(alloc.Routes) > 0 { + Expect(alloc.Routes).To(ContainElement( + And( + HaveField("Dest", "5.5.0.0/16"), + ))) + Expect(alloc.Routes).To(ContainElement( + And( + HaveField("Dest", "0.0.0.0/0"), + ))) + } + } }) It("Allocate static IP", func() { store.On("Open", mock.Anything).Return(session, nil) @@ -156,6 +176,44 @@ var _ = Describe("Handlers", func() { HaveField("Gateway", "192.168.0.1"), ))) }) + It("Allocate duplicate static routes", func() { + store.On("Open", mock.Anything).Return(session, nil) + allocators[testPoolName1].On("Allocate", "id1", "net0", mock.Anything, net.ParseIP("192.168.0.2")).Return( + ¤t.IPConfig{ + Gateway: net.ParseIP("192.168.0.1"), + Address: getIPWithMask("192.168.0.2/16"), + }, nil) + pool1Cfg := getPoolConfigs()[testPoolName1] + pool1Cfg.DefaultGateway = true + pool1Cfg.Routes = []pool.Route{ + { + Dst: "5.5.0.0/16", + }, + { + Dst: "5.5.0.0/16", + }, + { + Dst: "0.0.0.0/0", + }, + } + poolManager.On("GetPoolByKey", testPoolName1).Return(pool1Cfg) + session.On("Commit").Return(nil) + ipamParams := getValidIPAMParams() + ipamParams.Pools = []string{testPoolName1} + ipamParams.RequestedIps = []string{"192.168.0.2"} + resp, err := handlers.Allocate(ctx, &nodev1.AllocateRequest{Parameters: ipamParams}) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.Allocations).To(HaveLen(1)) + Expect(resp.Allocations[0].Routes).To(HaveLen(2)) + Expect(resp.Allocations[0].Routes).To(ContainElement( + And( + HaveField("Dest", "5.5.0.0/16"), + ))) + Expect(resp.Allocations[0].Routes).To(ContainElement( + And( + HaveField("Dest", "0.0.0.0/0"), + ))) + }) It("Allocate failed: static IP was not allocated", func() { store.On("Open", mock.Anything).Return(session, nil) poolManager.On("GetPoolByKey", testPoolName1).Return(getPoolConfigs()[testPoolName1]) @@ -241,6 +299,26 @@ var _ = Describe("Handlers", func() { _, err := handlers.Allocate(ctx, &nodev1.AllocateRequest{Parameters: getValidIPAMParams()}) Expect(status.Code(err) == codes.Canceled).To(BeTrue()) }) + It("Allocation failed: bad static route", func() { + store.On("Open", mock.Anything).Return(session, nil) + pool2Cfg := getPoolConfigs()[testPoolName2] + pool2Cfg.Routes[0].Dst = "foo" + poolManager.On("GetPoolByKey", testPoolName2).Return(pool2Cfg) + poolManager.On("GetPoolByKey", testPoolName1).Return(getPoolConfigs()[testPoolName1]) + allocators[testPoolName1].On("Allocate", "id1", "net0", mock.Anything, mock.Anything).Return( + ¤t.IPConfig{ + Gateway: net.ParseIP("192.168.0.1"), + Address: getIPWithMask("192.168.0.2/16"), + }, nil) + allocators[testPoolName2].On("Allocate", "id1", "net0", mock.Anything, mock.Anything).Return( + ¤t.IPConfig{ + Gateway: net.ParseIP("10.100.0.1"), + Address: getIPWithMask("10.100.0.2/16"), + }, nil) + session.On("Cancel").Return() + _, err := handlers.Allocate(ctx, &nodev1.AllocateRequest{Parameters: getValidIPAMParams()}) + Expect(status.Code(err) == codes.InvalidArgument).To(BeTrue()) + }) It("IsAllocated succeed", func() { store.On("Open", mock.Anything).Return(session, nil) session.On("GetReservationByID", testPoolName1, "id1", "net0").Return(&types.Reservation{}) diff --git a/pkg/pool/reader.go b/pkg/pool/reader.go index c888b1c..5d3a2eb 100644 --- a/pkg/pool/reader.go +++ b/pkg/pool/reader.go @@ -26,12 +26,14 @@ const ( // Pool represents generic pool configuration type Pool struct { - Name string `json:"-"` - Subnet string `json:"subnet"` - StartIP string `json:"startIP"` - EndIP string `json:"endIP"` - Gateway string `json:"gateway"` - Exclusions []ExclusionRange `json:"exclusions"` + Name string `json:"-"` + Subnet string `json:"subnet"` + StartIP string `json:"startIP"` + EndIP string `json:"endIP"` + Gateway string `json:"gateway"` + Exclusions []ExclusionRange `json:"exclusions"` + Routes []Route `json:"routes"` + DefaultGateway bool `json:"defaultGateway"` } // ExclusionRange contains range of IP to exclude from the allocation @@ -40,6 +42,11 @@ type ExclusionRange struct { EndIP string `json:"endIP"` } +// Route contains a destination CIDR to be added as static route via gateway +type Route struct { + Dst string `json:"dst"` +} + // String return string representation of the IPPool config func (p *Pool) String() string { //nolint:errchkjson