Skip to content

Commit

Permalink
NETOBSERV-386 IP categorization
Browse files Browse the repository at this point in the history 
Allows to configure IP ranges and assign them any name
IPs falling in those ranges are flagged with that name
  • Loading branch information
@jotak
jotak committed Jan 3, 2023
1 parent fa0662e commit 1772a81
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 17 deletions.
20 changes: 14 additions & 6 deletions pkg/api/transform_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@
package api

type TransformNetwork struct {
Rules NetworkTransformRules `yaml:"rules" json:"rules" doc:"list of transform rules, each includes:"`
KubeConfigPath string `yaml:"kubeConfigPath,omitempty" json:"kubeConfigPath,omitempty" doc:"path to kubeconfig file (optional)"`
ServicesFile string `yaml:"servicesFile,omitempty" json:"servicesFile,omitempty" doc:"path to services file (optional, default: /etc/services)"`
ProtocolsFile string `yaml:"protocolsFile,omitempty" json:"protocolsFile,omitempty" doc:"path to protocols file (optional, default: /etc/protocols)"`
DirectionInfo DirectionInfo `yaml:"directionInfo,omitempty" json:"directionInfo,omitempty" doc:"information to reinterpret flow direction (optional, to use with reinterpret_direction rule)"`
Rules NetworkTransformRules `yaml:"rules" json:"rules" doc:"list of transform rules, each includes:"`
KubeConfigPath string `yaml:"kubeConfigPath,omitempty" json:"kubeConfigPath,omitempty" doc:"path to kubeconfig file (optional)"`
ServicesFile string `yaml:"servicesFile,omitempty" json:"servicesFile,omitempty" doc:"path to services file (optional, default: /etc/services)"`
ProtocolsFile string `yaml:"protocolsFile,omitempty" json:"protocolsFile,omitempty" doc:"path to protocols file (optional, default: /etc/protocols)"`
IPCategories []NetworkTransformIPCategory `yaml:"ipCategories,omitempty" json:"ipCategories,omitempty" doc:"configure IP categories"`
DirectionInfo NetworkTransformDirectionInfo `yaml:"directionInfo,omitempty" json:"directionInfo,omitempty" doc:"information to reinterpret flow direction (optional, to use with reinterpret_direction rule)"`
}

func (tn *TransformNetwork) GetServiceFiles() (string, string) {
Expand All @@ -45,6 +46,7 @@ const (
OpAddService = "add_service"
OpAddKubernetes = "add_kubernetes"
OpReinterpretDirection = "reinterpret_direction"
OpAddIPCategory = "add_ip_category"
)

type TransformNetworkOperationEnum struct {
Expand All @@ -55,6 +57,7 @@ type TransformNetworkOperationEnum struct {
AddService string `yaml:"add_service" json:"add_service" doc:"add output network service field from input port and parameters protocol field"`
AddKubernetes string `yaml:"add_kubernetes" json:"add_kubernetes" doc:"add output kubernetes fields from input"`
ReinterpretDirection string `yaml:"reinterpret_direction" json:"reinterpret_direction" doc:"reinterpret flow direction at a higher level than the interface"`
AddIPCategory string `yaml:"add_ip_category" json:"add_ip_category" doc:"categorize IPs based on known subnets configuration"`
}

func TransformNetworkOperationName(operation string) string {
Expand All @@ -69,7 +72,7 @@ type NetworkTransformRule struct {
Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
}

type DirectionInfo struct {
type NetworkTransformDirectionInfo struct {
ReporterIPField string `yaml:"reporterIPField,omitempty" json:"reporterIPField,omitempty" doc:"field providing the reporter (agent) host IP"`
SrcHostField string `yaml:"srcHostField,omitempty" json:"srcHostField,omitempty" doc:"source host field"`
DstHostField string `yaml:"dstHostField,omitempty" json:"dstHostField,omitempty" doc:"destination host field"`
Expand All @@ -78,3 +81,8 @@ type DirectionInfo struct {
}

type NetworkTransformRules []NetworkTransformRule

type NetworkTransformIPCategory struct {
CIDRs []string `yaml:"cidrs,omitempty" json:"cidrs,omitempty" doc:"list of CIDRs to match a category"`
Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"name of the category"`
}
50 changes: 47 additions & 3 deletions pkg/pipeline/transform/transform_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,13 @@ var log = logrus.WithField("component", "transform.Network")

type Network struct {
api.TransformNetwork
svcNames *netdb.ServiceNames
svcNames *netdb.ServiceNames
categories []subnetCategory
}

type subnetCategory struct {
cidrs []*net.IPNet
name string
}

func (n *Network) Transform(inputEntry config.GenericMap) (config.GenericMap, bool) {
Expand Down Expand Up @@ -143,6 +149,13 @@ func (n *Network) Transform(inputEntry config.GenericMap) (config.GenericMap, bo
}
case api.OpReinterpretDirection:
reinterpretDirection(outputEntry, &n.DirectionInfo)
case api.OpAddIPCategory:
if strIP, ok := outputEntry[rule.Input].(string); ok {
ip := net.ParseIP(strIP)
if ip != nil {
outputEntry[rule.Output] = n.categorizeIP(ip)
}
}

default:
log.Panicf("unknown type %s for transform.Network rule: %v", rule.Type, rule)
Expand All @@ -152,6 +165,17 @@ func (n *Network) Transform(inputEntry config.GenericMap) (config.GenericMap, bo
return outputEntry, true
}

func (n *Network) categorizeIP(ip net.IP) string {
for _, subnetCat := range n.categories {
for _, cidr := range subnetCat.cidrs {
if cidr.Contains(ip) {
return subnetCat.name
}
}
}
return ""
}

// NewTransformNetwork create a new transform
func NewTransformNetwork(params config.StageParam) (Transformer, error) {
var needToInitLocationDB = false
Expand All @@ -171,9 +195,13 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
case api.OpAddService:
needToInitNetworkServices = true
case api.OpReinterpretDirection:
if err := validatereinterpretDirectionConfig(&jsonNetworkTransform.DirectionInfo); err != nil {
if err := validateReinterpretDirectionConfig(&jsonNetworkTransform.DirectionInfo); err != nil {
return nil, err
}
case api.OpAddIPCategory:
if len(jsonNetworkTransform.IPCategories) == 0 {
return nil, fmt.Errorf("a rule '%s' was found, but there are no IP categories configured", api.OpAddIPCategory)
}
}
}

Expand Down Expand Up @@ -211,11 +239,27 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
}
}

var subnetCats []subnetCategory
for _, category := range jsonNetworkTransform.IPCategories {
var cidrs []*net.IPNet
for _, cidr := range category.CIDRs {
_, parsed, err := net.ParseCIDR(cidr)
if err != nil {
return nil, fmt.Errorf("category %s: fail to parse CIDR, %w", category.Name, err)
}
cidrs = append(cidrs, parsed)
}
if len(cidrs) > 0 {
subnetCats = append(subnetCats, subnetCategory{name: category.Name, cidrs: cidrs})
}
}

return &Network{
TransformNetwork: api.TransformNetwork{
Rules: jsonNetworkTransform.Rules,
DirectionInfo: jsonNetworkTransform.DirectionInfo,
},
svcNames: servicesDB,
svcNames: servicesDB,
categories: subnetCats,
}, nil
}
4 changes: 2 additions & 2 deletions pkg/pipeline/transform/transform_network_direction.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ const (
egress = 1
)

func validatereinterpretDirectionConfig(info *api.DirectionInfo) error {
func validateReinterpretDirectionConfig(info *api.NetworkTransformDirectionInfo) error {
if info.FlowDirectionField == "" {
return fmt.Errorf("invalid config for transform.Network rule %s: missing FlowDirectionField", api.OpReinterpretDirection)
}
Expand All @@ -28,7 +28,7 @@ func validatereinterpretDirectionConfig(info *api.DirectionInfo) error {
return nil
}

func reinterpretDirection(output config.GenericMap, info *api.DirectionInfo) {
func reinterpretDirection(output config.GenericMap, info *api.NetworkTransformDirectionInfo) {
if fd, ok := output[info.FlowDirectionField]; ok && len(info.IfDirectionField) > 0 {
output[info.IfDirectionField] = fd
}
Expand Down
59 changes: 53 additions & 6 deletions pkg/pipeline/transform/transform_network_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,13 +368,60 @@ func (*fakeKubeData) GetInfo(n string) (*kubernetes.Info, error) {
}

func Test_Categorize(t *testing.T) {
entry := config.GenericMap{
"addr1": "10.1.2.3",
"addr2": "100.1.2.3",
"addr3": "100.2.3.4",
"addr4": "101.1.0.0",
}
cfg := config.StageParam{
Transform: &config.Transform{
Network: &api.TransformNetwork{
Rules: []api.NetworkTransformRule{
{Type: api.OpAddIPCategory, Input: "addr1", Output: "cat1"},
{Type: api.OpAddIPCategory, Input: "addr2", Output: "cat2"},
{Type: api.OpAddIPCategory, Input: "addr3", Output: "cat3"},
{Type: api.OpAddIPCategory, Input: "addr4", Output: "cat4"},
},
IPCategories: []api.NetworkTransformIPCategory{{
Name: "Pods overlay",
CIDRs: []string{"10.0.0.0/8"},
}, {
Name: "MySite.com",
CIDRs: []string{"101.1.0.0/32", "100.1.0.0/16"},
}, {
Name: "MyOtherSite.com",
CIDRs: []string{"100.2.3.10/32"},
}},
},
},
}

tr, err := NewTransformNetwork(cfg)
require.NoError(t, err)

output, ok := tr.Transform(entry)
require.True(t, ok)
require.Equal(t, config.GenericMap{
"addr1": "10.1.2.3",
"cat1": "Pods overlay",
"addr2": "100.1.2.3",
"cat2": "MySite.com",
"addr3": "100.2.3.4",
"cat3": "",
"addr4": "101.1.0.0",
"cat4": "MySite.com",
}, output)
}

func Test_ReinterpretDirection(t *testing.T) {
cfg := config.StageParam{
Transform: &config.Transform{
Network: &api.TransformNetwork{
Rules: []api.NetworkTransformRule{{
Type: "reinterpret_direction",
}},
DirectionInfo: api.DirectionInfo{
DirectionInfo: api.NetworkTransformDirectionInfo{
ReporterIPField: "ReporterIP",
SrcHostField: "SrcHostIP",
DstHostField: "DstHostIP",
Expand Down Expand Up @@ -511,7 +558,7 @@ func Test_ValidateReinterpretDirection(t *testing.T) {
Rules: []api.NetworkTransformRule{{
Type: "reinterpret_direction",
}},
DirectionInfo: api.DirectionInfo{
DirectionInfo: api.NetworkTransformDirectionInfo{
SrcHostField: "SrcHostIP",
DstHostField: "DstHostIP",
FlowDirectionField: "FlowDirection",
Expand All @@ -529,7 +576,7 @@ func Test_ValidateReinterpretDirection(t *testing.T) {
Rules: []api.NetworkTransformRule{{
Type: "reinterpret_direction",
}},
DirectionInfo: api.DirectionInfo{
DirectionInfo: api.NetworkTransformDirectionInfo{
ReporterIPField: "ReporterIP",
DstHostField: "DstHostIP",
FlowDirectionField: "FlowDirection",
Expand All @@ -547,7 +594,7 @@ func Test_ValidateReinterpretDirection(t *testing.T) {
Rules: []api.NetworkTransformRule{{
Type: "reinterpret_direction",
}},
DirectionInfo: api.DirectionInfo{
DirectionInfo: api.NetworkTransformDirectionInfo{
ReporterIPField: "ReporterIP",
SrcHostField: "SrcHostIP",
FlowDirectionField: "FlowDirection",
Expand All @@ -565,7 +612,7 @@ func Test_ValidateReinterpretDirection(t *testing.T) {
Rules: []api.NetworkTransformRule{{
Type: "reinterpret_direction",
}},
DirectionInfo: api.DirectionInfo{
DirectionInfo: api.NetworkTransformDirectionInfo{
ReporterIPField: "ReporterIP",
SrcHostField: "SrcHostIP",
DstHostField: "DstHostIP",
Expand All @@ -583,7 +630,7 @@ func Test_ValidateReinterpretDirection(t *testing.T) {
Rules: []api.NetworkTransformRule{{
Type: "reinterpret_direction",
}},
DirectionInfo: api.DirectionInfo{
DirectionInfo: api.NetworkTransformDirectionInfo{
ReporterIPField: "ReporterIP",
SrcHostField: "SrcHostIP",
DstHostField: "DstHostIP",
Expand Down

0 comments on commit 1772a81

Please sign in to comment.