From 1c0697b17d3ebd49a816b4da2205eb9cdd030cbc Mon Sep 17 00:00:00 2001
From: Ronen Schaffer <ronen.schaffer@ibm.com>
Date: Thu, 1 Sep 2022 12:50:13 +0300
Subject: [PATCH] Fix connection_rate_per_dest_subnet metric to use new
 conntrack

---
 contrib/dashboards/dashboard_details.json            |  2 +-
 contrib/dashboards/jsonnet/dashboard_details.jsonnet |  2 +-
 contrib/kubernetes/flowlogs-pipeline.conf.yaml       | 11 ++++-------
 docs/metrics.md                                      |  2 +-
 network_definitions/config.yaml                      |  2 ++
 .../connection_rate_per_dest_subnet.yaml             | 12 +++---------
 6 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/contrib/dashboards/dashboard_details.json b/contrib/dashboards/dashboard_details.json
index 6167f3809..5a6e2053a 100644
--- a/contrib/dashboards/dashboard_details.json
+++ b/contrib/dashboards/dashboard_details.json
@@ -363,7 +363,7 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "topk(10,rate(flp_connections_per_destination_subnet[1m]))",
+               "expr": "topk(10,rate(flp_connections_per_destination_subnet{_RecordType=\"newConnection\"}[1m]))",
                "format": "time_series",
                "intervalFactor": 2,
                "legendFormat": "",
diff --git a/contrib/dashboards/jsonnet/dashboard_details.jsonnet b/contrib/dashboards/jsonnet/dashboard_details.jsonnet
index 53847ed2a..fa7607fd9 100644
--- a/contrib/dashboards/jsonnet/dashboard_details.jsonnet
+++ b/contrib/dashboards/jsonnet/dashboard_details.jsonnet
@@ -90,7 +90,7 @@ dashboard.new(
   )
   .addTarget(
     prometheus.target(
-      expr='topk(10,rate(flp_connections_per_destination_subnet[1m]))',
+      expr='topk(10,rate(flp_connections_per_destination_subnet{_RecordType="newConnection"}[1m]))',
     )
   ), gridPos={
     x: 0,
diff --git a/contrib/kubernetes/flowlogs-pipeline.conf.yaml b/contrib/kubernetes/flowlogs-pipeline.conf.yaml
index bde7e503e..86c8c8a13 100644
--- a/contrib/kubernetes/flowlogs-pipeline.conf.yaml
+++ b/contrib/kubernetes/flowlogs-pipeline.conf.yaml
@@ -71,10 +71,6 @@ parameters:
         output: srcSubnet
         type: add_subnet
         parameters: /16
-      - input: '{{.srcIP}},{{.srcPort}},{{.dstIP}},{{.dstPort}},{{.proto}}'
-        output: isNewFlow
-        type: conn_tracking
-        parameters: "1"
       - input: dstIP
         output: dstSubnet
         type: add_subnet
@@ -100,10 +96,12 @@ parameters:
           fields:
           - srcIP
           - srcPort
+          - srcSubnet
         - name: dst
           fields:
           - dstIP
           - dstPort
+          - dstSubnet
         - name: protocol
           fields:
           - proto
@@ -181,7 +179,6 @@ parameters:
       - dstSubnet
       - _RecordType
       operation: count
-      recordKey: isNewFlow
     - name: src_connection_count
       by:
       - srcSubnet
@@ -324,8 +321,8 @@ parameters:
           value: dest_connection_subnet_count
         valueKey: recent_count
         labels:
-        - by
-        - aggregate
+        - _RecordType
+        - dstSubnet
         buckets: []
       - name: connections_per_source_subnet
         type: counter
diff --git a/docs/metrics.md b/docs/metrics.md
index 6ad750631..574d57ae2 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -71,7 +71,7 @@ and the transformation to generate the exported metric.
 | **Details** | Counts the number of connections per subnet with network prefix length /16 | 
 | **Usage** | Evaluate network connections per subnet | 
 | **Tags** | rate, subnet |
-| **Operation** | aggregate by `dstSubnet, _RecordType` and `count` field `isNewFlow` |
+| **Operation** | aggregate by `dstSubnet, _RecordType` and `count`  |
 | **Exposed as** | `flp_connections_per_destination_subnet` of type `counter` |
 | **Visualized as** | "Connections rate per destinationIP /16 subnets" on dashboard `details` |
 |||  
diff --git a/network_definitions/config.yaml b/network_definitions/config.yaml
index 3e0c7fd6b..67f1a653d 100644
--- a/network_definitions/config.yaml
+++ b/network_definitions/config.yaml
@@ -46,10 +46,12 @@ extract:
           fields:
             - srcIP
             - srcPort
+            - srcSubnet
         - name: dst
           fields:
             - dstIP
             - dstPort
+            - dstSubnet
         - name: protocol
           fields:
             - proto
diff --git a/network_definitions/connection_rate_per_dest_subnet.yaml b/network_definitions/connection_rate_per_dest_subnet.yaml
index 535e269ce..95afb8af0 100644
--- a/network_definitions/connection_rate_per_dest_subnet.yaml
+++ b/network_definitions/connection_rate_per_dest_subnet.yaml
@@ -10,11 +10,6 @@ tags:
   - subnet
 transform:
   rules:
-    # TODO: use new connection tracking module instead of the following
-    - input: "{{.srcIP}},{{.srcPort}},{{.dstIP}},{{.dstPort}},{{.proto}}"
-      output: isNewFlow
-      type: conn_tracking
-      parameters: "1"
     - input: dstIP
       output: dstSubnet
       type: add_subnet
@@ -27,7 +22,6 @@ extract:
         - dstSubnet
         - _RecordType
       operation: count
-      recordKey: isNewFlow
 encode:
   type: prom
   prom:
@@ -37,12 +31,12 @@ encode:
         filter: {key: name, value: dest_connection_subnet_count}
         valueKey: recent_count
         labels:
-          - by
-          - aggregate
+          - _RecordType
+          - dstSubnet
 visualization:
   type: grafana
   grafana:
-    - expr: 'topk(10,rate(flp_connections_per_destination_subnet[1m]))'
+    - expr: 'topk(10,rate(flp_connections_per_destination_subnet{_RecordType="newConnection"}[1m]))'
       type: graphPanel
       dashboard: details
       title: