conntrack: Remove old implementation

README.md

@@ -412,10 +412,6 @@ parameters:
             output: match-10.0
             type: add_regex_if
             parameters: 10.0.*
-          - input: "{{.srcIP}},{{.srcPort}},{{.dstIP}},{{.dstPort}},{{.protocol}}"
-            output: isNewFlow
-            type: conn_tracking
-            parameters: "1"
 ```
 
 The first rule `add_subnet` generates a new field named `srcSubnet` with the 
@@ -460,10 +456,6 @@ the contents of the `srcSubnet` field for entries that match regex expression sp
 in the `parameters` variable. In addition, the field `match-10.0_Matched` with 
 value `true` is added to all matched entries
 
-The seventh rule `conn_tracking` generates a new field named `isNewFlow` that contains
-the contents of the `parameters` variable **only for new entries** (first seen in 120 seconds) 
-that match hash of template fields from the `input` variable. 
-
 
 > Note: above example describes all available transform network `Type` options
 

contrib/dashboards/dashboard_details.json

@@ -363,7 +363,7 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "topk(10,rate(flp_connections_per_destination_subnet[1m]))",
+               "expr": "topk(10,rate(flp_connections_per_destination_subnet{_RecordType=\"newConnection\"}[1m]))",
                "format": "time_series",
                "intervalFactor": 2,
                "legendFormat": "",

contrib/dashboards/jsonnet/dashboard_details.jsonnet

@@ -90,7 +90,7 @@ dashboard.new(
   )
   .addTarget(
     prometheus.target(
-      expr='topk(10,rate(flp_connections_per_destination_subnet[1m]))',
+      expr='topk(10,rate(flp_connections_per_destination_subnet{_RecordType="newConnection"}[1m]))',
     )
   ), gridPos={
     x: 0,

contrib/kubernetes/flowlogs-pipeline.conf.yaml

@@ -71,10 +71,6 @@ parameters:
         output: srcSubnet
         type: add_subnet
         parameters: /16
-      - input: '{{.srcIP}},{{.srcPort}},{{.dstIP}},{{.dstPort}},{{.proto}}'
-        output: isNewFlow
-        type: conn_tracking
-        parameters: "1"
       - input: dstIP
         output: dstSubnet
         type: add_subnet
@@ -100,10 +96,12 @@ parameters:
           fields:
           - srcIP
           - srcPort
+          - srcSubnet
         - name: dst
           fields:
           - dstIP
           - dstPort
+          - dstSubnet
         - name: protocol
           fields:
           - proto
@@ -181,7 +179,6 @@ parameters:
       - dstSubnet
       - _RecordType
       operation: count
-      recordKey: isNewFlow
     - name: src_connection_count
       by:
       - srcSubnet
@@ -324,8 +321,8 @@ parameters:
           value: dest_connection_subnet_count
         valueKey: recent_count
         labels:
-        - by
-        - aggregate
+        - _RecordType
+        - dstSubnet
         buckets: []
       - name: connections_per_source_subnet
         type: counter

docs/api.md

@@ -123,7 +123,6 @@ Following is the supported API format for network transformations:
                  input: entry input field
                  output: entry output field
                  type: (enum) one of the following:
-                     conn_tracking: set output field to value of parameters field only for new flows by matching template in input field
                      add_regex_if: add output field if input field satisfies regex pattern from parameters field
                      add_if: add output field if input field satisfies criteria from parameters field
                      add_subnet: add output subnet field from input field and prefix length from parameters field

docs/metrics.md

@@ -68,10 +68,10 @@ and the transformation to generate the exported metric.
 ### connection rate per dest subnet
 | **Description** | This metric observes network connections rate per destination subnet | 
 |:---|:---|
-| **Details** | Counts the number of connections per subnet with network prefix length /16 (using conn_tracking sum isNewFlow field) | 
+| **Details** | Counts the number of connections per subnet with network prefix length /16 | 
 | **Usage** | Evaluate network connections per subnet | 
 | **Tags** | rate, subnet |
-| **Operation** | aggregate by `dstSubnet, _RecordType` and `count` field `isNewFlow` |
+| **Operation** | aggregate by `dstSubnet, _RecordType` and `count`  |
 | **Exposed as** | `flp_connections_per_destination_subnet` of type `counter` |
 | **Visualized as** | "Connections rate per destinationIP /16 subnets" on dashboard `details` |
 |||  

network_definitions/config.yaml

@@ -46,10 +46,12 @@ extract:
           fields:
             - srcIP
             - srcPort
+            - srcSubnet
         - name: dst
           fields:
             - dstIP
             - dstPort
+            - dstSubnet
         - name: protocol
           fields:
             - proto

network_definitions/connection_rate_per_dest_subnet.yaml

@@ -2,18 +2,14 @@
 description:
   This metric observes network connections rate per destination subnet
 details:
-  Counts the number of connections per subnet with network prefix length /16 (using conn_tracking sum isNewFlow field)
+  Counts the number of connections per subnet with network prefix length /16
 usage:
   Evaluate network connections per subnet
 tags:
   - rate
   - subnet
 transform:
   rules:
-    - input: "{{.srcIP}},{{.srcPort}},{{.dstIP}},{{.dstPort}},{{.proto}}"
-      output: isNewFlow
-      type: conn_tracking
-      parameters: "1"
     - input: dstIP
       output: dstSubnet
       type: add_subnet
@@ -26,7 +22,6 @@ extract:
         - dstSubnet
         - _RecordType
       operation: count
-      recordKey: isNewFlow
 encode:
   type: prom
   prom:
@@ -36,12 +31,12 @@ encode:
         filter: {key: name, value: dest_connection_subnet_count}
         valueKey: recent_count
         labels:
-          - by
-          - aggregate
+          - _RecordType
+          - dstSubnet
 visualization:
   type: grafana
   grafana:
-    - expr: 'topk(10,rate(flp_connections_per_destination_subnet[1m]))'
+    - expr: 'topk(10,rate(flp_connections_per_destination_subnet{_RecordType="newConnection"}[1m]))'
       type: graphPanel
       dashboard: details
       title:

pkg/api/api.go

@@ -34,7 +34,6 @@ const (
 	FilterType            = "filter"
 	ConnTrackType         = "conntrack"
 	NoneType              = "none"
-	ConnTrackingRuleType  = "conn_tracking"
 	AddRegExIfRuleType    = "add_regex_if"
 	AddIfRuleType         = "add_if"
 	AddSubnetRuleType     = "add_subnet"

pkg/api/transform_network.go

@@ -25,7 +25,6 @@ type TransformNetwork struct {
 }
 
 type TransformNetworkOperationEnum struct {
-	ConnTracking  string `yaml:"conn_tracking" json:"conn_tracking" doc:"set output field to value of parameters field only for new flows by matching template in input field"`
 	AddRegExIf    string `yaml:"add_regex_if" json:"add_regex_if" doc:"add output field if input field satisfies regex pattern from parameters field"`
 	AddIf         string `yaml:"add_if" json:"add_if" doc:"add output field if input field satisfies criteria from parameters field"`
 	AddSubnet     string `yaml:"add_subnet" json:"add_subnet" doc:"add output subnet field from input field and prefix length from parameters field"`