[exporter/logzioexporter] Added scope name to exported logs

.chloggen/logzio-log-scope.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: logzioexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: add scopename to exported logs
+
+# One or more tracking issues related to the change
+issues: [20659]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: when exists, scope name will be add to exported logs under the scopeName field.

exporter/logzioexporter/README.md

@@ -169,3 +169,173 @@ service:
     logs:
       level: debug #activate debug mode
 ```
+
+#### Scope Name
+When using the logs exporter with logs originating from instrumentation library (i.e opentelemetry log4j2 appender), the scopeName field will be added (if the field is populated in the original log). 
+
+
+| Status                   |                       |
+| ------------------------ | --------------------- |
+| Stability                | traces [beta]         |
+|                          | logs [beta]           |
+| Supported pipeline types | traces, logs |
+| Distributions            |  [contrib]   |
+
+This exporter supports sending trace and log data to [Logz.io](https://www.logz.io)
+
+### The following configuration options are supported:
+Logz.io exporter is utilizing opentelemetry [exporter helper](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) for `retry_on_failure`,`sending_queue` and `timeout` settings
+- `account_token` (Required): Your logz.io account token for your tracing or logs account.
+- `region` Your logz.io account [region code](https://docs.logz.io/user-guide/accounts/account-region.html#available-regions). Defaults to `us`. Required only if your logz.io region is different than US.
+- `endpoint` Custom endpoint, mostly used for dev or testing. This will override the region parameter.
+- `retry_on_failure`
+    - `enabled` (default = true)
+    - `initial_interval`: Time to wait after the first failure before retrying; ignored if `enabled` is `false`  (default = 5s)
+    - `max_interval`: Is the upper bound on backoff; ignored if `enabled` is `false` (default = 30s)
+    - `max_elapsed_time`: Is the maximum amount of time spent trying to send a batch; ignored if `enabled` is `false` (default = 300s)
+- `sending_queue`
+    - `enabled` (default = true)
+    - `num_consumers`: Number of consumers that dequeue batches; ignored if `enabled` is `false` (default = 10)
+    - `queue_size`: Maximum number of batches kept in memory before dropping; ignored if `enabled` is `false`
+      User should calculate this as `num_seconds * requests_per_second` where:
+        - `num_seconds` is the number of seconds to buffer in case of a backend outage
+        - `requests_per_second` is the average number of requests per seconds.
+        - default = 5000
+- `timeout`: Time to wait per individual attempt to send data to a backend. default = 30s
+
+#### Tracing example:
+* We recommend using `batch` processor. Batching helps better compress the data and reduce the number of outgoing connections required to transmit the data.
+
+```yaml
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "0.0.0.0:4317"
+      http:
+        endpoint: "0.0.0.0:4318"
+  jaeger:
+    protocols:
+      thrift_compact:
+        endpoint: "0.0.0.0:6831"
+      thrift_binary:
+        endpoint: "0.0.0.0:6832"
+      grpc:
+        endpoint: "0.0.0.0:14250"
+      thrift_http:
+        endpoint: "0.0.0.0:14268"
+processors:
+  batch:
+    send_batch_size: 10000
+    timeout: 1s
+exporters:
+  logzio/traces:
+    account_token: "LOGZIOtraceTOKEN"
+    region: "us"
+service:
+  pipelines:
+    traces:
+      receivers: [ otlp,jaeger ]
+      processors: [ batch ]
+      exporters: [ logzio/traces ]
+  telemetry:
+    logs:
+      level: "debug"
+```
+#### Logs example:
+* We recommend using `batch` processor. Batching helps better compress the data and reduce the number of outgoing connections required to transmit the data.
+* We recommend adding `type` attribute to classify your log records
+* We recommend adding `resourcedetection` processor to add metadata to your log records
+
+```yaml
+receivers:
+  filelog:
+    include: [ "/private/var/log/*.log" ] # MacOs system logs
+    include_file_name: false
+    include_file_path: true 
+    operators:
+      - type: move
+        from: attributes["log.file.path"]
+        to: attributes["log_file_path"]
+    attributes:
+      type: <<your-logzio-type>>
+processors:
+  batch:
+    send_batch_size: 10000
+    timeout: 1s
+  resourcedetection/system:
+    detectors: [ "system" ]
+    system:
+      hostname_sources: [ "os" ]
+exporters:
+  logzio/logs:
+    account_token: "LOGZIOlogsTOKEN"
+    region: "us"
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      processors: [ resourcedetection/system, batch ]
+      exporters: [logzio/logs]
+  telemetry:
+    logs:
+      level: "debug"
+```
+#### Metrics:
+In order to use the Prometheus backend you must use the standard prometheusremotewrite exporter as well. The following [regions](https://docs.logz.io/user-guide/accounts/account-region.html#supported-regions-for-prometheus-metrics) are supported and configured as follows. The Logz.io Listener URL for for your region, configured to use port 8052 for http traffic, or port 8053 for https traffic.
+Example:
+```yaml
+exporters:
+  prometheusremotewrite:
+    endpoint: "https://listener.logz.io:8053"
+    headers:
+      Authorization: "Bearer LOGZIOprometheusTOKEN"
+```
+
+Putting these both together it would look like this in a full configuration:
+
+```yaml
+receivers:
+  jaeger:
+    protocols:
+      thrift_http:
+        endpoint: "0.0.0.0:14278"
+
+  prometheus:
+    config:
+      scrape_configs:
+      - job_name: 'ratelimiter'
+        scrape_interval: 15s
+        static_configs:
+          - targets: [ "0.0.0.0:8889" ]
+
+exporters:
+  logzio/traces:
+    account_token: "LOGZIOtraceTOKEN"
+    region: "us"
+
+  prometheusremotewrite:
+    endpoint: "https://listener.logz.io:8053"
+    headers:
+      Authorization: "Bearer LOGZIOprometheusTOKEN"
+
+processors:
+  batch:
+    send_batch_size: 10000
+    timeout: 1s
+
+service:
+  pipelines:
+    traces:
+      receivers: [jaeger]
+      processors: [batch]
+      exporters: [logzio/traces]
+
+    metrics:
+      receivers: [prometheus]
+      exporters: [prometheusremotewrite]
+
+  telemetry:
+    logs:
+      level: debug #activate debug mode
+```

exporter/logzioexporter/exporter.go

@@ -70,6 +70,7 @@ func newLogzioTracesExporter(config *Config, set exporter.CreateSettings) (expor
 	if err != nil {
 		return nil, err
 	}
+
 	exporter.config.HTTPClientSettings.Endpoint, err = generateEndpoint(config)
 	if err != nil {
 		return nil, err
@@ -127,9 +128,10 @@ func (exporter *logzioExporter) pushLogData(ctx context.Context, ld plog.Logs) e
 		scopeLogs := resourceLogs.At(i).ScopeLogs()
 		for j := 0; j < scopeLogs.Len(); j++ {
 			logRecords := scopeLogs.At(j).LogRecords()
+			scope := scopeLogs.At(j).Scope()
 			for k := 0; k < logRecords.Len(); k++ {
 				log := logRecords.At(k)
-				jsonLog := convertLogRecordToJSON(log, resource)
+				jsonLog := convertLogRecordToJSON(log, scope.Name(), resource)
 				logzioLog, err := json.Marshal(jsonLog)
 				if err != nil {
 					return err

exporter/logzioexporter/exporter_test.go

@@ -51,11 +51,9 @@ func fillLogOne(log plog.LogRecord) {
 	log.SetSeverityText("Info")
 	log.SetSpanID([8]byte{0x01, 0x02, 0x04, 0x08})
 	log.SetTraceID([16]byte{0x08, 0x04, 0x02, 0x01})
-
 	attrs := log.Attributes()
 	attrs.PutStr("app", "server")
 	attrs.PutDouble("instance_num", 1)
-
 	// nested body map
 	attMap := log.Body().SetEmptyMap()
 	attMap.PutDouble("23", 45)
@@ -64,6 +62,7 @@ func fillLogOne(log plog.LogRecord) {
 	attNestedMap := attMap.PutEmptyMap("nested")
 	attNestedMap.PutStr("string", "v1")
 	attNestedMap.PutDouble("number", 499)
+
 }
 
 func fillLogTwo(log plog.LogRecord) {
@@ -92,6 +91,21 @@ func fillLogNoTimestamp(log plog.LogRecord) {
 	log.Body().SetStr("something happened")
 }
 
+func fillLogScopeName(log plog.LogRecord) {
+	log.SetTimestamp(TestLogTimestamp)
+	log.SetDroppedAttributesCount(2)
+	log.SetSeverityNumber(plog.SeverityNumberInfo)
+	log.SetSeverityText("Info")
+	log.SetSpanID([8]byte{0x01, 0x02, 0x04, 0x08})
+	log.SetTraceID([16]byte{0x08, 0x04, 0x02, 0x01})
+	attrs := log.Attributes()
+	attrs.PutStr("app", "log4j2")
+	attrs.PutDouble("instance_num", 1)
+	attrs.PutStr("scopeName", "test.class")
+	attrs.PutDouble("25", 36)
+	log.Body().SetStr("something happened in this scope")
+}
+
 func generateLogsOneEmptyTimestamp() plog.Logs {
 	ld := testdata.GenerateLogsOneEmptyLogRecord()
 	logs := ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords()

exporter/logzioexporter/jsonlog.go

@@ -11,7 +11,7 @@ import (
 )
 
 // convertLogRecordToJSON Takes `plog.LogRecord` and `pcommon.Resource` input, outputs byte array that represents the log record as json string
-func convertLogRecordToJSON(log plog.LogRecord, resource pcommon.Resource) map[string]interface{} {
+func convertLogRecordToJSON(log plog.LogRecord, scopeName string, resource pcommon.Resource) map[string]interface{} {
 	jsonLog := map[string]interface{}{}
 	if spanID := log.SpanID(); !spanID.IsEmpty() {
 		jsonLog["spanID"] = hex.EncodeToString(spanID[:])
@@ -26,6 +26,10 @@ func convertLogRecordToJSON(log plog.LogRecord, resource pcommon.Resource) map[s
 	if log.Timestamp().AsTime().UnixMilli() != 0 {
 		jsonLog["@timestamp"] = log.Timestamp().AsTime().UnixMilli()
 	}
+
+	if scopeName != "" {
+		jsonLog["scopeName"] = scopeName
+	}
 	// add resource attributes to each json log
 	resource.Attributes().Range(func(k string, v pcommon.Value) bool {
 		jsonLog[k] = v.AsRaw()

exporter/logzioexporter/jsonlog_test.go

@@ -33,11 +33,18 @@ func GenerateLogRecordWithMultiTypeValues() plog.LogRecord {
 	return lr
 }
 
+func GenerateLogWithScopeName() plog.LogRecord {
+	lr := plog.NewLogRecord()
+	fillLogScopeName(lr)
+	return lr
+}
+
 func TestConvertLogRecordToJSON(t *testing.T) {
 	type convertLogRecordToJSONTest struct {
-		log      plog.LogRecord
-		resource pcommon.Resource
-		expected map[string]interface{}
+		log       plog.LogRecord
+		resource  pcommon.Resource
+		expected  map[string]interface{}
+		scopeName string
 	}
 
 	var convertLogRecordToJSONTests = []convertLogRecordToJSONTest{
@@ -54,7 +61,7 @@ func TestConvertLogRecordToJSON(t *testing.T) {
 				"nested":       map[string]interface{}{"number": float64(499), "string": "v1"},
 				"spanID":       "0102040800000000",
 				"traceID":      "08040201000000000000000000000000",
-			},
+			}, "",
 		},
 		{GenerateLogRecordWithMultiTypeValues(),
 			pcommon.NewResource(),
@@ -66,15 +73,30 @@ func TestConvertLogRecordToJSON(t *testing.T) {
 				"@timestamp": TestLogTimeUnixMilli,
 				"message":    "something happened",
 				"number":     float64(64),
-			},
+			}, "",
+		},
+		{GenerateLogWithScopeName(),
+			pcommon.NewResource(),
+			map[string]interface{}{
+				"25":           float64(36),
+				"app":          "log4j2",
+				"instance_num": float64(1),
+				"level":        "Info",
+				"message":      "something happened in this scope",
+				"@timestamp":   TestLogTimeUnixMilli,
+				"spanID":       "0102040800000000",
+				"traceID":      "08040201000000000000000000000000",
+				"scopeName":    "test.class",
+			}, "test.class",
 		},
 	}
 	for _, test := range convertLogRecordToJSONTests {
-		output := convertLogRecordToJSON(test.log, test.resource)
+		output := convertLogRecordToJSON(test.log, test.scopeName, test.resource)
 		require.Equal(t, output, test.expected)
 	}
 
 }
+
 func TestSetTimeStamp(t *testing.T) {
 	var recordedRequests []byte
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {