Add browser traffic in loadgenerator and export traces in frontend (#…

…1345)

* Add locust_plugins as a dependency

* Increased memory constraints and introduced LOCUST_BROWSER_TRAFFIC_ENABLED environment variable

* Added playwright on load generator docker image

* Inceased delay interval in milliseconds between two consecutive exports

* Use different otel endpoint when request contains the synthetic_request=true header

* Add locust example using playwright. The header synthetic_request is set to true to identify them in the frontend

* Updated changelog

* Use the same otelName (OTEL_COLLECTOR_HOST) for both k8s and docker

* Removed select option event

* Revert - Removed select option event

* Added IS_SYNTHETIC_REQUEST window.env used to set the span web attribute in FrontendTracer.ts

---------

Co-authored-by: Juliano Costa <julianocosta89@outlook.com>

.env

@@ -67,6 +67,7 @@ LOCUST_HOST=http://${FRONTEND_PROXY_ADDR}
 LOCUST_WEB_HOST=loadgenerator
 LOCUST_AUTOSTART=true
 LOCUST_HEADLESS=false
+LOCUST_BROWSER_TRAFFIC_ENABLED=false
 
 # Payment Service
 PAYMENT_SERVICE_PORT=50051

CHANGELOG.md

@@ -17,6 +17,8 @@ release.
   ([#1335](https://github.com/open-telemetry/opentelemetry-demo/pull/1335))
 * [ffspostgres] define and use demo specific postgres image
   ([#1338](https://github.com/open-telemetry/opentelemetry-demo/pull/1338))
+* [loadgenerator, frontend] enable browser traffic in loadgenerator using playwright
+  ([#1345](https://github.com/open-telemetry/opentelemetry-demo/pull/1345))
 * [accountingservice] update wiki link
   ([#1346](https://github.com/open-telemetry/opentelemetry-demo/pull/1346))
 * [checkoutservice] update wiki link

docker-compose.yml

@@ -296,6 +296,7 @@ services:
       - PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
       - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE
       - WEB_OTEL_SERVICE_NAME=frontend-web
+      - OTEL_COLLECTOR_HOST
     depends_on:
       adservice:
         condition: service_started
@@ -371,7 +372,7 @@ services:
     deploy:
       resources:
         limits:
-          memory: 120M
+          memory: 1G
     restart: unless-stopped
     ports:
       - "${LOCUST_WEB_PORT}"
@@ -381,6 +382,7 @@ services:
       - LOCUST_HOST
       - LOCUST_HEADLESS
       - LOCUST_AUTOSTART
+      - LOCUST_BROWSER_TRAFFIC_ENABLED
       - OTEL_EXPORTER_OTLP_ENDPOINT
       - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE
       - OTEL_RESOURCE_ATTRIBUTES

kubernetes/opentelemetry-demo.yaml

@@ -9824,6 +9824,8 @@ spec:
             value: http://localhost:8080/otlp-http/v1/traces
           - name: OTEL_RESOURCE_ATTRIBUTES
             value: service.name=$(OTEL_SERVICE_NAME),service.namespace=opentelemetry-demo
+          - name: OTEL_COLLECTOR_HOST
+            value: $(OTEL_COLLECTOR_NAME)
           resources:
             limits:
               memory: 200Mi
@@ -10041,6 +10043,8 @@ spec:
             value: "false"
           - name: LOCUST_AUTOSTART
             value: "true"
+          - name: LOCUST_BROWSER_TRAFFIC_ENABLED
+            value: "false"
           - name: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION
             value: python
           - name: OTEL_EXPORTER_OTLP_ENDPOINT
@@ -10049,7 +10053,7 @@ spec:
             value: service.name=$(OTEL_SERVICE_NAME),service.namespace=opentelemetry-demo
           resources:
             limits:
-              memory: 120Mi
+              memory: 1Gi
 ---
 # Source: opentelemetry-demo/templates/component.yaml
 apiVersion: apps/v1

src/frontend/pages/_app.tsx

@@ -17,6 +17,7 @@ declare global {
       NEXT_PUBLIC_PLATFORM?: string;
       NEXT_PUBLIC_OTEL_SERVICE_NAME?: string;
       NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT?: string;
+      IS_SYNTHETIC_REQUEST?: string;
     };
   }
 }

src/frontend/pages/_document.tsx

@@ -3,16 +3,9 @@
 
 import Document, { DocumentContext, Html, Head, Main, NextScript } from 'next/document';
 import { ServerStyleSheet } from 'styled-components';
+import {context, propagation} from "@opentelemetry/api";
 
-const { ENV_PLATFORM, WEB_OTEL_SERVICE_NAME, PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT } = process.env;
-
-const envString = `
-window.ENV = {
-  NEXT_PUBLIC_PLATFORM: '${ENV_PLATFORM}',
-  NEXT_PUBLIC_OTEL_SERVICE_NAME: '${WEB_OTEL_SERVICE_NAME}',
-  NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: '${PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}',
-};
-`;
+const { ENV_PLATFORM, WEB_OTEL_SERVICE_NAME, PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_COLLECTOR_HOST} = process.env;
 
 export default class MyDocument extends Document<{ envString: string }> {
   static async getInitialProps(ctx: DocumentContext) {
@@ -26,6 +19,20 @@ export default class MyDocument extends Document<{ envString: string }> {
         });
 
       const initialProps = await Document.getInitialProps(ctx);
+      const baggage = propagation.getBaggage(context.active());
+      const isSyntheticRequest = baggage?.getEntry('synthetic_request')?.value === 'true';
+
+      const otlpTracesEndpoint = isSyntheticRequest
+          ? `http://${OTEL_COLLECTOR_HOST}:4318/v1/traces`
+          : PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT;
+
+      const envString = `
+        window.ENV = {
+          NEXT_PUBLIC_PLATFORM: '${ENV_PLATFORM}',
+          NEXT_PUBLIC_OTEL_SERVICE_NAME: '${WEB_OTEL_SERVICE_NAME}',
+          NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: '${otlpTracesEndpoint}',
+          IS_SYNTHETIC_REQUEST: '${isSyntheticRequest}',
+        };`;
       return {
         ...initialProps,
         styles: [initialProps.styles, sheet.getStyleElement()],

src/frontend/utils/telemetry/FrontendTracer.ts

@@ -12,7 +12,7 @@ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
 import { SessionIdProcessor } from './SessionIdProcessor';
 import { detectResourcesSync } from '@opentelemetry/resources/build/src/detect-resources';
 
-const { NEXT_PUBLIC_OTEL_SERVICE_NAME = '', NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = '' } =
+const { NEXT_PUBLIC_OTEL_SERVICE_NAME = '', NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = '', IS_SYNTHETIC_REQUEST = '' } =
   typeof window !== 'undefined' ? window.ENV : {};
 
 const FrontendTracer = async (collectorString: string) => {
@@ -34,7 +34,9 @@ const FrontendTracer = async (collectorString: string) => {
     new BatchSpanProcessor(
       new OTLPTraceExporter({
         url: NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || collectorString || 'http://localhost:4318/v1/traces',
-      })
+      }), {
+          scheduledDelayMillis : 500
+        }
     )
   );
 
@@ -55,7 +57,7 @@ const FrontendTracer = async (collectorString: string) => {
           propagateTraceHeaderCorsUrls: /.*/,
           clearTimingResources: true,
           applyCustomAttributesOnSpan(span) {
-            span.setAttribute('app.synthetic_request', 'false');
+            span.setAttribute('app.synthetic_request', IS_SYNTHETIC_REQUEST);
           },
         },
       }),

src/loadgenerator/Dockerfile

@@ -17,4 +17,6 @@ WORKDIR /usr/src/app/
 COPY --from=builder /reqs /usr/local
 COPY ./src/loadgenerator/locustfile.py .
 COPY ./src/loadgenerator/people.json .
+ENV LOCUST_PLAYWRIGHT=1
+RUN playwright install --with-deps chromium
 ENTRYPOINT locust

src/loadgenerator/locustfile.py

@@ -5,9 +5,11 @@
 
 
 import json
+import os
 import random
 import uuid
 from locust import HttpUser, task, between
+from locust_plugins.users.playwright import PlaywrightUser, pw, PageWithRetry, event
 
 from opentelemetry import context, baggage, trace
 from opentelemetry.metrics import set_meter_provider
@@ -21,6 +23,7 @@
 from opentelemetry.instrumentation.requests import RequestsInstrumentor
 from opentelemetry.instrumentation.system_metrics import SystemMetricsInstrumentor
 from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor
+from playwright.async_api import Route, Request
 
 exporter = OTLPMetricExporter(insecure=True)
 set_meter_provider(MeterProvider([PeriodicExportingMetricReader(exporter)]))
@@ -130,3 +133,42 @@ def on_start(self):
         context.attach(ctx)
         self.index()
 
+
+browser_traffic_enabled = os.environ.get('LOCUST_BROWSER_TRAFFIC_ENABLED', False)
+
+if browser_traffic_enabled:
+    class WebsiteBrowserUser(PlaywrightUser):
+        headless = True  # to use a headless browser, without a GUI
+
+        @task
+        @pw
+        async def open_cart_page_and_change_currency(self, page: PageWithRetry):
+            try:
+                page.on("console", lambda msg: print(msg.text))
+                await page.route('**/*', add_baggage_header)
+                await page.goto("/cart", wait_until="domcontentloaded")
+                await page.select_option('[name="currency_code"]', 'CHF')
+                await page.wait_for_timeout(2000)  # giving the browser time to export the traces
+            except:
+                pass
+
+        @task
+        @pw
+        async def add_product_to_cart(self, page: PageWithRetry):
+            try:
+                page.on("console", lambda msg: print(msg.text))
+                await page.route('**/*', add_baggage_header)
+                await page.goto("/", wait_until="domcontentloaded")
+                await page.click('p:has-text("Roof Binoculars")', wait_until="domcontentloaded")
+                await page.click('button:has-text("Add To Cart")', wait_until="domcontentloaded")
+                await page.wait_for_timeout(2000)  # giving the browser time to export the traces
+            except:
+                pass
+
+
+async def add_baggage_header(route: Route, request: Request):
+    headers = {
+        **request.headers,
+        'baggage': 'synthetic_request=true'
+    }
+    await route.continue_(headers=headers)

src/loadgenerator/requirements.txt

@@ -16,6 +16,7 @@ idna==3.4
 itsdangerous==2.1.2
 jinja2==3.1.2
 locust==2.18.2
+locust_plugins==3.4.0
 markupsafe==2.1.3
 msgpack==1.0.7
 opentelemetry-api==1.21.0