[APM] Service groups - dynamic query refresh (#140406)

* Service groups - dynamic query refresh

* fixes bug with group save API which omits the query params

* don't render 0 services if no services count is given

* adds support for dynamic refresh in getSortedAndFilteredServices

* only include a whitelisted set of metric doc fields in kuery bar suggestions

* clean up unused i18n translation

* address PR feedback

* fixes bug with a bad translation variable reference

Co-authored-by: Oliver Gupte <oliver.gupte@elastic.co>

x-pack/plugins/apm/common/service_groups.ts

@@ -13,7 +13,6 @@ export interface ServiceGroup {
   groupName: string;
   kuery: string;
   description?: string;
-  serviceNames: string[];
   color?: string;
 }
 

x-pack/plugins/apm/public/components/app/service_groups/service_group_save/save_modal.tsx

@@ -4,7 +4,6 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import datemath from '@kbn/datemath';
 import { EuiModal } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { useHistory } from 'react-router-dom';
@@ -60,14 +59,9 @@ export function SaveGroupModal({ onClose, savedServiceGroup }: Props) {
     async function (newServiceGroup: StagedServiceGroup) {
       setIsLoading(true);
       try {
-        const start = datemath.parse('now-24h')?.toISOString();
-        const end = datemath.parse('now', { roundUp: true })?.toISOString();
-        if (!start || !end) {
-          throw new Error('Unable to determine start/end time range.');
-        }
         await callApmApi('POST /internal/apm/service-group', {
           params: {
-            query: { start, end, serviceGroupId: savedServiceGroup?.id },
+            query: { serviceGroupId: savedServiceGroup?.id },
             body: {
               groupName: newServiceGroup.groupName,
               kuery: newServiceGroup.kuery,

x-pack/plugins/apm/public/components/app/service_groups/service_group_save/select_services.tsx

@@ -39,6 +39,13 @@ const MAX_CONTAINER_HEIGHT = 600;
 const MODAL_HEADER_HEIGHT = 122;
 const MODAL_FOOTER_HEIGHT = 80;
 
+const suggestedFieldsWhitelist = [
+  'agent.name',
+  'service.name',
+  'service.language.name',
+  'service.environment',
+];
+
 const Container = styled.div`
   width: 600px;
   height: ${MAX_CONTAINER_HEIGHT}px;
@@ -144,6 +151,21 @@ export function SelectServices({
                     setStagedKuery(value);
                   }}
                   value={kuery}
+                  suggestionFilter={(querySuggestion) => {
+                    if ('field' in querySuggestion) {
+                      const {
+                        field: {
+                          spec: { name: fieldName },
+                        },
+                      } = querySuggestion;
+
+                      return (
+                        fieldName.startsWith('label') ||
+                        suggestedFieldsWhitelist.includes(fieldName)
+                      );
+                    }
+                    return true;
+                  }}
                 />
               </EuiFlexItem>
               <EuiFlexItem grow={false}>

x-pack/plugins/apm/public/components/app/service_groups/service_groups_list/index.tsx

@@ -15,11 +15,12 @@ import {
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { isEmpty, sortBy } from 'lodash';
-import React, { useState, useCallback } from 'react';
+import React, { useState, useCallback, useMemo } from 'react';
 import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
 import { ServiceGroupsListItems } from './service_groups_list';
 import { Sort } from './sort';
 import { RefreshServiceGroupsSubscriber } from '../refresh_service_groups_subscriber';
+import { getDateRange } from '../../../../context/url_params_context/helpers';
 
 export type ServiceGroupsSortType = 'recently_added' | 'alphabetical';
 
@@ -38,6 +39,31 @@ export function ServiceGroupsList() {
     []
   );
 
+  const { start, end } = useMemo(
+    () =>
+      getDateRange({
+        rangeFrom: 'now-24h',
+        rangeTo: 'now',
+      }),
+    []
+  );
+
+  const { data: servicesCountData = { servicesCounts: {} } } = useFetcher(
+    (callApmApi) => {
+      if (start && end) {
+        return callApmApi('GET /internal/apm/service_groups/services_count', {
+          params: {
+            query: {
+              start,
+              end,
+            },
+          },
+        });
+      }
+    },
+    [start, end]
+  );
+
   const { serviceGroups } = data;
 
   const isLoading =
@@ -133,7 +159,11 @@ export function ServiceGroupsList() {
             </EuiFlexItem>
             <EuiFlexItem>
               {items.length ? (
-                <ServiceGroupsListItems items={items} isLoading={isLoading} />
+                <ServiceGroupsListItems
+                  items={items}
+                  servicesCounts={servicesCountData.servicesCounts}
+                  isLoading={isLoading}
+                />
               ) : (
                 <EuiEmptyPrompt
                   iconType="layers"

x-pack/plugins/apm/public/components/app/service_groups/service_groups_list/service_group_card.tsx

@@ -27,6 +27,7 @@ interface Props {
   onClick?: () => void;
   href?: string;
   withTour?: boolean;
+  servicesCount?: number;
 }
 
 export function ServiceGroupsCard({
@@ -35,6 +36,7 @@ export function ServiceGroupsCard({
   onClick,
   href,
   withTour,
+  servicesCount,
 }: Props) {
   const { tourEnabled, dismissTour } = useServiceGroupsTour('serviceGroupCard');
 
@@ -62,13 +64,17 @@ export function ServiceGroupsCard({
         {!hideServiceCount && (
           <EuiFlexItem>
             <EuiText size="s">
-              {i18n.translate(
-                'xpack.apm.serviceGroups.cardsList.serviceCount',
-                {
-                  defaultMessage:
-                    '{servicesCount} {servicesCount, plural, one {service} other {services}}',
-                  values: { servicesCount: serviceGroup.serviceNames.length },
-                }
+              {servicesCount === undefined ? (
+                <>&nbsp;</>
+              ) : (
+                i18n.translate(
+                  'xpack.apm.serviceGroups.cardsList.serviceCount',
+                  {
+                    defaultMessage:
+                      '{servicesCount} {servicesCount, plural, one {service} other {services}}',
+                    values: { servicesCount },
+                  }
+                )
               )}
             </EuiText>
           </EuiFlexItem>

x-pack/plugins/apm/public/components/app/service_groups/service_groups_list/service_groups_list.tsx

@@ -16,10 +16,11 @@ import { useDefaultEnvironment } from '../../../../hooks/use_default_environment
 
 interface Props {
   items: SavedServiceGroup[];
+  servicesCounts: Record<string, number>;
   isLoading: boolean;
 }
 
-export function ServiceGroupsListItems({ items }: Props) {
+export function ServiceGroupsListItems({ items, servicesCounts }: Props) {
   const router = useApmRouter();
   const { query } = useApmParams('/service-groups');
 
@@ -30,6 +31,7 @@ export function ServiceGroupsListItems({ items }: Props) {
       {items.map((item) => (
         <ServiceGroupsCard
           serviceGroup={item}
+          servicesCount={servicesCounts[item.id]}
           href={router.link('/services', {
             query: {
               ...query,
@@ -52,7 +54,6 @@ export function ServiceGroupsListItems({ items }: Props) {
             'xpack.apm.serviceGroups.list.allServices.description',
             { defaultMessage: 'View all services' }
           ),
-          serviceNames: [],
           color: SERVICE_GROUP_COLOR_DEFAULT,
         }}
         hideServiceCount

x-pack/plugins/apm/public/components/shared/kuery_bar/index.tsx

@@ -40,6 +40,7 @@ export function KueryBar(props: {
   onSubmit?: (value: string) => void;
   onChange?: (value: string) => void;
   value?: string;
+  suggestionFilter?: (querySuggestion: QuerySuggestion) => boolean;
 }) {
   const { path, query } = useApmParams('/*');
 
@@ -102,7 +103,7 @@ export function KueryBar(props: {
     currentRequestCheck = currentRequest;
 
     try {
-      const suggestions = (
+      const suggestions =
         (await unifiedSearch.autocomplete.getQuerySuggestions({
           language: 'kuery',
           indexPatterns: [dataView],
@@ -120,14 +121,21 @@ export function KueryBar(props: {
           selectionEnd: selectionStart,
           useTimeRange: true,
           method: 'terms_agg',
-        })) || []
-      ).slice(0, 15);
+        })) || [];
+
+      const filteredSuggestions = props.suggestionFilter
+        ? suggestions.filter(props.suggestionFilter)
+        : suggestions;
 
       if (currentRequest !== currentRequestCheck) {
         return;
       }
 
-      setState({ ...state, suggestions, isLoadingSuggestions: false });
+      setState({
+        ...state,
+        suggestions: filteredSuggestions.slice(0, 15),
+        isLoadingSuggestions: false,
+      });
     } catch (e) {
       console.error('Error while fetching suggestions', e);
     }

x-pack/plugins/apm/common/utils/service_group_query.ts → x-pack/plugins/apm/server/lib/service_group_query.ts

@@ -6,17 +6,11 @@
  */
 
 import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { SERVICE_NAME } from '../elasticsearch_fieldnames';
-import { ServiceGroup } from '../service_groups';
+import { kqlQuery } from '@kbn/observability-plugin/server';
+import { ServiceGroup } from '../../common/service_groups';
 
 export function serviceGroupQuery(
   serviceGroup?: ServiceGroup | null
 ): QueryDslQueryContainer[] {
-  if (!serviceGroup) {
-    return [];
-  }
-
-  return serviceGroup?.serviceNames
-    ? [{ terms: { [SERVICE_NAME]: serviceGroup.serviceNames } }]
-    : [];
+  return serviceGroup ? kqlQuery(serviceGroup?.kuery) : [];
 }

x-pack/plugins/apm/server/routes/service_groups/get_services_counts.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProcessorEvent } from '@kbn/observability-plugin/common';
+import { rangeQuery, kqlQuery } from '@kbn/observability-plugin/server';
+import { Setup } from '../../lib/helpers/setup_request';
+import { SERVICE_NAME } from '../../../common/elasticsearch_fieldnames';
+
+export async function getServicesCounts({
+  setup,
+  kuery,
+  maxNumberOfServices,
+  start,
+  end,
+}: {
+  setup: Setup;
+  kuery: string;
+  maxNumberOfServices: number;
+  start: number;
+  end: number;
+}) {
+  const { apmEventClient } = setup;
+
+  const response = await apmEventClient.search('get_services_count', {
+    apm: {
+      events: [
+        ProcessorEvent.metric,
+        ProcessorEvent.transaction,
+        ProcessorEvent.span,
+        ProcessorEvent.error,
+      ],
+    },
+    body: {
+      size: 0,
+      query: {
+        bool: {
+          filter: [...rangeQuery(start, end), ...kqlQuery(kuery)],
+        },
+      },
+      aggs: {
+        services_count: {
+          cardinality: {
+            field: SERVICE_NAME,
+          },
+        },
+      },
+    },
+  });
+
+  return response?.aggregations?.services_count.value ?? 0;
+}