Addressing PR comments

x-pack/plugins/cloud_security_posture/public/common/api/use_latest_findings_data_view.ts

@@ -15,6 +15,9 @@ import {
 } from '../../../common/constants';
 import { CspClientPluginStartDeps } from '../../types';
 
+/**
+ * TODO: Remove this static labels once https://github.com/elastic/kibana/issues/172615 is resolved
+ */
 const cloudSecurityFieldLabels: Record<string, string> = {
   'result.evaluation': i18n.translate(
     'xpack.csp.findings.findingsTable.findingsTableColumn.resultColumnLabel',
@@ -88,6 +91,9 @@ export const useLatestFindingsDataView = (dataView: string) => {
       throw new Error(`Data view not found [Name: {${dataView}}]`);
     }
 
+    /**
+     * TODO: Remove this update logic once https://github.com/elastic/kibana/issues/172615 is resolved
+     */
     if (
       dataView === LATEST_FINDINGS_INDEX_PATTERN ||
       dataView === LATEST_VULNERABILITIES_INDEX_PATTERN

x-pack/plugins/cloud_security_posture/public/common/constants.ts

@@ -228,3 +228,5 @@ export const NO_FINDINGS_STATUS_REFRESH_INTERVAL_MS = 10000;
 
 export const DETECTION_ENGINE_RULES_KEY = 'detection_engine_rules';
 export const DETECTION_ENGINE_ALERTS_KEY = 'detection_engine_alerts';
+
+export const DEFAULT_GROUPING_TABLE_HEIGHT = 512;

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/constants.ts

@@ -77,8 +77,6 @@ export const groupingTitle = i18n.translate('xpack.csp.findings.latestFindings.g
   defaultMessage: 'Group findings by',
 });
 
-export const DEFAULT_TABLE_HEIGHT = 512;
-
 export const getDefaultQuery = ({
   query,
   filters,

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/latest_findings_container.tsx

@@ -7,10 +7,10 @@
 import React from 'react';
 import { Filter } from '@kbn/es-query';
 import { EuiSpacer } from '@elastic/eui';
+import { DEFAULT_GROUPING_TABLE_HEIGHT } from '../../../common/constants';
 import { EmptyState } from '../../../components/empty_state';
 import { CloudSecurityGrouping } from '../../../components/cloud_security_grouping';
 import { FindingsSearchBar } from '../layout/findings_search_bar';
-import { DEFAULT_TABLE_HEIGHT } from './constants';
 import { useLatestFindingsGrouping } from './use_latest_findings_grouping';
 import { LatestFindingsTable } from './latest_findings_table';
 import { groupPanelRenderer, groupStatsRenderer } from './latest_findings_group_renderer';
@@ -22,7 +22,7 @@ export const LatestFindingsContainer = () => {
     return (
       <LatestFindingsTable
         nonPersistedFilters={groupFilters}
-        height={DEFAULT_TABLE_HEIGHT}
+        height={DEFAULT_GROUPING_TABLE_HEIGHT}
         showDistributionBar={false}
       />
     );

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts

@@ -26,6 +26,7 @@ import { showErrorToast } from '../../../common/utils/show_error_toast';
 interface UseFindingsOptions extends FindingsBaseEsQuery {
   sort: string[][];
   enabled: boolean;
+  pageSize: number;
 }
 
 export interface FindingsGroupByNoneQuery {
@@ -65,7 +66,7 @@ export const getFindingsQuery = ({ query, sort }: UseFindingsOptions, pageParam:
       ],
     },
   },
-  ...(pageParam ? { search_after: pageParam } : {}),
+  ...(pageParam ? { from: pageParam } : {}),
 });
 
 const getMultiFieldsSort = (sort: string[][]) => {
@@ -111,6 +112,12 @@ export const useLatestFindings = (options: UseFindingsOptions) => {
     data,
     notifications: { toasts },
   } = useKibana().services;
+  /**
+   * We're using useInfiniteQuery in this case to allow the user to fetch more data (if available and up to 10k)
+   * useInfiniteQuery differs from useQuery because it accumulates and caches a chunk of data from the previous fetches into an array
+   * it uses the getNextPageParam to know if there are more pages to load and retrieve the position of
+   * the last loaded record to be used as a from parameter to fetch the next chunk of data.
+   */
   return useInfiniteQuery(
     ['csp_findings', { params: options }],
     async ({ pageParam }) => {
@@ -135,9 +142,11 @@ export const useLatestFindings = (options: UseFindingsOptions) => {
       enabled: options.enabled,
       keepPreviousData: true,
       onError: (err: Error) => showErrorToast(toasts, err),
-      getNextPageParam: (lastPage) => {
-        if (lastPage.page.length === 0) return undefined;
-        return lastPage.page[lastPage.page.length - 1].raw.sort;
+      getNextPageParam: (lastPage, allPages) => {
+        if (lastPage.page.length < options.pageSize) {
+          return undefined;
+        }
+        return allPages.length * options.pageSize;
       },
     }
   );

x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_table.tsx

@@ -35,7 +35,8 @@ export const useLatestFindingsTable = ({
     nonPersistedFilters,
   });
 
-  const { query, sort, queryError, setUrlQuery, filters, getRowsFromPages } = cloudPostureDataTable;
+  const { query, sort, queryError, setUrlQuery, filters, getRowsFromPages, pageSize } =
+    cloudPostureDataTable;
 
   const {
     data,
@@ -46,6 +47,7 @@ export const useLatestFindingsTable = ({
     query,
     sort,
     enabled: !queryError,
+    pageSize,
   });
 
   const rows = useMemo(() => getRowsFromPages(data?.pages), [data?.pages, getRowsFromPages]);

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/constants.ts

@@ -10,8 +10,6 @@ import { FindingsBaseURLQuery } from '../../common/types';
 import { CloudSecurityDefaultColumn } from '../../components/cloud_security_data_table';
 import { GROUPING_LABELS } from './translations';
 
-export const DEFAULT_TABLE_HEIGHT = 512;
-
 export const VULNERABILITY_FIELDS = {
   VULNERABILITY_ID: 'vulnerability.id',
   SCORE_BASE: 'vulnerability.score.base',

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx

@@ -38,6 +38,7 @@ interface FindingsAggs {
 interface VulnerabilitiesQuery extends FindingsBaseEsQuery {
   sort: string[][];
   enabled: boolean;
+  pageSize: number;
 }
 
 const getMultiFieldsSort = (sort: string[][]) => {
@@ -76,14 +77,20 @@ export const getVulnerabilitiesQuery = (
       ],
     },
   },
-  ...(pageParam ? { search_after: pageParam } : {}),
+  ...(pageParam ? { from: pageParam } : {}),
 });
 
 export const useLatestVulnerabilities = (options: VulnerabilitiesQuery) => {
   const {
     data,
     notifications: { toasts },
   } = useKibana().services;
+  /**
+   * We're using useInfiniteQuery in this case to allow the user to fetch more data (if available and up to 10k)
+   * useInfiniteQuery differs from useQuery because it accumulates and caches a chunk of data from the previous fetches into an array
+   * it uses the getNextPageParam to know if there are more pages to load and retrieve the position of
+   * the last loaded record to be used as a from parameter to fetch the next chunk of data.
+   */
   return useInfiniteQuery(
     [LATEST_VULNERABILITIES_INDEX_PATTERN, options],
     async ({ pageParam }) => {
@@ -105,9 +112,11 @@ export const useLatestVulnerabilities = (options: VulnerabilitiesQuery) => {
       keepPreviousData: true,
       enabled: options.enabled,
       onError: (err: Error) => showErrorToast(toasts, err),
-      getNextPageParam: (lastPage) => {
-        if (lastPage.page.length === 0) return undefined;
-        return lastPage.page[lastPage.page.length - 1].raw.sort;
+      getNextPageParam: (lastPage, allPages) => {
+        if (lastPage.page.length < options.pageSize) {
+          return undefined;
+        }
+        return allPages.length * options.pageSize;
       },
     }
   );

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_table.tsx

@@ -28,7 +28,7 @@ export const useLatestVulnerabilitiesTable = ({
     nonPersistedFilters,
   });
 
-  const { query, sort, queryError, getRowsFromPages } = cloudPostureDataTable;
+  const { query, sort, queryError, getRowsFromPages, pageSize } = cloudPostureDataTable;
 
   const {
     data,
@@ -39,6 +39,7 @@ export const useLatestVulnerabilitiesTable = ({
     query,
     sort,
     enabled: !queryError,
+    pageSize,
   });
 
   const rows = useMemo(() => getRowsFromPages(data?.pages), [data?.pages, getRowsFromPages]);

x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/latest_vulnerabilities_container.tsx

@@ -7,21 +7,21 @@
 import { Filter } from '@kbn/es-query';
 import React from 'react';
 import { EuiSpacer } from '@elastic/eui';
-import { DEFAULT_TABLE_HEIGHT } from './constants';
 import { useLatestVulnerabilitiesGrouping } from './hooks/use_latest_vulnerabilities_grouping';
 import { LatestVulnerabilitiesTable } from './latest_vulnerabilities_table';
 import { groupPanelRenderer, groupStatsRenderer } from './latest_vulnerabilities_group_renderer';
 import { FindingsSearchBar } from '../configurations/layout/findings_search_bar';
 import { ErrorCallout } from '../configurations/layout/error_callout';
 import { EmptyState } from '../../components/empty_state';
 import { CloudSecurityGrouping } from '../../components/cloud_security_grouping';
+import { DEFAULT_GROUPING_TABLE_HEIGHT } from '../../common/constants';
 
 export const LatestVulnerabilitiesContainer = () => {
   const renderChildComponent = (groupFilters: Filter[]) => {
     return (
       <LatestVulnerabilitiesTable
         nonPersistedFilters={groupFilters}
-        height={DEFAULT_TABLE_HEIGHT}
+        height={DEFAULT_GROUPING_TABLE_HEIGHT}
       />
     );
   };