[Discover 2.0] Updating fetch functions to include local cluster (ope…

…nsearch-project#7542) * Update datasources fetch function to include local cluster * Check for duplicates when fetching external datasources (in the case local cluster is added as a datasource) * Clean up types in DataSetNavigator so items are displayed properly --------- Signed-off-by: Sean Li <lnse@amazon.com> Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
ananzh · Jul 30, 2024 · 1d6f0c3 · 1d6f0c3
1 parent f2eaa6e
commit 1d6f0c3
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 34 deletions.
diff --git a/changelogs/fragments/7542.yml b/changelogs/fragments/7542.yml
@@ -0,0 +1,2 @@
+fix:
+- Fix tables not displaying in navigator and add local cluster to datasources ([#7542](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7542))
diff --git a/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx b/src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx
@@ -67,12 +67,12 @@ interface DataSetNavigatorState {
   externalDataSources: SimpleDataSource[];
   currentDataSourceRef?: SimpleDataSource;
   currentDataSet?: SimpleDataSet;
-  cachedDatabases: any[];
+  cachedDatabases: SimpleObject[];
   cachedTables: SimpleObject[];
 }
 
 interface SelectedDataSetState extends SimpleDataSet {
-  database?: any | undefined;
+  database?: SimpleObject | undefined;
 }
 
 export const DataSetNavigator = (props: DataSetNavigatorProps) => {
@@ -287,7 +287,10 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
       if (status === DirectQueryLoadingStatus.SUCCESS) {
         setNavigatorState((prevState) => ({
           ...prevState,
-          cachedDatabases: dataSourceCache.databases,
+          cachedDatabases: dataSourceCache.databases.map((database) => ({
+            id: database.name,
+            title: database.name,
+          })),
         }));
       } else if (
         status === DirectQueryLoadingStatus.CANCELED ||
@@ -299,7 +302,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
   }, [databasesLoadStatus, selectedDataSetState?.dataSourceRef]);
 
   const handleSelectExternalDataSource = useCallback(
-    async (dataSource) => {
+    async (dataSource: SimpleDataSource) => {
       if (dataSource && dataSource.type === SIMPLE_DATA_SOURCE_TYPES.EXTERNAL) {
         const dataSourceCache = CatalogCacheManager.getOrCreateDataSource(
           dataSource.name,
@@ -317,13 +320,15 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
         } else if (dataSourceCache.status === CachedDataSourceStatus.Updated) {
           setNavigatorState((prevState) => ({
             ...prevState,
-            cachedDatabases: dataSourceCache.databases,
+            cachedDatabases: dataSourceCache.databases.map((database) => ({
+              id: database.name,
+              title: database.name,
+            })),
           }));
         }
         setSelectedDataSetState((prevState) => ({
-          ...prevState,
+          ...prevState!,
           dataSourceRef: dataSource,
-          isExternal: true,
         }));
       }
     },
@@ -332,13 +337,13 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
 
   // Start loading tables for selected database
   const handleSelectExternalDatabase = useCallback(
-    (externalDatabase: SimpleDataSource) => {
-      if (selectedDataSetState?.dataSourceRef && externalDatabase) {
+    (externalDatabase: SimpleObject) => {
+      if (selectedDataSetState?.dataSourceRef && externalDatabase && externalDatabase.title) {
         let databaseCache: CachedDatabase;
         try {
           databaseCache = CatalogCacheManager.getDatabase(
             selectedDataSetState.dataSourceRef.name,
-            externalDatabase.name,
+            externalDatabase.title,
             selectedDataSetState.dataSourceRef.id
           );
         } catch (error) {
@@ -351,7 +356,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
         ) {
           startLoadingTables({
             dataSourceName: selectedDataSetState.dataSourceRef.name,
-            databaseName: externalDatabase.name,
+            databaseName: externalDatabase.title,
             dataSourceMDSId: selectedDataSetState.dataSourceRef.id,
           });
         } else if (databaseCache.status === CachedDataSourceStatus.Updated) {
@@ -380,7 +385,7 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
       try {
         databaseCache = CatalogCacheManager.getDatabase(
           selectedDataSetState.dataSourceRef.name,
-          selectedDataSetState.database,
+          selectedDataSetState.database.title!,
           selectedDataSetState.dataSourceRef.id
         );
       } catch (error) {
@@ -551,10 +556,10 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
       : 'Databases',
     items: [
       ...navigatorState.cachedDatabases.map((db) => ({
-        name: db.name,
+        name: db.title,
         onClick: async () => {
           setSelectedDataSetState((prevState) => ({
-            ...prevState,
+            ...prevState!,
             database: db,
           }));
           await handleSelectExternalDatabase(db);
@@ -662,24 +667,26 @@ export const DataSetNavigator = (props: DataSetNavigatorProps) => {
           createDatabasesPanel(),
           {
             id: 6,
-            title: selectedDataSetState?.database ? selectedDataSetState.database.name : 'Tables',
+            title: selectedDataSetState?.database ? selectedDataSetState.database.title : 'Tables',
             items: [
               ...navigatorState.cachedTables.map((table) => ({
-                name: table.name,
+                name: table.title,
                 onClick: async () => {
-                  const tableObject = {
+                  const tableObject: SimpleDataSet = {
                     ...selectedDataSetState,
                     id: `${selectedDataSetState?.dataSourceRef!.name}.${
-                      selectedDataSetState?.database.name
-                    }.${table.name}`,
+                      selectedDataSetState?.database?.title
+                    }.${table.title}`,
                     title: `${selectedDataSetState?.dataSourceRef!.name}.${
-                      selectedDataSetState?.database.name
-                    }.${table.name}`,
-                    dataSourceRef: {
-                      id: selectedDataSetState?.dataSourceRef!.id,
-                      name: selectedDataSetState?.dataSourceRef!.name,
-                      type: selectedDataSetState?.dataSourceRef!.type,
-                    },
+                      selectedDataSetState?.database?.title
+                    }.${table.title}`,
+                    ...(selectedDataSetState?.dataSourceRef && {
+                      dataSourceRef: {
+                        id: selectedDataSetState?.dataSourceRef!.id,
+                        name: selectedDataSetState?.dataSourceRef!.name,
+                        type: selectedDataSetState?.dataSourceRef!.type,
+                      } as SimpleDataSource,
+                    }),
                     type: SIMPLE_DATA_SET_TYPES.TEMPORARY_ASYNC,
                   };
                   await handleSelectedDataSet(tableObject);

diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_data_sources.ts
@@ -11,9 +11,12 @@ export const fetchDataSources = async (client: SavedObjectsClientContract) => {
     type: 'data-source',
     perPage: 10000,
   });
-  return resp.savedObjects.map((savedObject) => ({
-    id: savedObject.id,
-    name: savedObject.attributes.title,
-    type: 'data-source',
-  })) as SimpleDataSource[];
+  const dataSources: SimpleDataSource[] = [{ id: '', name: 'Local Cluster', type: 'data-source' }];
+  return dataSources.concat([
+    ...(resp.savedObjects.map((savedObject) => ({
+      id: savedObject.id,
+      name: savedObject.attributes.title,
+      type: 'data-source',
+    })) as SimpleDataSource[]),
+  ]);
 };
diff --git a/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts b/src/plugins/data/public/ui/dataset_navigator/lib/utils/fetch_external_data_sources.ts
@@ -4,6 +4,7 @@
  */
 
 import { HttpStart } from 'opensearch-dashboards/public';
+import { DatasourceDetails } from '../types';
 
 export const fetchIfExternalDataSourcesEnabled = async (http: HttpStart) => {
   try {
@@ -19,8 +20,8 @@ export const fetchExternalDataSources = async (http: HttpStart, connectedCluster
     connectedClusters.map(async (cluster) => {
       const dataSources = await http.get(`/api/dataconnections/dataSourceMDSId=${cluster}`);
       return dataSources
-        .filter((dataSource) => dataSource.connector === 'S3GLUE')
-        .map((dataSource) => ({
+        .filter((dataSource: DatasourceDetails) => dataSource.connector === 'S3GLUE')
+        .map((dataSource: DatasourceDetails) => ({
           name: dataSource.name,
           status: dataSource.status,
           dataSourceRef: cluster,
@@ -29,5 +30,17 @@ export const fetchExternalDataSources = async (http: HttpStart, connectedCluster
   );
 
   const flattenedResults = results.flat();
-  return flattenedResults;
+  const uniqueResults = Array.from(
+    flattenedResults
+      .reduce((map, ds) => {
+        const key = `${ds.name}-${ds.status}`;
+        if (!map.has(key) || ds.dataSourceRef === '') {
+          map.set(key, ds);
+        }
+        return map;
+      }, new Map<string, any>())
+      .values()
+  );
+
+  return uniqueResults;
 };