diff --git a/packages/kbn-ai-playground/components/chat.tsx b/packages/kbn-ai-playground/components/chat.tsx index 1dbebe4a34fd24..1e0b30ced7c59a 100644 --- a/packages/kbn-ai-playground/components/chat.tsx +++ b/packages/kbn-ai-playground/components/chat.tsx @@ -8,7 +8,7 @@ import React, { useMemo } from 'react'; -import { Controller, FormProvider, useForm } from 'react-hook-form'; +import { Controller, useFormContext } from 'react-hook-form'; import { EuiButtonIcon, EuiFlexGroup, @@ -22,6 +22,7 @@ import { v4 as uuidv4 } from 'uuid'; import { i18n } from '@kbn/i18n'; +import { ChatSidebar } from './chat_sidebar'; import { useChat } from '../hooks/useChat'; import { ChatForm, ChatFormFields, MessageRole } from '../types'; @@ -31,18 +32,16 @@ import { QuestionInput } from './question_input'; import { TelegramIcon } from './telegram_icon'; import { transformFromChatMessages } from '../utils/transformToMessages'; -import { ChatSidebar } from '@kbn/ai-playground/components/chat_sidebar'; export const Chat = () => { const { euiTheme } = useEuiTheme(); - const form = useForm(); const { control, watch, formState: { isValid, isSubmitting }, resetField, handleSubmit, - } = form; + } = useFormContext(); const { messages, append, stop: stopRequest } = useChat(); const selectedIndicesCount = watch(ChatFormFields.indices, []).length; @@ -55,6 +54,7 @@ export const Chat = () => { indices: data[ChatFormFields.indices].join(), api_key: data[ChatFormFields.openAIKey], citations: data[ChatFormFields.citations].toString(), + elasticsearchQuery: JSON.stringify(data[ChatFormFields.elasticsearchQuery]), }, } ); @@ -74,90 +74,88 @@ export const Chat = () => { ); return ( - - - - - - {/* // Set scroll at the border of parent element*/} - - - + + + + + {/* // Set scroll at the border of parent element*/} + + + - - + + - + - !!rule?.trim(), - }} - render={({ field }) => ( - - ) : ( - - ) - } - /> - )} - /> - - - + !!rule?.trim(), + }} + render={({ field }) => ( + + ) : ( + + ) + } + /> + )} + /> + + + - - - - - - + + + + + ); }; diff --git a/packages/kbn-ai-playground/components/index.ts b/packages/kbn-ai-playground/components/index.ts index a0ec3667581672..491553ffd7093c 100644 --- a/packages/kbn-ai-playground/components/index.ts +++ b/packages/kbn-ai-playground/components/index.ts @@ -9,3 +9,4 @@ export * from './chat'; export * from './empty_index'; +export * from './view_query/view_query_action'; diff --git a/packages/kbn-ai-playground/components/sources_panel/add_indices_field.tsx b/packages/kbn-ai-playground/components/sources_panel/add_indices_field.tsx index b35e6b283b51cb..3194e987e71e99 100644 --- a/packages/kbn-ai-playground/components/sources_panel/add_indices_field.tsx +++ b/packages/kbn-ai-playground/components/sources_panel/add_indices_field.tsx @@ -9,9 +9,9 @@ import { EuiComboBox, EuiFormRow } from '@elastic/eui'; import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; -import { useQueryIndices } from '../../hooks/useQueryIndices'; import { EuiComboBoxOptionOption } from '@elastic/eui/src/components/combo_box/types'; import { IndexName } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { useQueryIndices } from '../../hooks/useQueryIndices'; interface AddIndicesFieldProps { selectedIndices: IndexName[]; diff --git a/packages/kbn-ai-playground/components/sources_panel/fields_panel.tsx b/packages/kbn-ai-playground/components/sources_panel/fields_panel.tsx deleted file mode 100644 index 440e7b849586dc..00000000000000 --- a/packages/kbn-ai-playground/components/sources_panel/fields_panel.tsx +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React, { useEffect } from 'react'; -import { i18n } from '@kbn/i18n'; -import { useState } from 'react'; -import { - EuiFlyout, - EuiFlyoutHeader, - EuiFlyoutBody, - EuiTitle, - EuiCodeBlock, - EuiFlexItem, - EuiFlexGroup, - EuiLink, -} from '@elastic/eui'; -import { getElasticsearchQuery } from '../../state/generate_query'; -import { useIndicesFields } from '../../hooks/useIndicesFields'; - -interface FieldsPanelProps { - indices: string[]; -} - -export const FieldsPanel: React.FC = ({ indices }) => { - const { fields, isLoading } = useIndicesFields(indices); - const [queryFields, setQueryFields] = useState([]); - const [showFlyout, setShowFlyout] = useState(false); - - useEffect(() => { - setQueryFields([]); - }, [indices]); - - const toggleQueryField = (field: string) => { - if (queryFields.includes(field)) { - setQueryFields(queryFields.filter((x) => x !== field)); - } else { - setQueryFields([...queryFields, field]); - } - }; - - let flyout; - - if (showFlyout && fields) { - flyout = ( - setShowFlyout(false)}> - - -

View Query

-
-
- - - - - {JSON.stringify(getElasticsearchQuery(queryFields, fields), null, 2)} - - - - {Object.keys(fields).map((index: string) => { - const group = fields[index]; - return ( - <> -

{index}

- {[...group.elser_query_fields, ...group.dense_vector_query_fields].map( - (field) => { - return ( - toggleQueryField(field.field)}> - {field.field} ({field.model_id}) - - ); - } - )} - {group.bm25_query_fields.map((field) => { - return toggleQueryField(field)}>{field}; - })} - - ); - })} -
-
-
-
- ); - } - - return ( -
- {flyout} - setShowFlyout(true)}>Edit Query -
- ); -}; diff --git a/packages/kbn-ai-playground/components/sources_panel/sources_panel_sidebar.tsx b/packages/kbn-ai-playground/components/sources_panel/sources_panel_sidebar.tsx index 279233a55c703d..5ab9a65d5f3db0 100644 --- a/packages/kbn-ai-playground/components/sources_panel/sources_panel_sidebar.tsx +++ b/packages/kbn-ai-playground/components/sources_panel/sources_panel_sidebar.tsx @@ -1,23 +1,43 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiCallOut, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import { IndicesList } from './indices_list'; -import { AddIndicesField } from './add_indices_field'; import { IndexName } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { useController } from 'react-hook-form'; +import { useIndicesFields } from '../../hooks/useIndicesFields'; +import { createQuery, getDefaultQueryFields } from '../../lib/create_query'; import { ChatFormFields } from '../../types'; +import { AddIndicesField } from './add_indices_field'; +import { IndicesList } from './indices_list'; export const SourcesPanelSidebar: React.FC = () => { const { field: { value: selectedIndices, onChange }, } = useController({ name: ChatFormFields.indices, defaultValue: [] }); + + const { fields } = useIndicesFields(selectedIndices || []); + + const { + field: { onChange: elasticsearchQueryOnChange }, + } = useController({ + name: ChatFormFields.elasticsearchQuery, + defaultValue: {}, + }); + + useEffect(() => { + if (fields) { + const defaultFields = getDefaultQueryFields(fields); + elasticsearchQueryOnChange(createQuery(defaultFields, fields)); + } + }, [selectedIndices, fields, elasticsearchQueryOnChange]); + const addIndex = (newIndex: IndexName) => { onChange([...selectedIndices, newIndex]); }; diff --git a/packages/kbn-ai-playground/components/view_query/view_query_action.tsx b/packages/kbn-ai-playground/components/view_query/view_query_action.tsx new file mode 100644 index 00000000000000..62fde840b93909 --- /dev/null +++ b/packages/kbn-ai-playground/components/view_query/view_query_action.tsx @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { useEffect, useMemo, useState } from 'react'; +import { + EuiFlyout, + EuiFlyoutHeader, + EuiFlyoutBody, + EuiTitle, + EuiCodeBlock, + EuiFlexItem, + EuiFlexGroup, + EuiButtonEmpty, + EuiLink, + EuiButton, +} from '@elastic/eui'; +import { useController, useFormContext } from 'react-hook-form'; +import { ChatForm, ChatFormFields } from '../../types'; +import { useIndicesFields } from '../../hooks/useIndicesFields'; +import { createQuery, getDefaultQueryFields } from '../../lib/create_query'; + +interface ViewQueryActionProps {} + +export const ViewQueryAction: React.FC = () => { + const { getValues } = useFormContext(); + const [showFlyout, setShowFlyout] = useState(false); + const selectedIndices: string[] = getValues('indices'); + const { fields } = useIndicesFields(selectedIndices || []); + const defaultFields = useMemo(() => getDefaultQueryFields(fields), [fields]); + const [queryFields, setQueryFields] = useState(defaultFields); + + const { + field: { onChange }, + } = useController({ + name: ChatFormFields.elasticsearchQuery, + defaultValue: {}, + }); + + useEffect(() => { + if (selectedIndices?.length > 0) { + setQueryFields(defaultFields); + } + }, [selectedIndices, defaultFields]); + + const isQueryFieldSelected = (index: string, field: string) => { + return queryFields[index].includes(field); + }; + + const toggleQueryField = (index: string, field: string) => { + if (isQueryFieldSelected(index, field)) { + setQueryFields({ + ...queryFields, + [index]: queryFields[index].filter((x: string) => x !== field), + }); + } else { + setQueryFields({ + ...queryFields, + [index]: [...queryFields[index], field], + }); + } + }; + + const saveQuery = () => { + onChange(createQuery(queryFields, fields)); + setShowFlyout(false); + }; + + let flyout; + + if (showFlyout) { + flyout = ( + setShowFlyout(false)}> + + +

View Query

+
+
+ + + + + {JSON.stringify(createQuery(queryFields, fields), null, 2)} + + + + {Object.keys(fields).map((index: string) => { + const group = fields[index]; + return ( + <> +

{index}

+
+ {[...group.elser_query_fields, ...group.dense_vector_query_fields].map( + (field) => { + return ( + toggleQueryField(index, field.field)} + color={isQueryFieldSelected(index, field.field) ? 'primary' : 'text'} + > + {field.field} ({field.model_id}) + + ); + } + )} + {group.bm25_query_fields.map((field) => { + return ( + toggleQueryField(index, field)} + color={isQueryFieldSelected(index, field) ? 'primary' : 'text'} + > + {field} + + ); + })} + + ); + })} +
+
+ + + setShowFlyout(false)}>Close + + + Save + + +
+
+ ); + } + + return ( + <> + {flyout} + {selectedIndices?.length > 0 && ( + setShowFlyout(true)}>View Query + )} + + ); +}; diff --git a/packages/kbn-ai-playground/index.ts b/packages/kbn-ai-playground/index.ts index 6364b0f00b65e7..dc8d3eb22fdad4 100644 --- a/packages/kbn-ai-playground/index.ts +++ b/packages/kbn-ai-playground/index.ts @@ -7,3 +7,4 @@ */ export * from './components'; +export * from './providers/ai_playground_provider'; diff --git a/packages/kbn-ai-playground/lib/create_query.test.ts b/packages/kbn-ai-playground/lib/create_query.test.ts new file mode 100644 index 00000000000000..b450631de846ca --- /dev/null +++ b/packages/kbn-ai-playground/lib/create_query.test.ts @@ -0,0 +1,382 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IndicesQuerySourceFields } from '../types'; +import { createQuery, getDefaultQueryFields } from './create_query'; + +describe('create_query', () => { + describe('createQuery', () => { + it('should return a query', () => { + const fields = { + index1: ['field1'], + }; + + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(createQuery(fields, fieldDescriptors)).toEqual({ + query: { + bool: { + should: [ + { + text_expansion: { + field1: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }); + }); + + it('should return a query from multiple ', () => { + const fields = { + index1: ['field1'], + index2: ['field1'], + }; + + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [], + }, + index2: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(createQuery(fields, fieldDescriptors)).toEqual({ + query: { + bool: { + should: [ + { + text_expansion: { + field1: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }); + }); + + it('should return a query from multiple fields', () => { + const fields = { + index1: ['field1'], + index2: ['field2'], + }; + + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [], + }, + index2: { + elser_query_fields: [{ field: 'field2', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(createQuery(fields, fieldDescriptors)).toEqual({ + query: { + bool: { + should: [ + { + text_expansion: { + field1: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + { + text_expansion: { + field2: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }); + }); + + it('should return a hybrid query', () => { + const fields = { + index1: ['field1', 'content', 'title'], + index2: ['field2'], + }; + + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: ['content', 'title'], + source_fields: [], + }, + index2: { + elser_query_fields: [{ field: 'field2', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(createQuery(fields, fieldDescriptors)).toEqual({ + query: { + bool: { + should: [ + { + text_expansion: { + field1: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + { + multi_match: { + query: '{query}', + fields: ['content', 'title'], + }, + }, + { + text_expansion: { + field2: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + }); + }); + + it('dense vector only', () => { + const fields = { + index1: ['field1'], + }; + + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [], + dense_vector_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + bm25_query_fields: ['content', 'title'], + source_fields: [], + }, + index2: { + elser_query_fields: [{ field: 'field2', model_id: 'model1', nested: false }], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(createQuery(fields, fieldDescriptors)).toEqual({ + knn: [ + { + field: 'field1', + k: 10, + num_candidates: 100, + query_vector_builder: { + text_embedding: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + ], + }); + }); + + it('dense vector + bm25 only', () => { + const fields = { + index1: ['field1', 'title', 'content'], + }; + + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [], + dense_vector_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + bm25_query_fields: ['content', 'title'], + source_fields: [], + }, + }; + + expect(createQuery(fields, fieldDescriptors)).toEqual({ + query: { + bool: { + should: [ + { + multi_match: { + query: '{query}', + fields: ['title', 'content'], + }, + }, + ], + minimum_should_match: 1, + }, + }, + knn: [ + { + field: 'field1', + k: 10, + num_candidates: 100, + query_vector_builder: { + text_embedding: { + model_id: 'model1', + model_text: '{query}', + }, + }, + }, + ], + }); + }); + }); + + describe('getDefaultQueryFields', () => { + it('should return default ELSER query fields', () => { + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [{ field: 'field1', model_id: 'dense_model', nested: false }], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(getDefaultQueryFields(fieldDescriptors)).toEqual({ index1: ['field1'] }); + }); + + it('should return default elser query fields for multiple indices', () => { + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [ + { field: 'dv_field1', model_id: 'dense_model', nested: false }, + ], + bm25_query_fields: [], + source_fields: [], + }, + index2: { + elser_query_fields: [{ field: 'vector', model_id: 'model1', nested: false }], + dense_vector_query_fields: [ + { field: 'dv_field1', model_id: 'dense_model', nested: false }, + ], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(getDefaultQueryFields(fieldDescriptors)).toEqual({ + index1: ['field1'], + index2: ['vector'], + }); + }); + + it('should return elser query fields for default fields', () => { + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [{ field: 'field1', model_id: 'model1', nested: false }], + dense_vector_query_fields: [ + { field: 'dv_field1', model_id: 'dense_model', nested: false }, + ], + bm25_query_fields: [], + source_fields: [], + }, + index2: { + elser_query_fields: [{ field: 'vector', model_id: 'model1', nested: false }], + dense_vector_query_fields: [ + { field: 'dv_field1', model_id: 'dense_model', nested: false }, + ], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(getDefaultQueryFields(fieldDescriptors)).toEqual({ + index1: ['field1'], + index2: ['vector'], + }); + }); + + it('should fallback to dense vector fields for index', () => { + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [], + dense_vector_query_fields: [ + { field: 'dv_field1', model_id: 'dense_model', nested: false }, + ], + bm25_query_fields: [], + source_fields: [], + }, + }; + + expect(getDefaultQueryFields(fieldDescriptors)).toEqual({ index1: ['dv_field1'] }); + }); + + it('should fallback to all BM25 fields in index, using suggested fields', () => { + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [], + dense_vector_query_fields: [], + bm25_query_fields: ['title', 'text', 'content'], + source_fields: [], + }, + }; + + expect(getDefaultQueryFields(fieldDescriptors)).toEqual({ + index1: ['title', 'text', 'content'], + }); + }); + + it('should fallback to all BM25 fields in index, only using first unrecognised field', () => { + const fieldDescriptors: IndicesQuerySourceFields = { + index1: { + elser_query_fields: [], + dense_vector_query_fields: [], + bm25_query_fields: ['unknown1', 'unknown2'], + source_fields: [], + }, + }; + + expect(getDefaultQueryFields(fieldDescriptors)).toEqual({ + index1: ['unknown1'], + }); + }); + }); +}); diff --git a/packages/kbn-ai-playground/lib/create_query.ts b/packages/kbn-ai-playground/lib/create_query.ts index fbe9eaeeca19b8..613e5b26a9c731 100644 --- a/packages/kbn-ai-playground/lib/create_query.ts +++ b/packages/kbn-ai-playground/lib/create_query.ts @@ -10,32 +10,166 @@ import { IndicesQuerySourceFields } from '../types'; type IndexFields = Record; +// These fields are used to suggest the fields to use for the query +// If the field is not found in the suggested fields, +// we will use the first field for BM25 and all fields for vectors +const SUGGESTED_SPARSE_FIELDS = [ + 'vector.tokens', // LangChain field +]; + +const SUGGESTED_BM25_FIELDS = ['title', 'body_content', 'text', 'content']; + +const SUGGESTED_DENSE_VECTOR_FIELDS = ['content_vector.tokens']; + +interface Matches { + queryMatches: any[]; + knnMatches: any[]; +} + export function createQuery(fields: IndexFields, fieldDescriptors: IndicesQuerySourceFields) { - const boolMatches = Object.keys(fields).reduce((acc, index) => { - const indexFields = fields[index]; - const indexFieldDescriptors = fieldDescriptors[index]; - - const matchRules = indexFields.map((field) => { - const elserField = indexFieldDescriptors.elser_query_fields.find((x) => x.field === field); - if (elserField) { - return { - text_expansion: { - [elserField.field]: { - model_id: elserField.model_id, - model_text: '{query}', + const boolMatches = Object.keys(fields).reduce( + (acc, index) => { + const indexFields = fields[index]; + const indexFieldDescriptors = fieldDescriptors[index]; + + const sparseMatches = + indexFields.map((field) => { + const elserField = indexFieldDescriptors.elser_query_fields.find( + (x) => x.field === field + ); + + if (elserField) { + // when another index has the same field, we don't want to duplicate the match rule + const hasExistingSparseMatch = acc.queryMatches.find( + (x: any) => + x?.text_expansion?.[field] && + x?.text_expansion?.[field].model_id === elserField?.model_id + ); + + if (hasExistingSparseMatch) { + return null; + } + + return { + text_expansion: { + [elserField.field]: { + model_id: elserField.model_id, + model_text: '{query}', + }, + }, + }; + } + return null; + }) || []; + + const bm25Fields = indexFields.filter((field) => + indexFieldDescriptors.bm25_query_fields.includes(field) + ); + + const bm25Match = + bm25Fields.length > 0 + ? { + multi_match: { + query: '{query}', + fields: bm25Fields, + }, + } + : null; + + const knnMatches = indexFields + .map((field) => { + const denseVectorField = indexFieldDescriptors.dense_vector_query_fields.find( + (x) => x.field === field + ); + + if (denseVectorField) { + return { + field: denseVectorField.field, + k: 10, + num_candidates: 100, + query_vector_builder: { + text_embedding: { + model_id: denseVectorField.model_id, + model_text: '{query}', + }, + }, + }; + } + return null; + }) + .filter((x) => !!x); + + const matches = [...sparseMatches, bm25Match].filter((x) => !!x); + + return { + queryMatches: [...acc.queryMatches, ...matches], + knnMatches: [...acc.knnMatches, ...knnMatches], + }; + }, + { + queryMatches: [], + knnMatches: [], + } + ); + + return { + ...(boolMatches.queryMatches.length > 0 + ? { + query: { + bool: { + should: boolMatches.queryMatches, + minimum_should_match: 1, }, }, - }; - } - }); + } + : {}), + ...(boolMatches.knnMatches.length > 0 ? { knn: boolMatches.knnMatches } : {}), + }; +} - return [...acc, ...matchRules]; - }, []); +export function getDefaultQueryFields(fieldDescriptors: IndicesQuerySourceFields): IndexFields { + const indexFields = Object.keys(fieldDescriptors).reduce( + (acc: IndexFields, index: string) => { + const indexFieldDescriptors = fieldDescriptors[index]; + const fields: string[] = []; - return { - bool: { - should: boolMatches, - minimum_should_match: 1, + if (indexFieldDescriptors.elser_query_fields.length > 0) { + const suggested = indexFieldDescriptors.elser_query_fields.filter((x) => + SUGGESTED_SPARSE_FIELDS.includes(x.field) + ); + if (suggested.length > 0) { + fields.push(...suggested.map((x) => x.field)); + } else { + fields.push(...indexFieldDescriptors.elser_query_fields.map((x) => x.field)); + } + } else if (indexFieldDescriptors.dense_vector_query_fields.length > 0) { + const suggested = indexFieldDescriptors.dense_vector_query_fields.filter((x) => + SUGGESTED_DENSE_VECTOR_FIELDS.includes(x.field) + ); + + if (suggested.length > 0) { + fields.push(...suggested.map((x) => x.field)); + } else { + fields.push(...indexFieldDescriptors.dense_vector_query_fields.map((x) => x.field)); + } + } else if (indexFieldDescriptors.bm25_query_fields.length > 0) { + const suggested = indexFieldDescriptors.bm25_query_fields.filter((x) => + SUGGESTED_BM25_FIELDS.includes(x) + ); + if (suggested.length > 0) { + fields.push(...suggested); + } else { + fields.push(indexFieldDescriptors.bm25_query_fields[0]); + } + } + + return { + ...acc, + [index]: fields, + }; }, - }; + {} + ); + + return indexFields; } diff --git a/packages/kbn-ai-playground/lib/fetch_query_source_fields.test.ts b/packages/kbn-ai-playground/lib/fetch_query_source_fields.test.ts new file mode 100644 index 00000000000000..cfd8a82b42caa9 --- /dev/null +++ b/packages/kbn-ai-playground/lib/fetch_query_source_fields.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + ELSER_PASSAGE_CHUNKED_TWO_INDICES, + ELSER_PASSAGE_CHUNKED_TWO_INDICES_DOCS, +} from './fetch_query_source_fields_mock'; +import { parseFieldsCapabilities } from './fetch_query_source_fields'; + +describe('fetch_query_source_fields', () => { + describe('parseFieldsCapabilities', () => { + it("should return the correct fields for the index 'workplace_index'", () => { + expect( + parseFieldsCapabilities(ELSER_PASSAGE_CHUNKED_TWO_INDICES, [ + { + index: 'workplace_index2', + doc: ELSER_PASSAGE_CHUNKED_TWO_INDICES_DOCS[0], + }, + { + index: 'workplace_index', + doc: ELSER_PASSAGE_CHUNKED_TWO_INDICES_DOCS[1], + }, + ]) + ).toEqual({ + workplace_index: { + elser_query_fields: [ + { + field: 'content_vector.tokens', + model_id: '.elser_model_2', + nested: false, + }, + ], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [ + 'metadata.summary', + 'content', + 'metadata.rolePermissions', + 'content_vector.model_id', + 'metadata.name', + ], + }, + workplace_index2: { + elser_query_fields: [ + { + field: 'vector.tokens', + model_id: '.elser_model_2', + nested: false, + }, + ], + dense_vector_query_fields: [], + bm25_query_fields: [], + source_fields: [ + 'metadata.summary', + 'vector.model_id', + 'metadata.rolePermissions', + 'text', + 'metadata.name', + ], + }, + }); + }); + }); +}); diff --git a/packages/kbn-ai-playground/lib/fetch_query_source_fields.ts b/packages/kbn-ai-playground/lib/fetch_query_source_fields.ts index 25568b2551feb4..c348e097a5467e 100644 --- a/packages/kbn-ai-playground/lib/fetch_query_source_fields.ts +++ b/packages/kbn-ai-playground/lib/fetch_query_source_fields.ts @@ -73,7 +73,7 @@ export const parseFieldsCapabilities = ( // if the field is present in all indices, the indices property is not present const indicesPresentIn: string[] = 'unmapped' in field - ? indices.filter((index) => field.unmapped.indices!.includes(index)) + ? indices.filter((index) => !field.unmapped.indices!.includes(index)) : (indices as unknown as string[]); for (const index of indicesPresentIn) { @@ -93,6 +93,7 @@ export const parseFieldsCapabilities = ( }; acc[index].dense_vector_query_fields.push(denseVectorField); } else if ('text' in field && field.text.searchable) { + acc[index].bm25_query_fields.push(fieldKey); acc[index].source_fields.push(fieldKey); } } diff --git a/packages/kbn-ai-playground/lib/fetch_query_source_fields_mock.ts b/packages/kbn-ai-playground/lib/fetch_query_source_fields_mock.ts index d06128ebc01996..0cfba82ffa961a 100644 --- a/packages/kbn-ai-playground/lib/fetch_query_source_fields_mock.ts +++ b/packages/kbn-ai-playground/lib/fetch_query_source_fields_mock.ts @@ -6,6 +6,46 @@ * Side Public License, v 1. */ +export const ELSER_PASSAGE_CHUNKED_TWO_INDICES_DOCS = [ + { + _index: 'workplace_index', + _id: '248629d8-64d7-4e91-a4eb-dbd8282d9f24', + _score: 1, + _ignored: ['metadata.summary.keyword', 'text.keyword'], + _source: { + metadata: { + summary: 'This policy', + rolePermissions: ['demo', 'manager'], + name: 'Work From Home Policy', + }, + vector: { + tokens: {}, + model_id: '.elser_model_2', + }, + text: 'Effective: March 2020', + }, + }, + { + _index: 'workplace_index2', + _id: 'b047762c-24eb-4846-aeb5-808346d54c54', + _score: 1, + _ignored: ['content.keyword', 'metadata.summary.keyword'], + _source: { + metadata: { + summary: + 'This policy outlines the guidelines for full-time remote work, including eligibility, equipment and resources, workspace requirements, communication expectations, performance expectations, time tracking and overtime, confidentiality and data security, health and well-being, and policy reviews and updates. Employees are encouraged to direct any questions or concerns', + rolePermissions: ['demo', 'manager'], + name: 'Work From Home Policy', + }, + content: 'Effective', + content_vector: { + tokens: {}, + model_id: '.elser_model_2', + }, + }, + }, +]; + export const ELSER_PASSAGE_CHUNKED_TWO_INDICES = { indices: ['workplace_index', 'workplace_index2'], fields: { diff --git a/packages/kbn-ai-playground/providers/ai_playground_provider.tsx b/packages/kbn-ai-playground/providers/ai_playground_provider.tsx new file mode 100644 index 00000000000000..8a08e1e286ba89 --- /dev/null +++ b/packages/kbn-ai-playground/providers/ai_playground_provider.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import React, { ReactNode } from 'react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { FormProvider, useForm } from 'react-hook-form'; +import { ChatForm } from '../types'; + +interface AIPlaygroundProviderProps { + children: ReactNode; +} + +const queryClient = new QueryClient({}); + +export const AIPlaygroundProvider: React.FC = ({ children }) => { + const form = useForm(); + + return ( + <> + + {children} + + + ); +}; diff --git a/packages/kbn-ai-playground/state/generate_query.ts b/packages/kbn-ai-playground/state/generate_query.ts deleted file mode 100644 index 8f794ff374accb..00000000000000 --- a/packages/kbn-ai-playground/state/generate_query.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { QueryDslTextExpansionQuery } from '@elastic/elasticsearch/lib/api/types'; -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { IndicesQuerySourceFields } from '../types'; - -export const getElasticsearchQuery = ( - queryFields: string[], - fieldsDescriptor: IndicesQuerySourceFields -) => { - const boolMatches = Object.keys(fieldsDescriptor).reduce( - (acc, index: string) => { - const indexFieldDescriptors = fieldsDescriptor[index]; - - const matchRules: QueryDslQueryContainer[] = queryFields - .map((field) => { - const elserField = indexFieldDescriptors.elser_query_fields.find( - (x) => x.field === field - ); - if (elserField) { - return { - text_expansion: { - [elserField.field]: { - model_id: elserField.model_id, - model_text: '{query}', - }, - } as unknown as QueryDslTextExpansionQuery, - }; - } - return {}; - }) - .filter((x) => Object.keys(x).length > 0); - - return [...acc, ...matchRules]; - }, - [] - ); - - return { - bool: { - should: boolMatches, - minimum_should_match: 1, - }, - }; -}; diff --git a/packages/kbn-ai-playground/types.ts b/packages/kbn-ai-playground/types.ts index 36b050c9ae5adc..c21b2357a8c80d 100644 --- a/packages/kbn-ai-playground/types.ts +++ b/packages/kbn-ai-playground/types.ts @@ -13,6 +13,7 @@ import { IndicesStatsIndexMetadataState, Uuid, HealthStatus, + QueryDslQueryContainer, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export enum MessageRole { @@ -45,6 +46,7 @@ export enum ChatFormFields { prompt = 'prompt', openAIKey = 'api_key', indices = 'indices', + elasticsearchQuery = 'elasticsearch_query', } export interface ChatForm { @@ -52,7 +54,8 @@ export interface ChatForm { [ChatFormFields.prompt]: string; [ChatFormFields.citations]: boolean; [ChatFormFields.openAIKey]: string; - [ChatFormFields.indices]: IndexName[]; + [ChatFormFields.indices]: string[]; + [ChatFormFields.elasticsearchQuery]: QueryDslQueryContainer; } export interface AIPlaygroundPluginStartDeps { diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/ai_playground/ai_playground.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/ai_playground/ai_playground.tsx index be266a97699614..40b26e2bff5ac2 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/ai_playground/ai_playground.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/ai_playground/ai_playground.tsx @@ -7,11 +7,10 @@ import React, { useCallback, useEffect } from 'react'; -import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { useValues, useActions } from 'kea'; import { EuiPageTemplate } from '@elastic/eui'; -import { Chat, EmptyIndex } from '@kbn/ai-playground'; +import { Chat, EmptyIndex, AIPlaygroundProvider, ViewQueryAction } from '@kbn/ai-playground'; import { i18n } from '@kbn/i18n'; import { KibanaLogic } from '../../../shared/kibana'; @@ -20,8 +19,6 @@ import { EnterpriseSearchContentPageTemplate } from '../layout/page_template'; import { IndicesLogic } from '../search_indices/indices_logic'; -const queryClient = new QueryClient({}); - export const AIPlayground: React.FC = () => { const { fetchIndices } = useActions(IndicesLogic); const { hasNoIndices, isLoading } = useValues(IndicesLogic); @@ -38,24 +35,25 @@ export const AIPlayground: React.FC = () => { }, []); return ( - - + + ], + }} + pageViewTelemetry="AI Playground" + restrictWidth={false} + isLoading={isLoading} + customPageSections + bottomBorder="extended" + > {hasNoIndices ? ( ) : ( @@ -69,7 +67,7 @@ export const AIPlayground: React.FC = () => { )} - - + + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/layout/page_template.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/layout/page_template.tsx index 0bc6dc03c43884..a4d514e0ec1d8b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/layout/page_template.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/layout/page_template.tsx @@ -17,6 +17,7 @@ export const EnterpriseSearchContentPageTemplate: React.FC = children, pageChrome, pageViewTelemetry, + restrictWidth = true, ...pageTemplateProps }) => { return ( @@ -26,7 +27,7 @@ export const EnterpriseSearchContentPageTemplate: React.FC = items: useEnterpriseSearchNav(), name: ENTERPRISE_SEARCH_CONTENT_PLUGIN.NAME, }} - restrictWidth + restrictWidth={restrictWidth} setPageChrome={pageChrome && } > {pageViewTelemetry && (