From 7c811f355749927a2232412458c08c2c08aa50b1 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 18 Jun 2019 08:15:46 -0400 Subject: [PATCH] Move dense_vector and sparse_vector to module (#43280) --- .../mapping/types/dense-vector.asciidoc | 2 + .../mapping/types/sparse-vector.asciidoc | 2 + .../query-dsl/script-score-query.asciidoc | 2 + docs/reference/rest-api/info.asciidoc | 4 ++ .../test/painless/71_context_api.yml | 4 +- .../index/mapper/MapperExtrasPlugin.java | 2 - .../query/DocValuesWhitelistExtension.java | 42 ----------- ...asticsearch.painless.spi.PainlessExtension | 1 - .../index/query/docvalues_whitelist.txt | 32 --------- .../mapper/DenseVectorFieldTypeTests.java | 28 -------- .../mapper/SparseVectorFieldTypeTests.java | 28 -------- .../license/XPackLicenseState.java | 9 +++ .../xpack/core/XPackClientPlugin.java | 5 +- .../elasticsearch/xpack/core/XPackField.java | 2 + .../xpack/core/XPackSettings.java | 4 ++ .../core/vectors/VectorsFeatureSetUsage.java | 24 +++++++ .../test/vectors/10_dense_vector_basic.yml | 0 .../vectors/20_dense_vector_special_cases.yml | 0 .../test/vectors/30_sparse_vector_basic.yml | 0 .../40_sparse_vector_special_cases.yml | 0 x-pack/plugin/vectors/build.gradle | 22 ++++++ .../elasticsearch/xpack/vectors/Vectors.java | 51 ++++++++++++++ .../xpack/vectors/VectorsFeatureSet.java | 54 ++++++++++++++ .../mapper/DenseVectorFieldMapper.java | 30 ++++---- .../mapper/SparseVectorFieldMapper.java | 29 +++----- .../vectors}/mapper/VectorEncoderDecoder.java | 22 ++---- .../query/DocValuesWhitelistExtension.java | 30 ++++++++ .../vectors}/query/ScoreScriptUtils.java | 26 ++----- .../query/VectorDVAtomicFieldData.java | 22 ++---- .../query/VectorDVIndexFieldData.java | 22 ++---- .../vectors}/query/VectorScriptDocValues.java | 22 ++---- ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/vectors/query/whitelist.txt | 18 +++++ .../xpack/vectors/VectorsFeatureSetTests.java | 70 +++++++++++++++++++ .../mapper/DenseVectorFieldMapperTests.java | 32 ++++----- .../mapper/DenseVectorFieldTypeTests.java | 19 +++++ .../mapper/SparseVectorFieldMapperTests.java | 31 ++++---- .../mapper/SparseVectorFieldTypeTests.java | 19 +++++ .../mapper/VectorEncoderDecoderTests.java | 21 ++---- .../vectors}/query/ScoreScriptUtilsTests.java | 33 +++------ 40 files changed, 430 insertions(+), 335 deletions(-) delete mode 100644 modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java delete mode 100644 modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension delete mode 100644 modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt delete mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java delete mode 100644 modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java rename modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_basic.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/10_dense_vector_basic.yml (100%) rename modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/20_special_cases.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/20_dense_vector_special_cases.yml (100%) rename modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_basic.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/30_sparse_vector_basic.yml (100%) rename modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/20_special_cases.yml => x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/40_sparse_vector_special_cases.yml (100%) create mode 100644 x-pack/plugin/vectors/build.gradle create mode 100644 x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java create mode 100644 x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsFeatureSet.java rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/mapper/DenseVectorFieldMapper.java (88%) rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/mapper/SparseVectorFieldMapper.java (89%) rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/mapper/VectorEncoderDecoder.java (88%) create mode 100644 x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/ScoreScriptUtils.java (89%) rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/VectorDVAtomicFieldData.java (69%) rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/VectorDVIndexFieldData.java (72%) rename {modules/mapper-extras/src/main/java/org/elasticsearch/index => x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors}/query/VectorScriptDocValues.java (67%) create mode 100644 x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt create mode 100644 x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsFeatureSetTests.java rename {modules/mapper-extras/src/test/java/org/elasticsearch/index => x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors}/mapper/DenseVectorFieldMapperTests.java (81%) create mode 100644 x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java rename {modules/mapper-extras/src/test/java/org/elasticsearch/index => x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors}/mapper/SparseVectorFieldMapperTests.java (90%) create mode 100644 x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java rename {modules/mapper-extras/src/test/java/org/elasticsearch/index => x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors}/mapper/VectorEncoderDecoderTests.java (81%) rename {modules/mapper-extras/src/test/java/org/elasticsearch/index => x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors}/query/ScoreScriptUtilsTests.java (67%) diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 335c8f16ba9f1..f79bdde9cc4dd 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -1,3 +1,5 @@ +[role="xpack"] +[testenv="basic"] [[dense-vector]] === Dense vector datatype diff --git a/docs/reference/mapping/types/sparse-vector.asciidoc b/docs/reference/mapping/types/sparse-vector.asciidoc index 70b2ce4ed3198..de63a1a822a0a 100644 --- a/docs/reference/mapping/types/sparse-vector.asciidoc +++ b/docs/reference/mapping/types/sparse-vector.asciidoc @@ -1,3 +1,5 @@ +[role="xpack"] +[testenv="basic"] [[sparse-vector]] === Sparse vector datatype diff --git a/docs/reference/query-dsl/script-score-query.asciidoc b/docs/reference/query-dsl/script-score-query.asciidoc index 5fe723c73d71e..dd68d18846028 100644 --- a/docs/reference/query-dsl/script-score-query.asciidoc +++ b/docs/reference/query-dsl/script-score-query.asciidoc @@ -72,6 +72,8 @@ to be the most efficient by using the internal mechanisms. -------------------------------------------------- // NOTCONSOLE +[role="xpack"] +[testenv="basic"] [[vector-functions]] ===== Functions for vector fields diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index db1add3d0310f..b2a3ad09f3641 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -107,6 +107,10 @@ Example response: "available" : true, "enabled" : true }, + "vectors" : { + "available" : true, + "enabled" : true + }, "watcher" : { "available" : true, "enabled" : true diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml index a185ede4db662..0413661fc586c 100644 --- a/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml +++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml @@ -17,6 +17,6 @@ - match: { classes.6.methods.0.parameters.0 : java.lang.CharSequence } - match: { classes.6.methods.0.parameters.1 : int } - match: { classes.6.methods.0.parameters.2 : int } - - match: { imported_methods.0.name: dotProduct } - - match: { class_bindings.0.name: cosineSimilarity } + - match: { imported_methods.0.name: saturation } + - match: { class_bindings.0.name: decayDateExp } - match: { instance_bindings: [] } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java index 45a067d7994d2..569e20a14161f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java @@ -39,8 +39,6 @@ public Map getMappers() { mappers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); mappers.put(RankFeatureFieldMapper.CONTENT_TYPE, new RankFeatureFieldMapper.TypeParser()); mappers.put(RankFeaturesFieldMapper.CONTENT_TYPE, new RankFeaturesFieldMapper.TypeParser()); - mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, new DenseVectorFieldMapper.TypeParser()); - mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, new SparseVectorFieldMapper.TypeParser()); mappers.put(SearchAsYouTypeFieldMapper.CONTENT_TYPE, new SearchAsYouTypeFieldMapper.TypeParser()); return Collections.unmodifiableMap(mappers); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java deleted file mode 100644 index f463135d69f71..0000000000000 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.query; - - -import org.elasticsearch.painless.spi.PainlessExtension; -import org.elasticsearch.painless.spi.Whitelist; -import org.elasticsearch.painless.spi.WhitelistLoader; -import org.elasticsearch.script.ScoreScript; -import org.elasticsearch.script.ScriptContext; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class DocValuesWhitelistExtension implements PainlessExtension { - - private static final Whitelist WHITELIST = - WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "docvalues_whitelist.txt"); - - @Override - public Map, List> getContextWhitelists() { - return Collections.singletonMap(ScoreScript.CONTEXT, Collections.singletonList(WHITELIST)); - } -} diff --git a/modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension deleted file mode 100644 index f4cc27a362e51..0000000000000 --- a/modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension +++ /dev/null @@ -1 +0,0 @@ -org.elasticsearch.index.query.DocValuesWhitelistExtension \ No newline at end of file diff --git a/modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt b/modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt deleted file mode 100644 index 3a8989e20b020..0000000000000 --- a/modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -class org.elasticsearch.index.query.VectorScriptDocValues { -} -class org.elasticsearch.index.query.VectorScriptDocValues$DenseVectorScriptDocValues { -} -class org.elasticsearch.index.query.VectorScriptDocValues$SparseVectorScriptDocValues { -} - -static_import { - double cosineSimilarity(List, VectorScriptDocValues.DenseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$CosineSimilarity - double dotProduct(List, VectorScriptDocValues.DenseVectorScriptDocValues) from_class org.elasticsearch.index.query.ScoreScriptUtils - double dotProductSparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$DotProductSparse - double cosineSimilaritySparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$CosineSimilaritySparse -} \ No newline at end of file diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java deleted file mode 100644 index 7bbf862e0fc2b..0000000000000 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -public class DenseVectorFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType() { - return new DenseVectorFieldMapper.DenseVectorFieldType(); - } -} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java deleted file mode 100644 index 9fe8d17e697e6..0000000000000 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -public class SparseVectorFieldTypeTests extends FieldTypeTestCase { - - @Override - protected MappedFieldType createDefaultFieldType() { - return new SparseVectorFieldMapper.SparseVectorFieldType(); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 32f163a773a8e..de2bb2af491b9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -687,6 +687,15 @@ public synchronized boolean isJdbcAllowed() { return licensed && localStatus.active; } + /** + * Determine if Vectors support should be enabled. + *

+ * Vectors is available for all license types except {@link OperationMode#MISSING} + */ + public synchronized boolean isVectorsAllowed() { + return status.active; + } + /** * Determine if ODBC support should be enabled. *

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index a145569898ee6..2f3eb27ee8e96 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -184,6 +184,7 @@ import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction; import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction; +import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage; import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage; import org.elasticsearch.xpack.core.watcher.WatcherMetaData; import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction; @@ -465,7 +466,9 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_FRAME, DataFrameFeatureSetUsage::new), new NamedWriteableRegistry.Entry(PersistentTaskParams.class, DataFrameField.TASK_NAME, DataFrameTransform::new), new NamedWriteableRegistry.Entry(Task.Status.class, DataFrameField.TASK_NAME, DataFrameTransformState::new), - new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new) + new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new), + // Vectors + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VECTORS, VectorsFeatureSetUsage::new) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index a5baf4d4f9382..e52fca72d4ca9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -37,6 +37,8 @@ public final class XPackField { public static final String CCR = "ccr"; /** Name constant for the data frame feature. */ public static final String DATA_FRAME = "data_frame"; + /** Name constant for the vectors feature. */ + public static final String VECTORS = "vectors"; private XPackField() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 4f6202f2b5c13..81ac6a4202007 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -119,6 +119,9 @@ private XPackSettings() { /** Setting for enabling or disabling sql. Defaults to true. */ public static final Setting SQL_ENABLED = Setting.boolSetting("xpack.sql.enabled", true, Setting.Property.NodeScope); + /** Setting for enabling or disabling vectors. Defaults to true. */ + public static final Setting VECTORS_ENABLED = Setting.boolSetting("xpack.vectors.enabled", true, Setting.Property.NodeScope); + public static final List DEFAULT_SUPPORTED_PROTOCOLS; static { @@ -249,6 +252,7 @@ public static List> getAllSettings() { settings.add(PASSWORD_HASHING_ALGORITHM); settings.add(INDEX_LIFECYCLE_ENABLED); settings.add(DATA_FRAME_ENABLED); + settings.add(VECTORS_ENABLED); return Collections.unmodifiableList(settings); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java new file mode 100644 index 0000000000000..00e42d46fb551 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.vectors; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; + +public class VectorsFeatureSetUsage extends XPackFeatureSet.Usage { + + public VectorsFeatureSetUsage(StreamInput input) throws IOException { + super(input); + } + + public VectorsFeatureSetUsage(boolean available, boolean enabled) { + super(XPackField.VECTORS, available, enabled); + } +} diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/10_dense_vector_basic.yml similarity index 100% rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/10_basic.yml rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/10_dense_vector_basic.yml diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/20_special_cases.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/20_dense_vector_special_cases.yml similarity index 100% rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/dense-vector/20_special_cases.yml rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/20_dense_vector_special_cases.yml diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/30_sparse_vector_basic.yml similarity index 100% rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/10_basic.yml rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/30_sparse_vector_basic.yml diff --git a/modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/20_special_cases.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/40_sparse_vector_special_cases.yml similarity index 100% rename from modules/mapper-extras/src/test/resources/rest-api-spec/test/sparse-vector/20_special_cases.yml rename to x-pack/plugin/src/test/resources/rest-api-spec/test/vectors/40_sparse_vector_special_cases.yml diff --git a/x-pack/plugin/vectors/build.gradle b/x-pack/plugin/vectors/build.gradle new file mode 100644 index 0000000000000..e780bd7f8a1a2 --- /dev/null +++ b/x-pack/plugin/vectors/build.gradle @@ -0,0 +1,22 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'vectors' + description 'A plugin for working with vectors' + classname 'org.elasticsearch.xpack.vectors.Vectors' + extendedPlugins = ['x-pack-core', 'lang-painless'] +} +archivesBaseName = 'x-pack-vectors' + +dependencies { + compileOnly project(':modules:lang-painless:spi') + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + if (isEclipse) { + testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts') + } +} + +integTest.enabled = false diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java new file mode 100644 index 0000000000000..1c24a11303e85 --- /dev/null +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.vectors; + +import org.elasticsearch.common.inject.Module; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.vectors.mapper.DenseVectorFieldMapper; +import org.elasticsearch.xpack.vectors.mapper.SparseVectorFieldMapper; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static java.util.Collections.emptyMap; + +public class Vectors extends Plugin implements MapperPlugin { + + public static final String NAME = "vectors"; + protected final boolean enabled; + + public Vectors(Settings settings) { + this.enabled = XPackSettings.VECTORS_ENABLED.get(settings); + } + + public Collection createGuiceModules() { + return Collections.singletonList(b -> { + XPackPlugin.bindFeatureSet(b, VectorsFeatureSet.class); + }); + } + + @Override + public Map getMappers() { + if (enabled == false) { + return emptyMap(); + } + Map mappers = new LinkedHashMap<>(); + mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, new DenseVectorFieldMapper.TypeParser()); + mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, new SparseVectorFieldMapper.TypeParser()); + return Collections.unmodifiableMap(mappers); + } +} diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsFeatureSet.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsFeatureSet.java new file mode 100644 index 0000000000000..022de96ff2dee --- /dev/null +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsFeatureSet.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.vectors; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage; + +import java.util.Map; + +public class VectorsFeatureSet implements XPackFeatureSet { + + private final boolean enabled; + private final XPackLicenseState licenseState; + + @Inject + public VectorsFeatureSet(Settings settings, XPackLicenseState licenseState) { + this.enabled = XPackSettings.VECTORS_ENABLED.get(settings); + this.licenseState = licenseState; + } + + @Override + public String name() { + return XPackField.VECTORS; + } + + @Override + public boolean available() { + return licenseState != null && licenseState.isVectorsAllowed(); + } + + @Override + public boolean enabled() { + return enabled; + } + + @Override + public Map nativeCodeInfo() { + return null; + } + + @Override + public void usage(ActionListener listener) { + listener.onResponse(new VectorsFeatureSetUsage(available(), enabled())); + } +} diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java similarity index 88% rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index d48a457ba08cd..70e597a9a489b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.mapper; + +package org.elasticsearch.xpack.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexOptions; @@ -29,8 +17,14 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.ArrayValueMapperParser; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.VectorDVIndexFieldData; +import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java similarity index 89% rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 931e27bc1c19f..3c551a4ee525f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.mapper; + +package org.elasticsearch.xpack.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexOptions; @@ -29,8 +17,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.VectorDVIndexFieldData; +import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData; import org.elasticsearch.search.DocValueFormat; import java.io.IOException; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/VectorEncoderDecoder.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoder.java similarity index 88% rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/VectorEncoderDecoder.java rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoder.java index fbf9955f46621..3058ff51434db 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/VectorEncoderDecoder.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoder.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.mapper; + +package org.elasticsearch.xpack.vectors.mapper; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InPlaceMergeSorter; diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java new file mode 100644 index 0000000000000..7d8d1d97da92e --- /dev/null +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +package org.elasticsearch.xpack.vectors.query; + + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class DocValuesWhitelistExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = + WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + return Collections.singletonMap(ScoreScript.CONTEXT, Collections.singletonList(WHITELIST)); + } +} diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/ScoreScriptUtils.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtils.java similarity index 89% rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/ScoreScriptUtils.java rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtils.java index 93e80d2a653fb..e73f40f900bac 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/ScoreScriptUtils.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtils.java @@ -1,32 +1,20 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.query; + +package org.elasticsearch.xpack.vectors.query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.mapper.VectorEncoderDecoder; +import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder; import java.util.Iterator; import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.VectorEncoderDecoder.sortSparseDimsDoubleValues; +import static org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder.sortSparseDimsDoubleValues; public class ScoreScriptUtils { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVAtomicFieldData.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java similarity index 69% rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVAtomicFieldData.java rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java index 99e581ce4e514..201bf00de1ea6 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVAtomicFieldData.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.query; + +package org.elasticsearch.xpack.vectors.query; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVIndexFieldData.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java similarity index 72% rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVIndexFieldData.java rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java index 9badf9f11b443..04922c4131426 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorDVIndexFieldData.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.query; + +package org.elasticsearch.xpack.vectors.query; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.SortField; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorScriptDocValues.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorScriptDocValues.java similarity index 67% rename from modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorScriptDocValues.java rename to x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorScriptDocValues.java index 603881d390718..000caf74656eb 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/query/VectorScriptDocValues.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorScriptDocValues.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.query; + +package org.elasticsearch.xpack.vectors.query; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.util.BytesRef; diff --git a/x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..0c8f2e6cfd023 --- /dev/null +++ b/x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.vectors.query.DocValuesWhitelistExtension \ No newline at end of file diff --git a/x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt b/x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt new file mode 100644 index 0000000000000..d385744e88fd5 --- /dev/null +++ b/x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt @@ -0,0 +1,18 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# +class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues { +} +class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$DenseVectorScriptDocValues { +} +class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$SparseVectorScriptDocValues { +} + +static_import { + double cosineSimilarity(List, VectorScriptDocValues.DenseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$CosineSimilarity + double dotProduct(List, VectorScriptDocValues.DenseVectorScriptDocValues) from_class org.elasticsearch.xpack.vectors.query.ScoreScriptUtils + double dotProductSparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$DotProductSparse + double cosineSimilaritySparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$CosineSimilaritySparse +} \ No newline at end of file diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsFeatureSetTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsFeatureSetTests.java new file mode 100644 index 0000000000000..ce38f99c13849 --- /dev/null +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsFeatureSetTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.vectors; + +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage; +import org.junit.Before; + +import static org.hamcrest.core.Is.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class VectorsFeatureSetTests extends ESTestCase { + + private XPackLicenseState licenseState; + + @Before + public void init() { + licenseState = mock(XPackLicenseState.class); + } + + public void testAvailable() throws Exception { + VectorsFeatureSet featureSet = new VectorsFeatureSet(Settings.EMPTY, licenseState); + boolean available = randomBoolean(); + when(licenseState.isVectorsAllowed()).thenReturn(available); + assertThat(featureSet.available(), is(available)); + + PlainActionFuture future = new PlainActionFuture<>(); + featureSet.usage(future); + XPackFeatureSet.Usage usage = future.get(); + assertThat(usage.available(), is(available)); + + BytesStreamOutput out = new BytesStreamOutput(); + usage.writeTo(out); + XPackFeatureSet.Usage serializedUsage = new VectorsFeatureSetUsage(out.bytes().streamInput()); + assertThat(serializedUsage.available(), is(available)); + } + + public void testEnabled() throws Exception { + boolean enabled = randomBoolean(); + Settings.Builder settings = Settings.builder(); + if (enabled) { + if (randomBoolean()) { + settings.put("xpack.vectors.enabled", enabled); + } + } else { + settings.put("xpack.vectors.enabled", enabled); + } + VectorsFeatureSet featureSet = new VectorsFeatureSet(settings.build(), licenseState); + assertThat(featureSet.enabled(), is(enabled)); + PlainActionFuture future = new PlainActionFuture<>(); + featureSet.usage(future); + XPackFeatureSet.Usage usage = future.get(); + assertThat(usage.enabled(), is(enabled)); + + BytesStreamOutput out = new BytesStreamOutput(); + usage.writeTo(out); + XPackFeatureSet.Usage serializedUsage = new VectorsFeatureSetUsage(out.bytes().streamInput()); + assertThat(serializedUsage.enabled(), is(enabled)); + } + +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java similarity index 81% rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldMapperTests.java rename to x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java index cf6fc99657756..b8a804effe78f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldMapperTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.mapper; + +package org.elasticsearch.xpack.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexableField; @@ -28,8 +16,16 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.vectors.Vectors; + import org.junit.Before; import java.io.IOException; @@ -59,7 +55,7 @@ public void setUpMapper() throws Exception { @Override protected Collection> getPlugins() { - return pluginList(MapperExtrasPlugin.class); + return pluginList(Vectors.class, XPackPlugin.class); } public void testDefaults() throws Exception { diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java new file mode 100644 index 0000000000000..51d4d3f52bd04 --- /dev/null +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +package org.elasticsearch.xpack.vectors.mapper; + +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; + +public class DenseVectorFieldTypeTests extends FieldTypeTestCase { + + @Override + protected MappedFieldType createDefaultFieldType() { + return new DenseVectorFieldMapper.DenseVectorFieldType(); + } +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java similarity index 90% rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldMapperTests.java rename to x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java index 754a6f1a31803..e1e110a750b93 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldMapperTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java @@ -1,23 +1,11 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.mapper; + +package org.elasticsearch.xpack.vectors.mapper; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.index.IndexableField; @@ -28,8 +16,15 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.vectors.Vectors; import org.hamcrest.Matchers; import org.junit.Before; @@ -64,7 +59,7 @@ public void setUpMapper() throws Exception { @Override protected Collection> getPlugins() { - return pluginList(MapperExtrasPlugin.class); + return pluginList(Vectors.class, XPackPlugin.class); } public void testDefaults() throws Exception { diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java new file mode 100644 index 0000000000000..7f6af470d1825 --- /dev/null +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +package org.elasticsearch.xpack.vectors.mapper; + +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; + +public class SparseVectorFieldTypeTests extends FieldTypeTestCase { + + @Override + protected MappedFieldType createDefaultFieldType() { + return new SparseVectorFieldMapper.SparseVectorFieldType(); + } +} diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/VectorEncoderDecoderTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoderTests.java similarity index 81% rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/VectorEncoderDecoderTests.java rename to x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoderTests.java index 9b8a741192c4f..9acbe44630d99 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/VectorEncoderDecoderTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoderTests.java @@ -1,23 +1,10 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.mapper; +package org.elasticsearch.xpack.vectors.mapper; import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/ScoreScriptUtilsTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtilsTests.java similarity index 67% rename from modules/mapper-extras/src/test/java/org/elasticsearch/index/query/ScoreScriptUtilsTests.java rename to x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtilsTests.java index bcdf0387c3f71..538b49977e108 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/query/ScoreScriptUtilsTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtilsTests.java @@ -1,38 +1,25 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.index.query; +package org.elasticsearch.xpack.vectors.query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.mapper.VectorEncoderDecoder; +import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.index.query.ScoreScriptUtils.CosineSimilarity; -import org.elasticsearch.index.query.ScoreScriptUtils.DotProductSparse; -import org.elasticsearch.index.query.ScoreScriptUtils.CosineSimilaritySparse; +import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.CosineSimilarity; +import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.DotProductSparse; +import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.CosineSimilaritySparse; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.VectorEncoderDecoderTests.mockEncodeDenseVector; -import static org.elasticsearch.index.query.ScoreScriptUtils.dotProduct; +import static org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoderTests.mockEncodeDenseVector; +import static org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.dotProduct; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when;