From 6bf2cf67e77deaf3698d46ec530fff5489774a1a Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 8 Dec 2023 21:23:20 +0000 Subject: [PATCH 01/55] Basic view stubs Signed-off-by: Peter Nied --- .../index/views/ViewDefinition.java | 26 ++++++++ .../opensearch/index/views/ViewService.java | 20 ++++++ .../opensearch/index/views/package-info.java | 10 +++ .../action/admin/indices/RestViewAction.java | 63 +++++++++++++++++++ 4 files changed, 119 insertions(+) create mode 100644 server/src/main/java/org/opensearch/index/views/ViewDefinition.java create mode 100644 server/src/main/java/org/opensearch/index/views/ViewService.java create mode 100644 server/src/main/java/org/opensearch/index/views/package-info.java create mode 100644 server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java diff --git a/server/src/main/java/org/opensearch/index/views/ViewDefinition.java b/server/src/main/java/org/opensearch/index/views/ViewDefinition.java new file mode 100644 index 0000000000000..b9fa80204e267 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/views/ViewDefinition.java @@ -0,0 +1,26 @@ +package org.opensearch.index.views; + +import java.util.List; + +import org.joda.time.DateTime; + +public class ViewDefinition /* implements Writeable, ToXContentFragment */ { + + private String identifier; + private String description; + private DateTime createdAt; + private DateTime modifiedAt; + private String query; /** TBD format? */ + private List indexPatterns; + + public static enum Computation { + Materialized, + Projected; + } + + public static class IndexPattern { + private String indexPattern; + + /** Define patterns properties */ + } +} diff --git a/server/src/main/java/org/opensearch/index/views/ViewService.java b/server/src/main/java/org/opensearch/index/views/ViewService.java new file mode 100644 index 0000000000000..30560d67db8ba --- /dev/null +++ b/server/src/main/java/org/opensearch/index/views/ViewService.java @@ -0,0 +1,20 @@ +package org.opensearch.index.views; + +import org.opensearch.common.annotation.ExperimentalApi; + +@ExperimentalApi +public class ViewService { + + public ViewDefinition getViewDefinition(final String id) {} + + public ViewDefinition createOrUpdateViewDefinition(final ViewDefinition view) {} + + public ViewDefinition removeDefinition(final String id) {} + + + public List listViews() {} + + + public + +} diff --git a/server/src/main/java/org/opensearch/index/views/package-info.java b/server/src/main/java/org/opensearch/index/views/package-info.java new file mode 100644 index 0000000000000..2b81ef1b1911a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/views/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Responsible for handling all projection of data in OpenSearch */ +package org.opensearch.index.views; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java new file mode 100644 index 0000000000000..3c74bcc62441e --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.admin.indices; + +import org.opensearch.rest.NamedRoute; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.Booleans; +import org.opensearch.common.logging.DeprecationLogger; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.RestResponse; + +import java.io.IOException; +import java.util.List; + +import static org.opensearch.rest.RestRequest.Method.DELETE; +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.rest.RestRequest.Method.PUT; + + +public class RestViewAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster.views.list").handler(this::handleGet).build(), + new NamedRoute.Builder().path("/views").method(POST).uniqueName("cluster.views.create").handler(this::handlePost).build(), + new NamedRoute.Builder().path("/views/{view_id}").method(GET).uniqueName("cluster.views.get").handler(this::handleSingleGet).build(), + new NamedRoute.Builder().path("/views/{view_id}").method(DELETE).uniqueName("cluster.views.delete").handler(this::handleSingleDelete).build(), + new NamedRoute.Builder().path("/views/{view_id}").method(PUT).uniqueName("cluster.views.update").handler(this::handleSinglePut).build() + ); + } + + @Override + public String getName() { + return "refresh_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> { + // Consume the rest channel + }; + } + + public RestResponse handleGet(final RestRequest r) { return null; } + public RestResponse handlePost(final RestRequest r) { return null; } + + public RestResponse handleSingleGet(final RestRequest r) { return null; } + public RestResponse handleSinglePut(final RestRequest r) { return null; } + public RestResponse handleSingleDelete(final RestRequest r) { return null; } + +} \ No newline at end of file From 3522cc7b45e02e55d1ed3b7bd368b806318ba902 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 14 Dec 2023 22:07:27 +0000 Subject: [PATCH 02/55] Cluster metadat Signed-off-by: Peter Nied --- .../org/opensearch/cluster/ClusterModule.java | 3 + .../opensearch/cluster/metadata/Metadata.java | 34 ++++ .../org/opensearch/cluster/metadata/View.java | 120 ++++++++++++ .../cluster/metadata/ViewMetadata.java | 185 ++++++++++++++++++ .../index/{views => view}/package-info.java | 4 +- .../index/views/ViewDefinition.java | 26 --- .../opensearch/index/views/ViewService.java | 20 -- .../action/admin/indices/RestViewAction.java | 51 +++-- 8 files changed, 379 insertions(+), 64 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/View.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java rename server/src/main/java/org/opensearch/index/{views => view}/package-info.java (66%) delete mode 100644 server/src/main/java/org/opensearch/index/views/ViewDefinition.java delete mode 100644 server/src/main/java/org/opensearch/index/views/ViewService.java diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java index bad881f8bda76..d2f4888ae8971 100644 --- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java @@ -49,6 +49,7 @@ import org.opensearch.cluster.metadata.MetadataMappingService; import org.opensearch.cluster.metadata.MetadataUpdateSettingsService; import org.opensearch.cluster.metadata.RepositoriesMetadata; +import org.opensearch.cluster.metadata.ViewMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; import org.opensearch.cluster.routing.DelayedAllocationService; import org.opensearch.cluster.routing.allocation.AllocationService; @@ -195,6 +196,7 @@ public static List getNamedWriteables() { ComposableIndexTemplateMetadata::readDiffFrom ); registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom); + registerMetadataCustom(entries, ViewMetadata.TYPE, ViewMetadata::new, ViewMetadata::readDiffFrom); registerMetadataCustom(entries, WeightedRoutingMetadata.TYPE, WeightedRoutingMetadata::new, WeightedRoutingMetadata::readDiffFrom); registerMetadataCustom( entries, @@ -292,6 +294,7 @@ public static List getNamedXWriteables() { DataStreamMetadata::fromXContent ) ); + entries.add(new NamedXContentRegistry.Entry(Metadata.Custom.class, new ParseField(ViewMetadata.TYPE), ViewMetadata::fromXContent)); entries.add( new NamedXContentRegistry.Entry( Metadata.Custom.class, diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 1871ed24973c2..69e49e7aec6eb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -831,6 +831,10 @@ public Map dataStreams() { .orElse(Collections.emptyMap()); } + public Map views() { + return Optional.ofNullable((ViewMetadata) this.custom(ViewMetadata.TYPE)).map(ViewMetadata::views).orElse(Collections.emptyMap()); + } + public DecommissionAttributeMetadata decommissionAttributeMetadata() { return custom(DecommissionAttributeMetadata.TYPE); } @@ -1325,6 +1329,36 @@ public Builder removeDataStream(String name) { return this; } + private Map getViews() { + return Optional.ofNullable(customs.get(ViewMetadata.TYPE)) + .map(o -> (ViewMetadata) o) + .map(vmd -> vmd.views()) + .orElse(new HashMap<>()); + } + + public View view(final String viewName) { + return getViews().get(viewName); + } + + public Builder views(final Map views) { + this.customs.put(ViewMetadata.TYPE, new ViewMetadata(views)); + return this; + } + + public Builder put(final View view) { + Objects.requireNonNull(view, "view cannot be null"); + final var replacementViews = new HashMap<>(getViews()); + replacementViews.put(view.name, view); + return views(replacementViews); + } + + public Builder removeView(final String viewName) { + Objects.requireNonNull(viewName, "viewName cannot be null"); + final var replacementViews = new HashMap<>(getViews()); + replacementViews.remove(viewName); + return views(replacementViews); + } + public Custom getCustom(String type) { return customs.get(type); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java new file mode 100644 index 0000000000000..a76c995915029 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -0,0 +1,120 @@ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.AbstractDiffable; +import org.opensearch.cluster.Diff; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; + +@ExperimentalApi +public class View extends AbstractDiffable implements ToXContentObject { + + final String name; + final String description; + final long createdAt; + final long modifiedAt; + final List targets; + + public View(final String name, final String description, final long createdAt, final long modifiedAt, final List targets) { + this.name = name; + this.description = description; + this.createdAt = createdAt; + this.modifiedAt = modifiedAt; + this.targets = targets; + } + + public View(final StreamInput in) throws IOException { + this(in.readString(), in.readString(), in.readVLong(), in.readVLong(), in.readList(Target::new)); + } + + public static Diff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(View::new, in); + } + + public static class Target implements Writeable, ToXContentObject { + + private String indexPattern; + + private Target(final String indexPattern) { + this.indexPattern = indexPattern; + } + + private Target(final StreamInput in) throws IOException { + this(in.readString()); + } + + private static final ParseField INDEX_PATTERN_FIELD = new ParseField("indexPattern"); + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(INDEX_PATTERN_FIELD.getPreferredName(), indexPattern); + builder.endObject(); + return builder; + } + + public static ViewMetadata fromXContent(final XContentParser parser) throws IOException { + return null; // TODO + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(indexPattern); + } + } + + private static final ParseField NAME_FIELD = new ParseField("name"); + private static final ParseField DESCRIPTION_FIELD = new ParseField("description"); + private static final ParseField CREATED_AT_FIELD = new ParseField("createdAt"); + private static final ParseField MODIFIED_AT_FIELD = new ParseField("modifiedAt"); + private static final ParseField TARGETS_FIELD = new ParseField("targets"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view", + args -> new View((String) args[0], (String) args[1], (Long) args[2], (Long) args[3], (List) args[4]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), DESCRIPTION_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), CREATED_AT_FIELD); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODIFIED_AT_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Target.fromXContent(p), TARGETS_FIELD); + } + + public static View fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(NAME_FIELD.getPreferredName(), name); + builder.field(DESCRIPTION_FIELD.getPreferredName(), description); + builder.field(CREATED_AT_FIELD.getPreferredName(), createdAt); + builder.field(MODIFIED_AT_FIELD.getPreferredName(), modifiedAt); + builder.field(TARGETS_FIELD.getPreferredName(), targets); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(name); + out.writeString(description); + out.writeVLong(createdAt); + out.writeVLong(modifiedAt); + out.writeList(targets); + } +} diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java new file mode 100644 index 0000000000000..b2db4c9cb564b --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -0,0 +1,185 @@ +package org.opensearch.cluster.metadata; + +import org.opensearch.Version; +import org.opensearch.cluster.Diff; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.NamedDiff; +import org.opensearch.core.ParseField; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata.MINIMMAL_SUPPORTED_VERSION; + +/** + * TODO: Tests with failures? `./gradlew :server:test` + + - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDeleteIOException + - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsDeleteDedup + - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDeleteNoSuchFileException + - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDelete + * + */ + +public class ViewMetadata implements Metadata.Custom { + + public static final String TYPE = "view"; + private static final ParseField VIEW_FIELD = new ParseField("view"); + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE, + false, + a -> new ViewMetadata((Map) a[0]) + ); + + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { + Map views = new HashMap<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + views.put(p.currentName(), View.fromXContent(p)); + } + return views; + }, VIEW_FIELD); + } + + private final Map views; + + public ViewMetadata(final Map views) { + this.views = views; + } + + public ViewMetadata(final StreamInput in) throws IOException { + this.views = in.readMap(StreamInput::readString, View::new); + } + + public Map views() { + return this.views; + } + + @Override + public Diff diff(final Metadata.Custom before) { + return new ViewMetadata.ViewMetadataDiff((ViewMetadata) before, this); + } + + public static NamedDiff readDiffFrom(final StreamInput in) throws IOException { + return new ViewMetadata.ViewMetadataDiff(in); + } + + @Override + public EnumSet context() { + return Metadata.ALL_CONTEXTS; + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public Version getMinimalSupportedVersion() { + return MINIMMAL_SUPPORTED_VERSION; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeMap(this.views, StreamOutput::writeString, (stream, val) -> val.writeTo(stream)); + } + + public static ViewMetadata fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(VIEW_FIELD.getPreferredName()); + for (Map.Entry entry : views.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + return builder; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public int hashCode() { + return Objects.hash(this.views); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + ViewMetadata other = (ViewMetadata) obj; + return Objects.equals(this.views, other.views); + } + + @Override + public String toString() { + return Strings.toString(MediaTypeRegistry.JSON, this); + } + + /** + * Builder of view metadata. + */ + public static class Builder { + + private final Map views = new HashMap<>(); + + public Builder putDataStream(final View view) { + views.put(view.name, view); + return this; + } + + public ViewMetadata build() { + return new ViewMetadata(views); + } + } + + /** + * A diff between view metadata. + */ + static class ViewMetadataDiff implements NamedDiff { + + final Diff> dataStreamDiff; + + ViewMetadataDiff(ViewMetadata before, ViewMetadata after) { + this.dataStreamDiff = DiffableUtils.diff(before.views, after.views, DiffableUtils.getStringKeySerializer()); + } + + ViewMetadataDiff(StreamInput in) throws IOException { + this.dataStreamDiff = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), View::new, View::readDiffFrom); + } + + @Override + public Metadata.Custom apply(Metadata.Custom part) { + return new ViewMetadata(dataStreamDiff.apply(((ViewMetadata) part).views)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + dataStreamDiff.writeTo(out); + } + + @Override + public String getWriteableName() { + return TYPE; + } + } +} diff --git a/server/src/main/java/org/opensearch/index/views/package-info.java b/server/src/main/java/org/opensearch/index/view/package-info.java similarity index 66% rename from server/src/main/java/org/opensearch/index/views/package-info.java rename to server/src/main/java/org/opensearch/index/view/package-info.java index 2b81ef1b1911a..bb65723bdd5cd 100644 --- a/server/src/main/java/org/opensearch/index/views/package-info.java +++ b/server/src/main/java/org/opensearch/index/view/package-info.java @@ -6,5 +6,5 @@ * compatible open source license. */ -/** Responsible for handling all projection of data in OpenSearch */ -package org.opensearch.index.views; +/** Core classes responsible for handling all view operations */ +package org.opensearch.index.view; diff --git a/server/src/main/java/org/opensearch/index/views/ViewDefinition.java b/server/src/main/java/org/opensearch/index/views/ViewDefinition.java deleted file mode 100644 index b9fa80204e267..0000000000000 --- a/server/src/main/java/org/opensearch/index/views/ViewDefinition.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.opensearch.index.views; - -import java.util.List; - -import org.joda.time.DateTime; - -public class ViewDefinition /* implements Writeable, ToXContentFragment */ { - - private String identifier; - private String description; - private DateTime createdAt; - private DateTime modifiedAt; - private String query; /** TBD format? */ - private List indexPatterns; - - public static enum Computation { - Materialized, - Projected; - } - - public static class IndexPattern { - private String indexPattern; - - /** Define patterns properties */ - } -} diff --git a/server/src/main/java/org/opensearch/index/views/ViewService.java b/server/src/main/java/org/opensearch/index/views/ViewService.java deleted file mode 100644 index 30560d67db8ba..0000000000000 --- a/server/src/main/java/org/opensearch/index/views/ViewService.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.opensearch.index.views; - -import org.opensearch.common.annotation.ExperimentalApi; - -@ExperimentalApi -public class ViewService { - - public ViewDefinition getViewDefinition(final String id) {} - - public ViewDefinition createOrUpdateViewDefinition(final ViewDefinition view) {} - - public ViewDefinition removeDefinition(final String id) {} - - - public List listViews() {} - - - public - -} diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 3c74bcc62441e..e6be71dd93b19 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -8,14 +8,9 @@ package org.opensearch.rest.action.admin.indices; -import org.opensearch.rest.NamedRoute; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.Booleans; -import org.opensearch.common.logging.DeprecationLogger; -import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; @@ -27,7 +22,6 @@ import static org.opensearch.rest.RestRequest.Method.POST; import static org.opensearch.rest.RestRequest.Method.PUT; - public class RestViewAction extends BaseRestHandler { @Override @@ -35,9 +29,21 @@ public List routes() { return List.of( new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster.views.list").handler(this::handleGet).build(), new NamedRoute.Builder().path("/views").method(POST).uniqueName("cluster.views.create").handler(this::handlePost).build(), - new NamedRoute.Builder().path("/views/{view_id}").method(GET).uniqueName("cluster.views.get").handler(this::handleSingleGet).build(), - new NamedRoute.Builder().path("/views/{view_id}").method(DELETE).uniqueName("cluster.views.delete").handler(this::handleSingleDelete).build(), - new NamedRoute.Builder().path("/views/{view_id}").method(PUT).uniqueName("cluster.views.update").handler(this::handleSinglePut).build() + new NamedRoute.Builder().path("/views/{view_id}") + .method(GET) + .uniqueName("cluster.views.get") + .handler(this::handleSingleGet) + .build(), + new NamedRoute.Builder().path("/views/{view_id}") + .method(DELETE) + .uniqueName("cluster.views.delete") + .handler(this::handleSingleDelete) + .build(), + new NamedRoute.Builder().path("/views/{view_id}") + .method(PUT) + .uniqueName("cluster.views.update") + .handler(this::handleSinglePut) + .build() ); } @@ -53,11 +59,24 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC }; } - public RestResponse handleGet(final RestRequest r) { return null; } - public RestResponse handlePost(final RestRequest r) { return null; } + public RestResponse handleGet(final RestRequest r) { + return null; + } + + public RestResponse handlePost(final RestRequest r) { + return null; + } - public RestResponse handleSingleGet(final RestRequest r) { return null; } - public RestResponse handleSinglePut(final RestRequest r) { return null; } - public RestResponse handleSingleDelete(final RestRequest r) { return null; } + public RestResponse handleSingleGet(final RestRequest r) { + return null; + } + + public RestResponse handleSinglePut(final RestRequest r) { + return null; + } + + public RestResponse handleSingleDelete(final RestRequest r) { + return null; + } -} \ No newline at end of file +} From 4e12632fa439e81dc9de5c789e0ba886dec63551 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 15 Dec 2023 22:41:55 +0000 Subject: [PATCH 03/55] Basic request handling Signed-off-by: Peter Nied --- .../cluster/metadata/ViewMetadata.java | 2 +- .../action/admin/indices/RestViewAction.java | 106 +++++++++++++----- 2 files changed, 82 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java index b2db4c9cb564b..fc70f77b18ef9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -28,7 +28,7 @@ - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsDeleteDedup - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDeleteNoSuchFileException - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDelete - * + * */ public class ViewMetadata implements Metadata.Custom { diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index e6be71dd93b19..31d5615e3f1ed 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -8,14 +8,29 @@ package org.opensearch.rest.action.admin.indices; +import joptsimple.internal.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.NamedRoute; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; import java.io.IOException; import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; import static org.opensearch.rest.RestRequest.Method.DELETE; import static org.opensearch.rest.RestRequest.Method.GET; @@ -24,26 +39,27 @@ public class RestViewAction extends BaseRestHandler { + private final static Logger LOG = LogManager.getLogger(RestViewAction.class); + + private static final String VIEW_ID = "view_id"; + + private final ClusterService clusterService; + + @Inject + public RestViewAction(final ClusterService clusterService) { + this.clusterService = clusterService; + } + @Override public List routes() { + final String viewIdParameter = "{" + VIEW_ID + "}"; + return List.of( - new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster.views.list").handler(this::handleGet).build(), - new NamedRoute.Builder().path("/views").method(POST).uniqueName("cluster.views.create").handler(this::handlePost).build(), - new NamedRoute.Builder().path("/views/{view_id}") - .method(GET) - .uniqueName("cluster.views.get") - .handler(this::handleSingleGet) - .build(), - new NamedRoute.Builder().path("/views/{view_id}") - .method(DELETE) - .uniqueName("cluster.views.delete") - .handler(this::handleSingleDelete) - .build(), - new NamedRoute.Builder().path("/views/{view_id}") - .method(PUT) - .uniqueName("cluster.views.update") - .handler(this::handleSinglePut) - .build() + new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster.views.list").build(), + new NamedRoute.Builder().path("/views").method(POST).uniqueName("cluster.views.create").build(), + new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster.views.get").build(), + new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster.views.delete").build(), + new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster.views.update").build() ); } @@ -54,21 +70,61 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - return channel -> { - // Consume the rest channel - }; + return channel -> {}; } - public RestResponse handleGet(final RestRequest r) { - return null; + public RestResponse handleGet(final RestRequest r, final XContentBuilder builder) throws IOException { + final List views = Optional.ofNullable(clusterService.state().getMetadata()) + .map(m -> m.views()) + .map(v -> v.values()) + .map(v -> v.stream().collect(Collectors.toList())) + .orElse(List.of()); + + return new BytesRestResponse(RestStatus.OK, builder.startObject().array("views", views).endObject()); } - public RestResponse handlePost(final RestRequest r) { + public RestResponse handlePost(final RestRequest r, final XContentBuilder builder) throws IOException { + final View view; + try (final XContentParser parser = r.contentParser()) { + view = View.fromXContent(parser); + } + + var task = new ClusterStateUpdateTask() { + @Override + public ClusterState execute(final ClusterState currentState) throws Exception { + return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) + .build(); + } + + @Override + public void onFailure(final String source, final Exception e) { + LOG.error("Unable to create view, due to {}", source, e); + } + }; + + clusterService.submitStateUpdateTask("create_view_task", task); + + // TODO: How to unasync? + // TODO: Handle CREATED vs UPDATED return null; } - public RestResponse handleSingleGet(final RestRequest r) { - return null; + public RestResponse handleSingleGet(final RestRequest r, final XContentBuilder builder) throws IOException { + final String viewId = r.param(VIEW_ID); + + if (Strings.isNullOrEmpty(viewId)) { + return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + } + + final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) + .map(m -> m.views()) + .map(views -> views.get(viewId)); + + if (view.isEmpty()) { + return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + } + + return new BytesRestResponse(RestStatus.OK, builder.startObject().value(view).endObject()); } public RestResponse handleSinglePut(final RestRequest r) { From 3a158633eac21a51d163134e2a0c9c941860bd8b Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 15 Dec 2023 23:58:02 +0000 Subject: [PATCH 04/55] Accepting requests Signed-off-by: Peter Nied --- .../org/opensearch/action/ActionModule.java | 1 + .../org/opensearch/cluster/metadata/View.java | 32 ++++++---- .../main/java/org/opensearch/node/Node.java | 3 + .../java/org/opensearch/rest/NamedRoute.java | 2 +- .../action/admin/indices/RestViewAction.java | 60 ++++++++++++++----- 5 files changed, 69 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 46775466aa615..1512eeb1ff44a 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -409,6 +409,7 @@ import org.opensearch.rest.action.admin.indices.RestUpgradeAction; import org.opensearch.rest.action.admin.indices.RestUpgradeStatusAction; import org.opensearch.rest.action.admin.indices.RestValidateQueryAction; +import org.opensearch.rest.action.admin.indices.RestViewAction; import org.opensearch.rest.action.cat.AbstractCatAction; import org.opensearch.rest.action.cat.RestAliasAction; import org.opensearch.rest.action.cat.RestAllocationAction; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index a76c995915029..184a17a0df23f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -19,11 +19,11 @@ @ExperimentalApi public class View extends AbstractDiffable implements ToXContentObject { - final String name; - final String description; - final long createdAt; - final long modifiedAt; - final List targets; + public final String name; + public final String description; + public final long createdAt; + public final long modifiedAt; + public final List targets; public View(final String name, final String description, final long createdAt, final long modifiedAt, final List targets) { this.name = name; @@ -34,7 +34,7 @@ public View(final String name, final String description, final long createdAt, f } public View(final StreamInput in) throws IOException { - this(in.readString(), in.readString(), in.readVLong(), in.readVLong(), in.readList(Target::new)); + this(in.readString(), in.readOptionalString(), in.readVLong(), in.readVLong(), in.readList(Target::new)); } public static Diff readDiffFrom(StreamInput in) throws IOException { @@ -62,9 +62,17 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.endObject(); return builder; } + + private static final ConstructingObjectParser T_PARSER = new ConstructingObjectParser<>( + "target", + args -> new Target((String) args[0]) + ); + static { + T_PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN_FIELD); + } - public static ViewMetadata fromXContent(final XContentParser parser) throws IOException { - return null; // TODO + public static Target fromXContent(final XContentParser parser) throws IOException { + return T_PARSER.parse(parser, null); } @Override @@ -87,9 +95,9 @@ public void writeTo(StreamOutput out) throws IOException { static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), DESCRIPTION_FIELD); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), CREATED_AT_FIELD); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), MODIFIED_AT_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DESCRIPTION_FIELD); + PARSER.declareLongOrNull(ConstructingObjectParser.optionalConstructorArg(), -1L, CREATED_AT_FIELD); + PARSER.declareLongOrNull(ConstructingObjectParser.optionalConstructorArg(), -1L, MODIFIED_AT_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Target.fromXContent(p), TARGETS_FIELD); } @@ -112,7 +120,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa @Override public void writeTo(final StreamOutput out) throws IOException { out.writeString(name); - out.writeString(description); + out.writeOptionalString(description); out.writeVLong(createdAt); out.writeVLong(modifiedAt); out.writeList(targets); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 4cbf8dc191a9d..8115d583543f8 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -203,6 +203,7 @@ import org.opensearch.repositories.RepositoriesModule; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; +import org.opensearch.rest.action.admin.indices.RestViewAction; import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptModule; @@ -896,6 +897,8 @@ protected Node( ); modules.add(actionModule); + actionModule.getRestController().registerHandler(new RestViewAction(clusterService)); + final RestController restController = actionModule.getRestController(); final NodeResourceUsageTracker nodeResourceUsageTracker = new NodeResourceUsageTracker( diff --git a/server/src/main/java/org/opensearch/rest/NamedRoute.java b/server/src/main/java/org/opensearch/rest/NamedRoute.java index 109f688a4924e..768d290f0fec9 100644 --- a/server/src/main/java/org/opensearch/rest/NamedRoute.java +++ b/server/src/main/java/org/opensearch/rest/NamedRoute.java @@ -144,7 +144,7 @@ private NamedRoute(Builder builder) { "Invalid route name specified. The route name may include the following characters" + " 'a-z', 'A-Z', '0-9', ':', '/', '*', '_' and be less than " + MAX_LENGTH_OF_ACTION_NAME - + " characters" + + " characters, " + builder.uniqueName ); } this.uniqueName = builder.uniqueName; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 31d5615e3f1ed..3800339ec222a 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -24,6 +24,7 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.NamedRoute; +import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; @@ -55,22 +56,39 @@ public List routes() { final String viewIdParameter = "{" + VIEW_ID + "}"; return List.of( - new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster.views.list").build(), - new NamedRoute.Builder().path("/views").method(POST).uniqueName("cluster.views.create").build(), - new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster.views.get").build(), - new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster.views.delete").build(), - new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster.views.update").build() + new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster:views:list").build(), + new NamedRoute.Builder().path("/views").method(POST).uniqueName("cluster:views:create").build(), + new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster:views:get").build(), + new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster:views:delete").build(), + new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster:views:update").build() ); } @Override public String getName() { - return "refresh_action"; + return "view_actions"; } @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - return channel -> {}; + if (!request.hasParam(VIEW_ID)) { + if (request.method() == RestRequest.Method.GET) { + return channel -> channel.sendResponse(handleGet(request, channel.newBuilder())); + } + + if (request.method() == RestRequest.Method.POST) { + return channel -> handlePost(request, channel); + } + + } else if (request.hasParam(VIEW_ID)) { + if (request.method() == RestRequest.Method.GET && request.hasParam(VIEW_ID)) { + return channel -> channel.sendResponse(handleSingleGet(request, channel.newBuilder())); + } + + } + + // Unhandled + return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Unable to process this request")); } public RestResponse handleGet(final RestRequest r, final XContentBuilder builder) throws IOException { @@ -80,16 +98,19 @@ public RestResponse handleGet(final RestRequest r, final XContentBuilder builder .map(v -> v.stream().collect(Collectors.toList())) .orElse(List.of()); - return new BytesRestResponse(RestStatus.OK, builder.startObject().array("views", views).endObject()); + return new BytesRestResponse(RestStatus.OK, builder.startObject().field("views", views).endObject()); } - public RestResponse handlePost(final RestRequest r, final XContentBuilder builder) throws IOException { - final View view; + public RestResponse handlePost(final RestRequest r, final RestChannel channel) throws IOException { + final View inputView; try (final XContentParser parser = r.contentParser()) { - view = View.fromXContent(parser); + inputView = View.fromXContent(parser); } - var task = new ClusterStateUpdateTask() { + final long currentTime = System.currentTimeMillis(); + final View view = new View(inputView.name, inputView.description, currentTime, currentTime, inputView.targets); + + clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { @Override public ClusterState execute(final ClusterState currentState) throws Exception { return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) @@ -99,12 +120,19 @@ public ClusterState execute(final ClusterState currentState) throws Exception { @Override public void onFailure(final String source, final Exception e) { LOG.error("Unable to create view, due to {}", source, e); + channel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.")); } - }; - clusterService.submitStateUpdateTask("create_view_task", task); - - // TODO: How to unasync? + @Override + public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + try { + channel.sendResponse(new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject())); + } catch (final IOException e) { + // TODO? + LOG.error(e); + } + } + }); // TODO: Handle CREATED vs UPDATED return null; } From 23632b6fb6443c3446177d5d70e376731ec58aa7 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 18 Dec 2023 23:35:08 +0000 Subject: [PATCH 05/55] Search rewriting and design dco Signed-off-by: Peter Nied --- .../org/opensearch/action/ActionModule.java | 1 - .../org/opensearch/cluster/metadata/View.java | 4 +- .../org/opensearch/index/view/views-design.md | 32 ++++++ .../java/org/opensearch/rest/NamedRoute.java | 3 +- .../action/admin/indices/RestViewAction.java | 28 +++-- .../admin/indices/RestViewSearchAction.java | 104 ++++++++++++++++++ 6 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/view/views-design.md create mode 100644 server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 1512eeb1ff44a..46775466aa615 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -409,7 +409,6 @@ import org.opensearch.rest.action.admin.indices.RestUpgradeAction; import org.opensearch.rest.action.admin.indices.RestUpgradeStatusAction; import org.opensearch.rest.action.admin.indices.RestValidateQueryAction; -import org.opensearch.rest.action.admin.indices.RestViewAction; import org.opensearch.rest.action.cat.AbstractCatAction; import org.opensearch.rest.action.cat.RestAliasAction; import org.opensearch.rest.action.cat.RestAllocationAction; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index 184a17a0df23f..3546613d9c4dd 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -43,7 +43,7 @@ public static Diff readDiffFrom(StreamInput in) throws IOException { public static class Target implements Writeable, ToXContentObject { - private String indexPattern; + public final String indexPattern; private Target(final String indexPattern) { this.indexPattern = indexPattern; @@ -62,7 +62,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.endObject(); return builder; } - + private static final ConstructingObjectParser T_PARSER = new ConstructingObjectParser<>( "target", args -> new Target((String) args[0]) diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md new file mode 100644 index 0000000000000..c2a859c956adf --- /dev/null +++ b/server/src/main/java/org/opensearch/index/view/views-design.md @@ -0,0 +1,32 @@ +# Views + +Views define how searches are performed against indices on a cluster, uniform data access that is configured separately from the queries. + + +## Design + +```mermaid +sequenceDiagram + participant Client + participant HTTP_Request as ActionHandler + participant Cluster_Metadata as Cluster Metadata Store + participant Data_Store as Indices + + Client->>HTTP_Request: View List/Get/Update/Create/Delete
/views or /views/{view_id} + HTTP_Request->>Cluster_Metadata: Query Views + alt Update/Create/Delete + Cluster_Metadata->>Cluster_Metadata: Refresh Cluster + end + Cluster_Metadata-->>HTTP_Request: Return + HTTP_Request-->>Client: Return + + Client->>HTTP_Request: Search View
/views/{view_id}/search + HTTP_Request->>Cluster_Metadata: Query Views + Cluster_Metadata-->>HTTP_Request: Return + HTTP_Request->>HTTP_Request: Rewrite Search Request + HTTP_Request->>HTTP_Request: Validate Search Request + HTTP_Request->>Data_Store: Search indices + Data_Store-->>HTTP_Request: Return + HTTP_Request-->>Client: Return +``` + diff --git a/server/src/main/java/org/opensearch/rest/NamedRoute.java b/server/src/main/java/org/opensearch/rest/NamedRoute.java index 768d290f0fec9..b477cd8571416 100644 --- a/server/src/main/java/org/opensearch/rest/NamedRoute.java +++ b/server/src/main/java/org/opensearch/rest/NamedRoute.java @@ -144,7 +144,8 @@ private NamedRoute(Builder builder) { "Invalid route name specified. The route name may include the following characters" + " 'a-z', 'A-Z', '0-9', ':', '/', '*', '_' and be less than " + MAX_LENGTH_OF_ACTION_NAME - + " characters, " + builder.uniqueName + + " characters, " + + builder.uniqueName ); } this.uniqueName = builder.uniqueName; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 3800339ec222a..02c6e2adbcafe 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -42,7 +42,7 @@ public class RestViewAction extends BaseRestHandler { private final static Logger LOG = LogManager.getLogger(RestViewAction.class); - private static final String VIEW_ID = "view_id"; + static final String VIEW_ID = "view_id"; private final ClusterService clusterService; @@ -81,14 +81,22 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } } else if (request.hasParam(VIEW_ID)) { - if (request.method() == RestRequest.Method.GET && request.hasParam(VIEW_ID)) { + if (request.method() == RestRequest.Method.GET) { return channel -> channel.sendResponse(handleSingleGet(request, channel.newBuilder())); } + if (request.method() == RestRequest.Method.PUT) { + return channel -> handleSinglePut(request); + } + + if (request.method() == RestRequest.Method.DELETE) { + return channel -> handleSingleDelete(request); + } } - // Unhandled - return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Unable to process this request")); + return channel -> channel.sendResponse( + new BytesRestResponse(RestStatus.BAD_REQUEST, "Unable to process " + request.method() + " on this endpoint " + request.path()) + ); } public RestResponse handleGet(final RestRequest r, final XContentBuilder builder) throws IOException { @@ -120,13 +128,17 @@ public ClusterState execute(final ClusterState currentState) throws Exception { @Override public void onFailure(final String source, final Exception e) { LOG.error("Unable to create view, due to {}", source, e); - channel.sendResponse(new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.")); + channel.sendResponse( + new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.") + ); } @Override public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { try { - channel.sendResponse(new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject())); + channel.sendResponse( + new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject()) + ); } catch (final IOException e) { // TODO? LOG.error(e); @@ -156,11 +168,11 @@ public RestResponse handleSingleGet(final RestRequest r, final XContentBuilder b } public RestResponse handleSinglePut(final RestRequest r) { - return null; + return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); } public RestResponse handleSingleDelete(final RestRequest r) { - return null; + return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); } } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java new file mode 100644 index 0000000000000..cd3b0fc06e508 --- /dev/null +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rest.action.admin.indices; + +import joptsimple.internal.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.BytesRestResponse; +import org.opensearch.rest.NamedRoute; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestCancellableNodeClient; +import org.opensearch.rest.action.RestStatusToXContentListener; +import org.opensearch.rest.action.search.RestSearchAction; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.function.IntConsumer; +import java.util.stream.Collectors; + +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.rest.RestRequest.Method.POST; + +public class RestViewSearchAction extends BaseRestHandler { + + private final static Logger LOG = LogManager.getLogger(RestViewSearchAction.class); + + private static final String VIEW_ID = "view_id"; + + private final ClusterService clusterService; + + @Inject + public RestViewSearchAction(final ClusterService clusterService) { + this.clusterService = clusterService; + } + + @Override + public List routes() { + final String viewIdParameter = "{" + VIEW_ID + "}"; + + return List.of( + new NamedRoute.Builder().path("/views/" + viewIdParameter + "/_search").method(GET).uniqueName("cluster:views:search").build(), + new NamedRoute.Builder().path("/views/" + viewIdParameter + "/_search").method(POST).uniqueName("cluster:views:search").build() + ); + } + + @Override + public String getName() { + return "view_search_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> { + final String viewId = request.param(VIEW_ID); + + if (Strings.isNullOrEmpty(viewId)) { + channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "")); + } + + final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) + .map(m -> m.views()) + .map(views -> views.get(viewId)); + + if (view.isEmpty()) { + channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "")); + } + + final SearchRequest searchRequest = new SearchRequest(); + final IntConsumer setSize = size -> searchRequest.source().size(size); + + request.withContentOrSourceParamParserOrNull( + parser -> RestSearchAction.parseSearchRequest(searchRequest, request, parser, client.getNamedWriteableRegistry(), setSize) + ); + + // TODO: Only allow specific operations that are supported + + final String[] indices = view.get().targets.stream() + .map(target -> target.indexPattern) + .collect(Collectors.toList()) + .toArray(new String[0]); + searchRequest.indices(indices); + + // Resource leak on cancelClient??? Note; is already leaking in + // server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java + final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + cancelClient.execute(SearchAction.INSTANCE, searchRequest, new RestStatusToXContentListener<>(channel)); + }; + } +} From 603134775c06274d76778ff6f90d79c7a94f255c Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 19 Dec 2023 12:59:37 +0000 Subject: [PATCH 06/55] Functional views Signed-off-by: Peter Nied --- .../org/opensearch/index/view/views-design.md | 129 ++++++++++++++++++ .../main/java/org/opensearch/node/Node.java | 2 + .../admin/indices/RestViewSearchAction.java | 6 +- 3 files changed, 134 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md index c2a859c956adf..ee6d10e7af3ef 100644 --- a/server/src/main/java/org/opensearch/index/view/views-design.md +++ b/server/src/main/java/org/opensearch/index/view/views-design.md @@ -30,3 +30,132 @@ sequenceDiagram HTTP_Request-->>Client: Return ``` + +### Local Testing + +``` +curl localhost:9200/abc/_doc \ + -XPOST \ + --header "Content-Type: application/json" \ + --data '{"foo":"bar"}' \ + +curl localhost:9200/views \ + -XPOST \ + --header "Content-Type: application/json" \ + --data '{"name":"hi", "createdAt": -1, "modifiedAt": -1, "targets":[]}' \ + -v + +curl localhost:9200/views \ + -XPOST \ + --header "Content-Type: application/json" \ + --data '{"name":"hi", "createdAt": -1, "modifiedAt": -1, "targets":[{"indexPattern":"abc"}]}' \ + -v + + +curl localhost:9200/views/hi/_search +``` + +### Appendix + +VIEW MODEL +{ + name: STRING, // [Optional] Friendly name resolves to ID + id: STRING, // Non-mutatable identifier + description: STRING, // [Optional] Description of the view + created: DATE, // Creation time of the view + modified: DATE // Last modified time of the view + query: QUERY, // enforced query + filter: QUERY, // P2 enforced query after transformations + targets: [ + { + indexPattern: STRING, // No wildcard/aliases! + // P2 Allow wildcard/aliases query parameter + query: QUERY, // enforced query specific for this target + filter: QUERY, // P2 enforced query specific after transformations + documentTransformer: SCRIPT // P2 Convert the results in some way + } + ], + documentTransformer: SCRIPT // P2 Convert the results in some way +} + + +View Operations +POST /views +GET /views/{view_id} +PUT /views/{view_id} +PATCH /views/{view_id} +DELETE /views/{view_id} + +Enumerate Views +GET /views + +Search Views // P2? +GET /views/_search +POST /views/_search + +Mapping // P2? Need to understand the utility / impact of not having this +GET /views/{view_id}/_mappings +PUT /views/{view_id}/_mappings +PATCH /views/{view_id}/_mappings + +Search Views +GET /views/{view_id}/_search +POST /views/{view_id}/_search + +// Results do not include any fields '_', how to protect leaking data? + + + +### Response on Create/Enumerate/Search + +views: [ + { + name: STRING, + id: STRING, + description: STRING, + created: DATE, + modified: DATE + } +] + + +## Permissions + +Views can be permissioned directly by name or id. User must also have READ permission for all indices targeted by the view, authorized before any of these operations. + +index:views.view.read[:{view_id}] +index:views.view.modify[:{view_id}] +index:views.view.delete[:{view_id}] + +Searching with a view. Read permission of the view is *not* required. + +index:views.view.search[:{view_id}] + +Creation / Enumeration / Searching views is supported. Only the view name and view id are returned. Create view call requires READ permission for all indices targeted by the view. + +index:views.enumerate +index:views.search +index:views.create + +### Internal functionality + +Search does not expose information about the targets or associated mappings. Internally used to resolve all indices and check permissions. + +GET /views/{view_id}/_resolveTargets + +{ +targets: [ + { + index: STRING, + fields: [ + { + fieldName: STRING + } + ], + _view: VIEW_MIN_OBJECT + }, ... +], +fields: [ FIELD_NAME,... ] +} + + diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 8115d583543f8..d9c139be9f915 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -204,6 +204,7 @@ import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.rest.action.admin.indices.RestViewAction; +import org.opensearch.rest.action.admin.indices.RestViewSearchAction; import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptModule; @@ -898,6 +899,7 @@ protected Node( modules.add(actionModule); actionModule.getRestController().registerHandler(new RestViewAction(clusterService)); + actionModule.getRestController().registerHandler(new RestViewSearchAction(clusterService)); final RestController restController = actionModule.getRestController(); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java index cd3b0fc06e508..4fe645dc20053 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java @@ -65,8 +65,8 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_ID); return channel -> { - final String viewId = request.param(VIEW_ID); if (Strings.isNullOrEmpty(viewId)) { channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "")); @@ -87,7 +87,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC parser -> RestSearchAction.parseSearchRequest(searchRequest, request, parser, client.getNamedWriteableRegistry(), setSize) ); - // TODO: Only allow specific operations that are supported + // TODO: Only allow operations that are supported final String[] indices = view.get().targets.stream() .map(target -> target.indexPattern) @@ -95,7 +95,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC .toArray(new String[0]); searchRequest.indices(indices); - // Resource leak on cancelClient??? Note; is already leaking in + // TODO: Look into resource leak on cancelClient? Note; is already leaking in // server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); cancelClient.execute(SearchAction.INSTANCE, searchRequest, new RestStatusToXContentListener<>(channel)); From c66425adb835337899fd8207561f0a4219d02bda Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 19 Dec 2023 15:02:27 +0000 Subject: [PATCH 07/55] Minor changes Signed-off-by: Peter Nied --- .../java/org/opensearch/cluster/metadata/View.java | 12 +++++++----- .../java/org/opensearch/index/view/views-design.md | 2 +- .../rest/action/admin/indices/RestViewAction.java | 1 + .../action/admin/indices/RestViewSearchAction.java | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index 3546613d9c4dd..ee777ff0c9115 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.List; +/** TODO */ @ExperimentalApi public class View extends AbstractDiffable implements ToXContentObject { @@ -25,11 +26,11 @@ public class View extends AbstractDiffable implements ToXContentObject { public final long modifiedAt; public final List targets; - public View(final String name, final String description, final long createdAt, final long modifiedAt, final List targets) { + public View(final String name, final String description, final Long createdAt, final Long modifiedAt, final List targets) { this.name = name; this.description = description; - this.createdAt = createdAt; - this.modifiedAt = modifiedAt; + this.createdAt = createdAt != null ? createdAt : -1; + this.modifiedAt = modifiedAt != null ? modifiedAt : -1; this.targets = targets; } @@ -37,10 +38,11 @@ public View(final StreamInput in) throws IOException { this(in.readString(), in.readOptionalString(), in.readVLong(), in.readVLong(), in.readList(Target::new)); } - public static Diff readDiffFrom(StreamInput in) throws IOException { + public static Diff readDiffFrom(final StreamInput in) throws IOException { return readDiffFrom(View::new, in); } + /** TODO */ public static class Target implements Writeable, ToXContentObject { public final String indexPattern; @@ -76,7 +78,7 @@ public static Target fromXContent(final XContentParser parser) throws IOExceptio } @Override - public void writeTo(StreamOutput out) throws IOException { + public void writeTo(final StreamOutput out) throws IOException { out.writeString(indexPattern); } } diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md index ee6d10e7af3ef..b297941083ddc 100644 --- a/server/src/main/java/org/opensearch/index/view/views-design.md +++ b/server/src/main/java/org/opensearch/index/view/views-design.md @@ -55,7 +55,7 @@ curl localhost:9200/views \ curl localhost:9200/views/hi/_search ``` -### Appendix +## Appendix VIEW MODEL { diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 02c6e2adbcafe..c12f12eec4a29 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -38,6 +38,7 @@ import static org.opensearch.rest.RestRequest.Method.POST; import static org.opensearch.rest.RestRequest.Method.PUT; +/** TODO */ public class RestViewAction extends BaseRestHandler { private final static Logger LOG = LogManager.getLogger(RestViewAction.class); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java index 4fe645dc20053..5a92caff6deab 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java @@ -35,6 +35,7 @@ import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; +/** TODO */ public class RestViewSearchAction extends BaseRestHandler { private final static Logger LOG = LogManager.getLogger(RestViewSearchAction.class); From c191cc86f54237ecdc4abdc427960210004b6c62 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 19 Dec 2023 18:59:58 +0000 Subject: [PATCH 08/55] Resource Requset Signed-off-by: Peter Nied --- .../opensearch/action/ResourceRequest.java | 41 +++++++ .../action/search/SearchRequest.java | 10 +- .../action/search/ViewSearchRequest.java | 113 ++++++++++++++++++ .../org/opensearch/index/view/views-design.md | 55 +++++---- .../action/admin/indices/RestViewAction.java | 2 +- .../admin/indices/RestViewSearchAction.java | 18 +-- 6 files changed, 203 insertions(+), 36 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/ResourceRequest.java create mode 100644 server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java diff --git a/server/src/main/java/org/opensearch/action/ResourceRequest.java b/server/src/main/java/org/opensearch/action/ResourceRequest.java new file mode 100644 index 0000000000000..64fdc8d647649 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/ResourceRequest.java @@ -0,0 +1,41 @@ +package org.opensearch.action; + +import static org.opensearch.action.ValidateActions.addValidationError; + +import java.util.Map; +import java.util.regex.Pattern; + +import org.opensearch.common.ValidationException; +import org.opensearch.common.annotation.ExperimentalApi; + +/** + * TODO: This validation should be associated with REST requests to ensure the parameter is from the URL + * + * Context: If all of the resource information is in the URL, caching which can include responses or authorization are trivial + */ +@ExperimentalApi +public interface ResourceRequest { + + static final Pattern ALLOWED_KEY_PATTERN = Pattern.compile("[a-zA-Z_-]+"); + static final Pattern ALLOWED_VALUE_PATTERN = Pattern.compile("[a-zA-Z_-]+"); + + /** + * Extracts resource key value pairs from the request parameters + * Note; these resource must be part of the + */ + Map getResourceIds(); + + public static ActionRequestValidationException validResourceIds(final ResourceRequest resourceRequest, final ActionRequestValidationException validationException) { + resourceRequest.getResourceIds().entrySet().forEach(entry -> { + if (!ALLOWED_KEY_PATTERN.matcher(entry.getKey()).matches()) { + addValidationError("Unsupported resource key is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_VALUE_PATTERN.pattern(), validationException); + } + + if (!ALLOWED_VALUE_PATTERN.matcher(entry.getKey()).matches()) { + addValidationError("Unsupported resource value is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_VALUE_PATTERN.pattern(), validationException); + } + }); + + return validationException; + } +} diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java index 96cea17ff4972..67b78d84ce35d 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java @@ -82,13 +82,13 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla private static final long DEFAULT_ABSOLUTE_START_MILLIS = -1; - private final String localClusterAlias; - private final long absoluteStartMillis; - private final boolean finalReduce; + protected final String localClusterAlias; + protected final long absoluteStartMillis; + protected final boolean finalReduce; private SearchType searchType = SearchType.DEFAULT; - private String[] indices = Strings.EMPTY_ARRAY; + protected String[] indices = Strings.EMPTY_ARRAY; @Nullable private String routing; @@ -189,7 +189,7 @@ static SearchRequest subSearchRequest( return new SearchRequest(originalSearchRequest, indices, clusterAlias, absoluteStartMillis, finalReduce); } - private SearchRequest( + protected SearchRequest( SearchRequest searchRequest, String[] indices, String localClusterAlias, diff --git a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java new file mode 100644 index 0000000000000..36c9f3342c2b8 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.search; + +import static org.opensearch.action.ValidateActions.addValidationError; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.regex.Pattern; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ResourceRequest; +import org.opensearch.cluster.metadata.View; +import org.opensearch.common.ValidationException; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.rest.action.admin.indices.RestViewAction; + +/** Wraps the functionality of search requests and tailors for what is available when searching through views + */ +@ExperimentalApi +public class ViewSearchRequest extends SearchRequest implements ResourceRequest { + + public final View view; + + public ViewSearchRequest(final View view) { + super(); + this.view = view; + } + + public ViewSearchRequest(final StreamInput in) throws IOException { + super(in); + view = new View(in); + } + + + @Override + public ActionRequestValidationException validate() { + final Function unsupported = (String x) -> x + " is not supported when searching views"; + ActionRequestValidationException validationException = super.validate(); + + if (scroll() != null) { + validationException = addValidationError(unsupported.apply("Scroll"), validationException); + } + + // TODO: Filter out anything additional search features that are not supported + + validationException = ResourceRequest.validResourceIds(this, validationException); + + return validationException; + } + + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + view.writeTo(out); + } + + @Override + public boolean equals(final Object o) { + // TODO: Maybe this isn't standard practice + return this.hashCode() == o.hashCode(); + } + + @Override + public int hashCode() { + return Objects.hash(view, super.hashCode()); + } + + @Override + public String toString() { + return super.toString().replace("SearchRequest{", "ViewSearchRequest{view=" + view + ","); + } + + @Override + public Map getResourceIds() { + return Map.of(RestViewAction.VIEW_ID, view.name); + } +} diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md index b297941083ddc..c2b9d0882a1dd 100644 --- a/server/src/main/java/org/opensearch/index/view/views-design.md +++ b/server/src/main/java/org/opensearch/index/view/views-design.md @@ -57,6 +57,7 @@ curl localhost:9200/views/hi/_search ## Appendix +``` VIEW MODEL { name: STRING, // [Optional] Friendly name resolves to ID @@ -77,34 +78,45 @@ VIEW MODEL ], documentTransformer: SCRIPT // P2 Convert the results in some way } +``` +### View Operations -View Operations -POST /views -GET /views/{view_id} -PUT /views/{view_id} -PATCH /views/{view_id} -DELETE /views/{view_id} +| Method | Path | +| - | - | +| POST | /views | +| GET | /views/{view_id} | +| PUT | /views/{view_id} | +| PATCH | /views/{view_id} | +| DELETE | /views/{view_id} | -Enumerate Views -GET /views +### Enumerate Views -Search Views // P2? -GET /views/_search -POST /views/_search +| Method | Path | +| - | - | +| GET | /views | -Mapping // P2? Need to understand the utility / impact of not having this -GET /views/{view_id}/_mappings -PUT /views/{view_id}/_mappings -PATCH /views/{view_id}/_mappings +### Perform a Search on a view +| Method | Path | +| - | - | +| GET | /views/{view_id}/_search | +| POST | /views/{view_id}/_search | -Search Views -GET /views/{view_id}/_search -POST /views/{view_id}/_search +### Search Views // P2? +| Method | Path | +| - | - | +| GET | /views/_search | +| POST | /views/_search | -// Results do not include any fields '_', how to protect leaking data? +### Mapping // P2? Need to understand the utility / impact of not having this +| Method | Path | +| - | - | +| GET | /views/{view_id}/_mappings | +| PUT | /views/{view_id}/_mappings | +| PATCH | /views/{view_id}/_mappings | +*Results do not include any fields '_', how to protect leaking data?* ### Response on Create/Enumerate/Search @@ -141,8 +153,8 @@ index:views.create Search does not expose information about the targets or associated mappings. Internally used to resolve all indices and check permissions. +``` GET /views/{view_id}/_resolveTargets - { targets: [ { @@ -157,5 +169,4 @@ targets: [ ], fields: [ FIELD_NAME,... ] } - - +``` diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index c12f12eec4a29..44ace1fcb1f6f 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -43,7 +43,7 @@ public class RestViewAction extends BaseRestHandler { private final static Logger LOG = LogManager.getLogger(RestViewAction.class); - static final String VIEW_ID = "view_id"; + public static final String VIEW_ID = "view_id"; private final ClusterService clusterService; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java index 5a92caff6deab..ab9e57dfc6bd0 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java @@ -13,6 +13,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.ViewSearchRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; @@ -73,33 +74,34 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "")); } - final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) + final Optional optView = Optional.ofNullable(clusterService.state().getMetadata()) .map(m -> m.views()) .map(views -> views.get(viewId)); - if (view.isEmpty()) { + if (optView.isEmpty()) { channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "")); } + final View view = optView.get(); - final SearchRequest searchRequest = new SearchRequest(); - final IntConsumer setSize = size -> searchRequest.source().size(size); + final ViewSearchRequest viewSearchRequest = new ViewSearchRequest(view); + final IntConsumer setSize = size -> viewSearchRequest.source().size(size); request.withContentOrSourceParamParserOrNull( - parser -> RestSearchAction.parseSearchRequest(searchRequest, request, parser, client.getNamedWriteableRegistry(), setSize) + parser -> RestSearchAction.parseSearchRequest(viewSearchRequest, request, parser, client.getNamedWriteableRegistry(), setSize) ); // TODO: Only allow operations that are supported - final String[] indices = view.get().targets.stream() + final String[] indices = view.targets.stream() .map(target -> target.indexPattern) .collect(Collectors.toList()) .toArray(new String[0]); - searchRequest.indices(indices); + viewSearchRequest.indices(indices); // TODO: Look into resource leak on cancelClient? Note; is already leaking in // server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); - cancelClient.execute(SearchAction.INSTANCE, searchRequest, new RestStatusToXContentListener<>(channel)); + cancelClient.execute(SearchAction.INSTANCE, viewSearchRequest, new RestStatusToXContentListener<>(channel)); }; } } From 4abefd0a214ae9afb0bdf201e40c8535dea66c5e Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 19 Dec 2023 21:47:36 +0000 Subject: [PATCH 09/55] Add FAQ sectino Signed-off-by: Peter Nied --- .../opensearch/action/ResourceRequest.java | 24 ++++++++++--------- .../action/search/ViewSearchRequest.java | 2 +- .../org/opensearch/index/view/views-design.md | 5 ++++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/ResourceRequest.java b/server/src/main/java/org/opensearch/action/ResourceRequest.java index 64fdc8d647649..5606a13dc985c 100644 --- a/server/src/main/java/org/opensearch/action/ResourceRequest.java +++ b/server/src/main/java/org/opensearch/action/ResourceRequest.java @@ -5,34 +5,36 @@ import java.util.Map; import java.util.regex.Pattern; -import org.opensearch.common.ValidationException; import org.opensearch.common.annotation.ExperimentalApi; /** * TODO: This validation should be associated with REST requests to ensure the parameter is from the URL + * Note; this should work well in RestHandlers * * Context: If all of the resource information is in the URL, caching which can include responses or authorization are trivial */ @ExperimentalApi public interface ResourceRequest { - static final Pattern ALLOWED_KEY_PATTERN = Pattern.compile("[a-zA-Z_-]+"); - static final Pattern ALLOWED_VALUE_PATTERN = Pattern.compile("[a-zA-Z_-]+"); + static final Pattern ALLOWED_RESOURCE_NAME_PATTERN = Pattern.compile("[a-zA-Z_-]+"); + /** + * Don't allow wildcard patterns + * this has large impact to perf and cachability */ + static final Pattern ALLOWED_RESOURCE_ID_PATTERN = Pattern.compile("[a-zA-Z_-]+"); /** - * Extracts resource key value pairs from the request parameters - * Note; these resource must be part of the + * Validates the resource type and id pairs are in an allowed format */ - Map getResourceIds(); + Map getResourceTypeAndIds(); public static ActionRequestValidationException validResourceIds(final ResourceRequest resourceRequest, final ActionRequestValidationException validationException) { - resourceRequest.getResourceIds().entrySet().forEach(entry -> { - if (!ALLOWED_KEY_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError("Unsupported resource key is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_VALUE_PATTERN.pattern(), validationException); + resourceRequest.getResourceTypeAndIds().entrySet().forEach(entry -> { + if (!ALLOWED_RESOURCE_NAME_PATTERN.matcher(entry.getKey()).matches()) { + addValidationError("Unsupported resource key is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_RESOURCE_NAME_PATTERN.pattern(), validationException); } - if (!ALLOWED_VALUE_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError("Unsupported resource value is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_VALUE_PATTERN.pattern(), validationException); + if (!ALLOWED_RESOURCE_ID_PATTERN.matcher(entry.getKey()).matches()) { + addValidationError("Unsupported resource value is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_RESOURCE_ID_PATTERN.pattern(), validationException); } }); diff --git a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java index 36c9f3342c2b8..00c1c3c86a04f 100644 --- a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java @@ -107,7 +107,7 @@ public String toString() { } @Override - public Map getResourceIds() { + public Map getResourceTypeAndIds() { return Map.of(RestViewAction.VIEW_ID, view.name); } } diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md index c2b9d0882a1dd..4563cb114d2f8 100644 --- a/server/src/main/java/org/opensearch/index/view/views-design.md +++ b/server/src/main/java/org/opensearch/index/view/views-design.md @@ -30,6 +30,11 @@ sequenceDiagram HTTP_Request-->>Client: Return ``` +### Frequently Asked Questions + +#### How do views work with fine grain access control of index data? + +#### What happens with existing DLS and FLS rules and searches on views? ### Local Testing From dace87ac950f99e366b8de8ac589678b13b0cf09 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 22 Dec 2023 01:25:24 +0000 Subject: [PATCH 10/55] Tidying up Signed-off-by: Peter Nied --- .../opensearch/action/ResourceRequest.java | 46 ++++-- .../action/search/ViewSearchRequest.java | 46 +----- .../org/opensearch/cluster/metadata/View.java | 7 + .../cluster/metadata/ViewMetadata.java | 8 + .../org/opensearch/index/view/views-design.md | 153 ++++++++++++------ .../admin/indices/RestViewSearchAction.java | 9 +- 6 files changed, 167 insertions(+), 102 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/ResourceRequest.java b/server/src/main/java/org/opensearch/action/ResourceRequest.java index 5606a13dc985c..84766aaf810f4 100644 --- a/server/src/main/java/org/opensearch/action/ResourceRequest.java +++ b/server/src/main/java/org/opensearch/action/ResourceRequest.java @@ -1,40 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + package org.opensearch.action; -import static org.opensearch.action.ValidateActions.addValidationError; +import org.opensearch.common.annotation.ExperimentalApi; import java.util.Map; import java.util.regex.Pattern; -import org.opensearch.common.annotation.ExperimentalApi; +import static org.opensearch.action.ValidateActions.addValidationError; /** * TODO: This validation should be associated with REST requests to ensure the parameter is from the URL * Note; this should work well in RestHandlers - * + * * Context: If all of the resource information is in the URL, caching which can include responses or authorization are trivial */ @ExperimentalApi public interface ResourceRequest { static final Pattern ALLOWED_RESOURCE_NAME_PATTERN = Pattern.compile("[a-zA-Z_-]+"); - /** + /** * Don't allow wildcard patterns * this has large impact to perf and cachability */ static final Pattern ALLOWED_RESOURCE_ID_PATTERN = Pattern.compile("[a-zA-Z_-]+"); + /** Returns the resource types and ids associated with this request */ + Map getResourceTypeAndIds(); + /** * Validates the resource type and id pairs are in an allowed format */ - Map getResourceTypeAndIds(); - - public static ActionRequestValidationException validResourceIds(final ResourceRequest resourceRequest, final ActionRequestValidationException validationException) { + public static ActionRequestValidationException validResourceIds( + final ResourceRequest resourceRequest, + final ActionRequestValidationException validationException + ) { resourceRequest.getResourceTypeAndIds().entrySet().forEach(entry -> { if (!ALLOWED_RESOURCE_NAME_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError("Unsupported resource key is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_RESOURCE_NAME_PATTERN.pattern(), validationException); + addValidationError( + "Unsupported resource key is not supported; key: " + + entry.getKey() + + " value: " + + entry.getValue() + + " allowed pattern " + + ALLOWED_RESOURCE_NAME_PATTERN.pattern(), + validationException + ); } if (!ALLOWED_RESOURCE_ID_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError("Unsupported resource value is not supported; key: " + entry.getKey() + " value: " + entry.getValue() + " allowed pattern " + ALLOWED_RESOURCE_ID_PATTERN.pattern(), validationException); + addValidationError( + "Unsupported resource value is not supported; key: " + + entry.getKey() + + " value: " + + entry.getValue() + + " allowed pattern " + + ALLOWED_RESOURCE_ID_PATTERN.pattern(), + validationException + ); } }); diff --git a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java index 00c1c3c86a04f..88d610fe29480 100644 --- a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java @@ -6,50 +6,20 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.action.search; -import static org.opensearch.action.ValidateActions.addValidationError; - -import java.io.IOException; -import java.util.Map; -import java.util.Objects; -import java.util.function.Function; -import java.util.regex.Pattern; - import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ResourceRequest; import org.opensearch.cluster.metadata.View; -import org.opensearch.common.ValidationException; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.at org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.rest.action.admin.indices.RestViewAction; +import org.opensearch.rest.action.admin.indicport java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +import static org.opensearch.action.ValidateActions.addValidationError; -/** Wraps the functionality of search requests and tailors for what is available when searching through views +/** Wraps the functionality of search requests and tailors for what is available when searching through views */ @ExperimentalApi public class ViewSearchRequest extends SearchRequest implements ResourceRequest { @@ -66,7 +36,6 @@ public ViewSearchRequest(final StreamInput in) throws IOException { view = new View(in); } - @Override public ActionRequestValidationException validate() { final Function unsupported = (String x) -> x + " is not supported when searching views"; @@ -83,7 +52,6 @@ public ActionRequestValidationException validate() { return validationException; } - @Override public void writeTo(final StreamOutput out) throws IOException { super.writeTo(out); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index ee777ff0c9115..c32bd2e53842f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -1,3 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ package org.opensearch.cluster.metadata; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java index fc70f77b18ef9..f308cfdfb55bf 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -1,3 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + package org.opensearch.cluster.metadata; import org.opensearch.Version; diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md index 4563cb114d2f8..773b359cc1953 100644 --- a/server/src/main/java/org/opensearch/index/view/views-design.md +++ b/server/src/main/java/org/opensearch/index/view/views-design.md @@ -5,6 +5,44 @@ Views define how searches are performed against indices on a cluster, uniform da ## Design +### View data + +Views create a mapping to the resources that hold information to be searched over in a consistent manner. This abstraction allows for indirection with the backing indices, so they might be changed without callers being impacted. This can also be used to simplify the security model - searches over views do not require permissions to the backing indices only permissions to the view itself. + +```mermaid +classDiagram + class View { + +String name + +String description + +long createdAt + +long modifiedAt + +List targets + +toXContent(XContentBuilder, Params) XContentBuilder + +writeTo(StreamOutput) void + } + class Target { + +String indexPattern + +toXContent(XContentBuilder, Params) XContentBuilder + +writeTo(StreamOutput) void + } + class StreamOutput + class XContentBuilder + + View -- Target : contains + View -- StreamOutput : writes to + View -- XContentBuilder : outputs to + Target -- StreamOutput : writes to + Target -- XContentBuilder : outputs to +``` + +### View persistence + +Views are long lived objects in OpenSearch, all operations on them should be fully committed before responding to the caller. Views are intentionally created for user scenarios following a similar creation cadence to indices. + +Committed implies that the updates are synchronized across all nodes in a cluster. The Cluster Metadata Store is already available and allows for acknowledging that changes have been applied to all nodes. While this data could be stored in a new purpose built index, index data replication has delays and ensuring synchronization is non-trivial to implement as is seen in the Security plugins [1]. + +- [1] https://github.com/opensearch-project/security/issues/3275 + ```mermaid sequenceDiagram participant Client @@ -30,11 +68,63 @@ sequenceDiagram HTTP_Request-->>Client: Return ``` -### Frequently Asked Questions +### Resource Request + +In order to permissions views OpenSearch needs a way to consistently refer to them, this is a generic problem and views will be a first use case. Resource requests require a map of types to identifiers for the request, multiple resources could be part of a single request, but only one of each type. + +Considering the request to search a view, `POST /view/{view_id}/_search`, the path parameter 'view_id' is the type and the value from the request would be the identifier. + +```java +public interface ResourceRequest { + /** Returns the resource types and ids associated with this request */ + Map getResourceTypeAndIds(); + + /** Validates the resource type and id pairs are in an allowed format */ + public static ActionRequestValidationException validResourceIds( + final ResourceRequest resourceRequest, + final ActionRequestValidationException validationException + ) {;} +} +``` + +### Resource Permission Grants +With requests include resource type and identifiers the security plugin will need to allow for grants to these new types. Modify the security role to include this information so it can be checked and then the request can be permitted. + +```yaml +all_access: + reserved: true + hidden: false + static: true + description: "Allow full access to all indices and all cluster APIs" + cluster_permissions: + - "*" + index_permissions: + - index_patterns: + - "*" + allowed_actions: + - "*" + tenant_permissions: + - tenant_patterns: + - "*" + allowed_actions: + - "kibana_all_write" + resource_permissions: + - resource_type: "view" + resource_ids: ["songs", "albums"] +``` + +## Frequently Asked Questions + +### How do views work with fine grain access control of index data? +*To be determined...* -#### How do views work with fine grain access control of index data? +### What happens with existing DLS and FLS rules and searches on views? +*To be determined...* -#### What happens with existing DLS and FLS rules and searches on views? +### Additional Question(s) +*To be determined...* + +## Appendix ### Local Testing @@ -60,7 +150,7 @@ curl localhost:9200/views \ curl localhost:9200/views/hi/_search ``` -## Appendix +### v0 View Data Model ``` VIEW MODEL @@ -85,7 +175,7 @@ VIEW MODEL } ``` -### View Operations +#### View Operations | Method | Path | | - | - | @@ -95,25 +185,25 @@ VIEW MODEL | PATCH | /views/{view_id} | | DELETE | /views/{view_id} | -### Enumerate Views +#### Enumerate Views | Method | Path | | - | - | | GET | /views | -### Perform a Search on a view +#### Perform a Search on a view | Method | Path | | - | - | | GET | /views/{view_id}/_search | | POST | /views/{view_id}/_search | -### Search Views // P2? +#### Search Views // P2? | Method | Path | | - | - | | GET | /views/_search | | POST | /views/_search | -### Mapping // P2? Need to understand the utility / impact of not having this +#### Mapping // P2? Need to understand the utility / impact of not having this | Method | Path | | - | - | | GET | /views/{view_id}/_mappings | @@ -123,7 +213,7 @@ VIEW MODEL *Results do not include any fields '_', how to protect leaking data?* -### Response on Create/Enumerate/Search +#### Response on Create/Enumerate/Search views: [ { @@ -133,45 +223,4 @@ views: [ created: DATE, modified: DATE } -] - - -## Permissions - -Views can be permissioned directly by name or id. User must also have READ permission for all indices targeted by the view, authorized before any of these operations. - -index:views.view.read[:{view_id}] -index:views.view.modify[:{view_id}] -index:views.view.delete[:{view_id}] - -Searching with a view. Read permission of the view is *not* required. - -index:views.view.search[:{view_id}] - -Creation / Enumeration / Searching views is supported. Only the view name and view id are returned. Create view call requires READ permission for all indices targeted by the view. - -index:views.enumerate -index:views.search -index:views.create - -### Internal functionality - -Search does not expose information about the targets or associated mappings. Internally used to resolve all indices and check permissions. - -``` -GET /views/{view_id}/_resolveTargets -{ -targets: [ - { - index: STRING, - fields: [ - { - fieldName: STRING - } - ], - _view: VIEW_MIN_OBJECT - }, ... -], -fields: [ FIELD_NAME,... ] -} -``` +] \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java index ab9e57dfc6bd0..bdda2971ac38a 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchAction; -import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.ViewSearchRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.View; @@ -87,7 +86,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final IntConsumer setSize = size -> viewSearchRequest.source().size(size); request.withContentOrSourceParamParserOrNull( - parser -> RestSearchAction.parseSearchRequest(viewSearchRequest, request, parser, client.getNamedWriteableRegistry(), setSize) + parser -> RestSearchAction.parseSearchRequest( + viewSearchRequest, + request, + parser, + client.getNamedWriteableRegistry(), + setSize + ) ); // TODO: Only allow operations that are supported From c88febe47927d99230c0128d29363c8d4c629981 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 17 Jan 2024 22:54:56 +0000 Subject: [PATCH 11/55] Read prepare for merge into OpenSearch Signed-off-by: Peter Nied --- .../opensearch/action/ResourceRequest.java | 71 ------------------- .../action/search/ViewSearchRequest.java | 5 +- .../org/opensearch/index/view/views-design.md | 60 +--------------- 3 files changed, 2 insertions(+), 134 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/action/ResourceRequest.java diff --git a/server/src/main/java/org/opensearch/action/ResourceRequest.java b/server/src/main/java/org/opensearch/action/ResourceRequest.java deleted file mode 100644 index 84766aaf810f4..0000000000000 --- a/server/src/main/java/org/opensearch/action/ResourceRequest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.action; - -import org.opensearch.common.annotation.ExperimentalApi; - -import java.util.Map; -import java.util.regex.Pattern; - -import static org.opensearch.action.ValidateActions.addValidationError; - -/** - * TODO: This validation should be associated with REST requests to ensure the parameter is from the URL - * Note; this should work well in RestHandlers - * - * Context: If all of the resource information is in the URL, caching which can include responses or authorization are trivial - */ -@ExperimentalApi -public interface ResourceRequest { - - static final Pattern ALLOWED_RESOURCE_NAME_PATTERN = Pattern.compile("[a-zA-Z_-]+"); - /** - * Don't allow wildcard patterns - * this has large impact to perf and cachability */ - static final Pattern ALLOWED_RESOURCE_ID_PATTERN = Pattern.compile("[a-zA-Z_-]+"); - - /** Returns the resource types and ids associated with this request */ - Map getResourceTypeAndIds(); - - /** - * Validates the resource type and id pairs are in an allowed format - */ - public static ActionRequestValidationException validResourceIds( - final ResourceRequest resourceRequest, - final ActionRequestValidationException validationException - ) { - resourceRequest.getResourceTypeAndIds().entrySet().forEach(entry -> { - if (!ALLOWED_RESOURCE_NAME_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError( - "Unsupported resource key is not supported; key: " - + entry.getKey() - + " value: " - + entry.getValue() - + " allowed pattern " - + ALLOWED_RESOURCE_NAME_PATTERN.pattern(), - validationException - ); - } - - if (!ALLOWED_RESOURCE_ID_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError( - "Unsupported resource value is not supported; key: " - + entry.getKey() - + " value: " - + entry.getValue() - + " allowed pattern " - + ALLOWED_RESOURCE_ID_PATTERN.pattern(), - validationException - ); - } - }); - - return validationException; - } -} diff --git a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java index 88d610fe29480..9010681362ed4 100644 --- a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java @@ -9,7 +9,6 @@ package org.opensearch.action.search; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.ResourceRequest; import org.opensearch.cluster.metadata.View; import org.opensearch.common.at org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -22,7 +21,7 @@ /** Wraps the functionality of search requests and tailors for what is available when searching through views */ @ExperimentalApi -public class ViewSearchRequest extends SearchRequest implements ResourceRequest { +public class ViewSearchRequest extends SearchRequest { public final View view; @@ -47,8 +46,6 @@ public ActionRequestValidationException validate() { // TODO: Filter out anything additional search features that are not supported - validationException = ResourceRequest.validResourceIds(this, validationException); - return validationException; } diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md index 773b359cc1953..4bd5d9585c5c2 100644 --- a/server/src/main/java/org/opensearch/index/view/views-design.md +++ b/server/src/main/java/org/opensearch/index/view/views-design.md @@ -2,7 +2,6 @@ Views define how searches are performed against indices on a cluster, uniform data access that is configured separately from the queries. - ## Design ### View data @@ -67,63 +66,6 @@ sequenceDiagram Data_Store-->>HTTP_Request: Return HTTP_Request-->>Client: Return ``` - -### Resource Request - -In order to permissions views OpenSearch needs a way to consistently refer to them, this is a generic problem and views will be a first use case. Resource requests require a map of types to identifiers for the request, multiple resources could be part of a single request, but only one of each type. - -Considering the request to search a view, `POST /view/{view_id}/_search`, the path parameter 'view_id' is the type and the value from the request would be the identifier. - -```java -public interface ResourceRequest { - /** Returns the resource types and ids associated with this request */ - Map getResourceTypeAndIds(); - - /** Validates the resource type and id pairs are in an allowed format */ - public static ActionRequestValidationException validResourceIds( - final ResourceRequest resourceRequest, - final ActionRequestValidationException validationException - ) {;} -} -``` - -### Resource Permission Grants -With requests include resource type and identifiers the security plugin will need to allow for grants to these new types. Modify the security role to include this information so it can be checked and then the request can be permitted. - -```yaml -all_access: - reserved: true - hidden: false - static: true - description: "Allow full access to all indices and all cluster APIs" - cluster_permissions: - - "*" - index_permissions: - - index_patterns: - - "*" - allowed_actions: - - "*" - tenant_permissions: - - tenant_patterns: - - "*" - allowed_actions: - - "kibana_all_write" - resource_permissions: - - resource_type: "view" - resource_ids: ["songs", "albums"] -``` - -## Frequently Asked Questions - -### How do views work with fine grain access control of index data? -*To be determined...* - -### What happens with existing DLS and FLS rules and searches on views? -*To be determined...* - -### Additional Question(s) -*To be determined...* - ## Appendix ### Local Testing @@ -223,4 +165,4 @@ views: [ created: DATE, modified: DATE } -] \ No newline at end of file +] From 61e8c988751c413791805e632da9ef6ade71573f Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Wed, 17 Jan 2024 23:29:41 +0000 Subject: [PATCH 12/55] Fix build issue Signed-off-by: Peter Nied --- .../opensearch/action/search/ViewSearchRequest.java | 12 ++++-------- .../opensearch/cluster/metadata/ViewMetadata.java | 10 ---------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java index 9010681362ed4..bd190ec1576f2 100644 --- a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java @@ -10,9 +10,11 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.cluster.metadata.View; -import org.opensearch.common.at org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.rest.action.admin.indicport java.util.Map; +import java.io.IOException; +import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -57,7 +59,6 @@ public void writeTo(final StreamOutput out) throws IOException { @Override public boolean equals(final Object o) { - // TODO: Maybe this isn't standard practice return this.hashCode() == o.hashCode(); } @@ -70,9 +71,4 @@ public int hashCode() { public String toString() { return super.toString().replace("SearchRequest{", "ViewSearchRequest{view=" + view + ","); } - - @Override - public Map getResourceTypeAndIds() { - return Map.of(RestViewAction.VIEW_ID, view.name); - } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java index f308cfdfb55bf..78f9707f4b536 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -29,16 +29,6 @@ import static org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata.MINIMMAL_SUPPORTED_VERSION; -/** - * TODO: Tests with failures? `./gradlew :server:test` - - - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDeleteIOException - - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsDeleteDedup - - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDeleteNoSuchFileException - - org.opensearch.index.store.RemoteSegmentStoreDirectoryTests.testDeleteStaleCommitsActualDelete - * - */ - public class ViewMetadata implements Metadata.Custom { public static final String TYPE = "view"; From ac25c960045ab6b7ac3ceea77a0ad7a380aa028e Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 29 Jan 2024 22:25:41 +0000 Subject: [PATCH 13/55] Fix build issues Signed-off-by: Peter Nied --- .../opensearch/action/ResourceRequest.java | 71 ------------------- .../action/search/ViewSearchRequest.java | 15 ++-- .../org/opensearch/cluster/metadata/View.java | 3 +- 3 files changed, 6 insertions(+), 83 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/action/ResourceRequest.java diff --git a/server/src/main/java/org/opensearch/action/ResourceRequest.java b/server/src/main/java/org/opensearch/action/ResourceRequest.java deleted file mode 100644 index 84766aaf810f4..0000000000000 --- a/server/src/main/java/org/opensearch/action/ResourceRequest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.action; - -import org.opensearch.common.annotation.ExperimentalApi; - -import java.util.Map; -import java.util.regex.Pattern; - -import static org.opensearch.action.ValidateActions.addValidationError; - -/** - * TODO: This validation should be associated with REST requests to ensure the parameter is from the URL - * Note; this should work well in RestHandlers - * - * Context: If all of the resource information is in the URL, caching which can include responses or authorization are trivial - */ -@ExperimentalApi -public interface ResourceRequest { - - static final Pattern ALLOWED_RESOURCE_NAME_PATTERN = Pattern.compile("[a-zA-Z_-]+"); - /** - * Don't allow wildcard patterns - * this has large impact to perf and cachability */ - static final Pattern ALLOWED_RESOURCE_ID_PATTERN = Pattern.compile("[a-zA-Z_-]+"); - - /** Returns the resource types and ids associated with this request */ - Map getResourceTypeAndIds(); - - /** - * Validates the resource type and id pairs are in an allowed format - */ - public static ActionRequestValidationException validResourceIds( - final ResourceRequest resourceRequest, - final ActionRequestValidationException validationException - ) { - resourceRequest.getResourceTypeAndIds().entrySet().forEach(entry -> { - if (!ALLOWED_RESOURCE_NAME_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError( - "Unsupported resource key is not supported; key: " - + entry.getKey() - + " value: " - + entry.getValue() - + " allowed pattern " - + ALLOWED_RESOURCE_NAME_PATTERN.pattern(), - validationException - ); - } - - if (!ALLOWED_RESOURCE_ID_PATTERN.matcher(entry.getKey()).matches()) { - addValidationError( - "Unsupported resource value is not supported; key: " - + entry.getKey() - + " value: " - + entry.getValue() - + " allowed pattern " - + ALLOWED_RESOURCE_ID_PATTERN.pattern(), - validationException - ); - } - }); - - return validationException; - } -} diff --git a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java index 88d610fe29480..0c3215e8e647a 100644 --- a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java @@ -9,11 +9,11 @@ package org.opensearch.action.search; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.ResourceRequest; import org.opensearch.cluster.metadata.View; -import org.opensearch.common.at org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.rest.action.admin.indicport java.util.Map; +import java.io.IOException; import java.util.Objects; import java.util.function.Function; @@ -22,7 +22,7 @@ /** Wraps the functionality of search requests and tailors for what is available when searching through views */ @ExperimentalApi -public class ViewSearchRequest extends SearchRequest implements ResourceRequest { +public class ViewSearchRequest extends SearchRequest { public final View view; @@ -47,8 +47,6 @@ public ActionRequestValidationException validate() { // TODO: Filter out anything additional search features that are not supported - validationException = ResourceRequest.validResourceIds(this, validationException); - return validationException; } @@ -73,9 +71,4 @@ public int hashCode() { public String toString() { return super.toString().replace("SearchRequest{", "ViewSearchRequest{view=" + view + ","); } - - @Override - public Map getResourceTypeAndIds() { - return Map.of(RestViewAction.VIEW_ID, view.name); - } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index c32bd2e53842f..26678b9eb123c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -50,11 +50,12 @@ public static Diff readDiffFrom(final StreamInput in) throws IOException { } /** TODO */ + @ExperimentalApi public static class Target implements Writeable, ToXContentObject { public final String indexPattern; - private Target(final String indexPattern) { + Target(final String indexPattern) { this.indexPattern = indexPattern; } From 8e41fbeeb6ac54aeea57600da03860fb4b6d1881 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 29 Jan 2024 22:49:43 +0000 Subject: [PATCH 14/55] Unit tests for view Signed-off-by: Peter Nied --- .../cluster/metadata/ViewTests.java | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java new file mode 100644 index 0000000000000..4eab59d32ca0c --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.cluster.metadata; + +import org.opensearch.cluster.metadata.View.Target; +import org.opensearch.common.UUIDs; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.index.Index; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.cluster.DataStreamTestHelper.createTimestampField; +import static org.opensearch.cluster.metadata.DataStream.getDefaultBackingIndexName; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; + +public class ViewTests extends AbstractSerializingTestCase { + + private static List randomTargets() { + int numTargets = randomIntBetween(0, 128); + List targets = new ArrayList<>(numTargets); + for (int i = 0; i < numTargets; i++) { + targets.add(new Target(randomAlphaOfLength(10).toLowerCase(Locale.ROOT)); + } + return targets; + } + + private static View randomInstance() { + final List targets = randomTargets(); + final String viewName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + final String description = randomAlphaOfLength(100).toLowerCase(Locale.ROOT); + return new View(viewName, description, Math.abs(randomLong()), Math.abs(randomLong()), targets); + } + + @Override + protected View doParseInstance(XContentParser parser) throws IOException { + return View.fromXContent(parser); + } + + @Override + protected Writeable.Reader instanceReader() { + return View::new; + } + + @Override + protected View createTestInstance() { + return randomInstance(); + } + + public void testNullName() { + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View(null, null, null, null, null)); + + assertThat(npe.getMessage(), equalTo("Name must be provided")); + } + + public void testNullTargets() { + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View("name", null, null, null, null)); + + assertThat(npe.getMessage(), equalTo("Targets are required on a view")); + } + + public void testNullTargetIndexPattern() { + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View.Target(null)); + + assertThat(npe.getMessage(), equalTo("IndexPattern is required")); + } + + + public void testDefaultValues() { + final View view = new View("myName", null, null, null, List.of()); + + assertThat(view.name, equalTo("myName")); + assertThat(view.description, equalTo(null)); + assertThat(view.createdAt, equalTo(-1L)); + assertThat(view.modifiedAt, equalTo(-1L)); + assertThat(view.targets, empty()); + } + + +} From cf22a8d7f7875e1ac30e176a67cd7a754bc2728d Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 1 Feb 2024 20:39:38 +0000 Subject: [PATCH 15/55] Basic pattern for decoupled views in metadata vs transport requests Signed-off-by: Peter Nied --- .../org/opensearch/action/ActionModule.java | 4 + .../admin/indices/view/CreateViewAction.java | 193 ++++++++++++++++++ .../admin/indices/view/package-info.java | 10 + .../org/opensearch/cluster/metadata/View.java | 11 +- .../cluster/metadata/ViewService.java | 54 +++++ .../main/java/org/opensearch/node/Node.java | 9 +- .../cluster/metadata/ViewTests.java | 3 - 7 files changed, 273 insertions(+), 11 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/ViewService.java diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index 46775466aa615..e2a738ac959a3 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -224,6 +224,7 @@ import org.opensearch.action.admin.indices.upgrade.post.UpgradeSettingsAction; import org.opensearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.opensearch.action.admin.indices.validate.query.ValidateQueryAction; +import org.opensearch.action.admin.indices.view.CreateViewAction; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.TransportBulkAction; import org.opensearch.action.bulk.TransportShardBulkAction; @@ -721,6 +722,9 @@ public void reg actions.register(ResolveIndexAction.INSTANCE, ResolveIndexAction.TransportAction.class); actions.register(DataStreamsStatsAction.INSTANCE, DataStreamsStatsAction.TransportAction.class); + // Views: + actions.register(CreateViewAction.INSTANCE, CreateViewAction.TransportAction.class); + // Persistent tasks: actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class); actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java new file mode 100644 index 0000000000000..10d84bd832229 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -0,0 +1,193 @@ +package org.opensearch.action.admin.indices.view; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.ValidateActions; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.ViewService; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +/** Action to create a view */ +public class CreateViewAction extends ActionType { + + public static final CreateViewAction INSTANCE = new CreateViewAction(); + public static final String NAME = "cluster:views:create"; + + private CreateViewAction() { + super(NAME, CreateViewAction.Response::new); + } + + + /** View target representation for create requests */ + public static class ViewTarget implements Writeable { + public final String indexPattern; + + public ViewTarget(final String indexPattern) { + this.indexPattern = indexPattern; + } + + public ViewTarget(final StreamInput in) throws IOException { + this.indexPattern = in.readString(); + } + + public String getIndexPattern() { + return indexPattern; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(indexPattern); + } + + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(indexPattern)) { + validationException = ValidateActions.addValidationError("index pattern cannot be empty or null", validationException); + } + + return validationException; + } + + } + + /** + * Request for Creating View + */ + public static class Request extends ClusterManagerNodeRequest { + private final String name; + private final String description; + private final List targets; + + public Request(final String name, final String description, final List targets) { + this.name = name; + this.description = description; + this.targets = targets; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public List getTargets() { + return new ArrayList<>(targets); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = ValidateActions.addValidationError("Name is cannot be empty or null", validationException); + } + if (targets.isEmpty()) { + validationException = ValidateActions.addValidationError("targets cannot be empty", validationException); + } + + for (final ViewTarget target : targets) { + validationException = target.validate(); + } + + return validationException; + } + + public Request(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.description = in.readString(); + this.targets = in.readList(ViewTarget::new); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeString(description); + out.writeList(targets); + } + } + + /** Response after view is created */ + public static class Response extends ActionResponse { + + private final org.opensearch.cluster.metadata.View createdView; + + public Response(final org.opensearch.cluster.metadata.View createdView) { + this.createdView = createdView; + } + + public Response(final StreamInput in) throws IOException { + super(in); + this.createdView = new org.opensearch.cluster.metadata.View(in); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + this.createdView.writeTo(out); + } + } + + /** + * Transport Action for creating a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response read(StreamInput in) throws IOException { + return new Response(in); + } + + @Override + protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) + throws Exception { + viewService.createView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java b/server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java new file mode 100644 index 0000000000000..db0556b1bf334 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** View transport handlers. */ +package org.opensearch.action.admin.indices.view; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index 26678b9eb123c..8db65c6afaebe 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; /** TODO */ @ExperimentalApi @@ -34,11 +35,11 @@ public class View extends AbstractDiffable implements ToXContentObject { public final List targets; public View(final String name, final String description, final Long createdAt, final Long modifiedAt, final List targets) { - this.name = name; + this.name = Objects.requireNonNull(name, "Name must be provided"); this.description = description; this.createdAt = createdAt != null ? createdAt : -1; this.modifiedAt = modifiedAt != null ? modifiedAt : -1; - this.targets = targets; + this.targets = Objects.requireNonNull(targets, "Targets are required on a view"); } public View(final StreamInput in) throws IOException { @@ -55,11 +56,11 @@ public static class Target implements Writeable, ToXContentObject { public final String indexPattern; - Target(final String indexPattern) { - this.indexPattern = indexPattern; + public Target(final String indexPattern) { + this.indexPattern = Objects.requireNonNull(indexPattern, "IndexPattern is required"); } - private Target(final StreamInput in) throws IOException { + public Target(final StreamInput in) throws IOException { this(in.readString()); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewService.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewService.java new file mode 100644 index 0000000000000..4056e477f2226 --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewService.java @@ -0,0 +1,54 @@ +package org.opensearch.cluster.metadata; + +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.action.ActionListener; + +/** Service to interact with views, create, retrieve, update, and delete */ +public class ViewService { + + private final static Logger LOG = LogManager.getLogger(ViewService.class); + private final ClusterService clusterService; + + public ViewService(final ClusterService clusterService) { + this.clusterService = clusterService; + } + + public void createView(final CreateViewAction.Request request, final ActionListener listener) { + final long currentTime = System.currentTimeMillis(); + + final List targets = request.getTargets() + .stream() + .map(target -> new View.Target(target.getIndexPattern())) + .collect(Collectors.toList()); + final View view = new View(request.getName(), request.getDescription(), currentTime, currentTime, targets); + + clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(final ClusterState currentState) throws Exception { + return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) + .build(); + } + + @Override + public void onFailure(final String source, final Exception e) { + LOG.error("Unable to create view, due to {}", source, e); + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + final View createdView = newState.getMetadata().views().get(request.getName()); + final CreateViewAction.Response response = new CreateViewAction.Response(createdView); + listener.onResponse(response); + } + }); + } +} diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index d9c139be9f915..d26a7deae9e77 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -72,6 +72,7 @@ import org.opensearch.cluster.metadata.MetadataIndexUpgradeService; import org.opensearch.cluster.metadata.SystemIndexMetadataUpgradeService; import org.opensearch.cluster.metadata.TemplateUpgradeService; +import org.opensearch.cluster.metadata.ViewService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.routing.BatchedRerouteService; @@ -862,6 +863,10 @@ protected Node( metadataCreateIndexService ); + final ViewService viewService = new ViewService( + clusterService + ); + Collection pluginComponents = pluginsService.filterPlugins(Plugin.class) .stream() .flatMap( @@ -898,9 +903,6 @@ protected Node( ); modules.add(actionModule); - actionModule.getRestController().registerHandler(new RestViewAction(clusterService)); - actionModule.getRestController().registerHandler(new RestViewSearchAction(clusterService)); - final RestController restController = actionModule.getRestController(); final NodeResourceUsageTracker nodeResourceUsageTracker = new NodeResourceUsageTracker( @@ -1222,6 +1224,7 @@ protected Node( b.bind(MetadataCreateIndexService.class).toInstance(metadataCreateIndexService); b.bind(AwarenessReplicaBalance.class).toInstance(awarenessReplicaBalance); b.bind(MetadataCreateDataStreamService.class).toInstance(metadataCreateDataStreamService); + b.bind(ViewService.class).toInstance(viewService); b.bind(SearchService.class).toInstance(searchService); b.bind(SearchTransportService.class).toInstance(searchTransportService); b.bind(SearchPhaseController.class) diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java index 4eab59d32ca0c..c1184ddeca915 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java @@ -82,7 +82,6 @@ public void testNullTargetIndexPattern() { assertThat(npe.getMessage(), equalTo("IndexPattern is required")); } - public void testDefaultValues() { final View view = new View("myName", null, null, null, List.of()); @@ -92,6 +91,4 @@ public void testDefaultValues() { assertThat(view.modifiedAt, equalTo(-1L)); assertThat(view.targets, empty()); } - - } From d45fcb933047c59dd894777c95eca50c7c83bb8b Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Sat, 3 Feb 2024 00:51:23 +0000 Subject: [PATCH 16/55] Integration tests and some unit tests Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 77 ++++++ .../org/opensearch/action/ActionModule.java | 7 + .../admin/indices/view/CreateViewAction.java | 207 +++++++++----- .../admin/indices/view/SearchViewAction.java | 134 +++++++++ .../admin/indices/view}/ViewService.java | 35 ++- .../action/search/ViewSearchRequest.java | 73 ----- .../opensearch/client/IndicesAdminClient.java | 15 ++ .../client/support/AbstractClient.java | 22 ++ .../opensearch/cluster/metadata/Metadata.java | 2 +- .../org/opensearch/cluster/metadata/View.java | 91 +++++-- .../cluster/metadata/ViewMetadata.java | 3 +- .../main/java/org/opensearch/node/Node.java | 7 +- .../action/admin/indices/RestViewAction.java | 255 ++++++++++-------- .../admin/indices/RestViewSearchAction.java | 112 -------- .../admin/indices/view/CreateViewTests.java | 51 ++++ .../admin/indices/view/SearchViewTests.java | 55 ++++ .../cluster/metadata/ViewTests.java | 28 +- 17 files changed, 765 insertions(+), 409 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java rename server/src/main/java/org/opensearch/{cluster/metadata => action/admin/indices/view}/ViewService.java (61%) delete mode 100644 server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java delete mode 100644 server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java new file mode 100644 index 0000000000000..ac706491478d1 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -0,0 +1,77 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import static org.hamcrest.Matchers.is; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +import java.util.List; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; +import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.opensearch.test.junit.annotations.TestLogging; + +@ClusterScope(scope = Scope.TEST, numDataNodes = 2) +public class ViewIT extends OpenSearchIntegTestCase { + + private int createIndexWithDocs(final String indexName) throws Exception { + createIndex(indexName); + ensureGreen(indexName); + + final int numOfDocs = scaledRandomIntBetween(0, 200); + try (final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) { + waitForDocs(numOfDocs, indexer); + } + + refresh(indexName); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs); + return numOfDocs; + } + + private CreateViewAction.Response createView(final String name, final String indexPattern) throws Exception { + final CreateViewAction.Request request = new CreateViewAction.Request(name, null, List.of(new CreateViewAction.Request.Target(indexPattern))); + final CreateViewAction.Response response = client().admin().indices().createView(request).actionGet(); + performRemoteStoreTestAction(); + return response; + } + + private SearchResponse searchView(final String viewName) throws Exception { + final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); + final SearchResponse response = client().searchView(request).actionGet(); + return response; + } + + public void testBasicOperations() throws Exception { + final String indexInView1 = "index-1"; + final String indexInView2 = "index-2"; + final String indexNotInView = "another-index-1"; + + final int indexInView1DocCount = createIndexWithDocs(indexInView1); + final int indexInView2DocCount = createIndexWithDocs(indexInView2); + createIndexWithDocs(indexNotInView); + + logger.info("Testing view with no matches"); + createView("no-matches", "this-pattern-will-match-nothing"); + final IndexNotFoundException ex = assertThrows(IndexNotFoundException.class, () -> searchView("no-matches")); + assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]")); + + logger.info("Testing view with exact index match"); + createView("only-index-1", "index-1"); + assertHitCount(searchView("only-index-1"), indexInView1DocCount); + + logger.info("Testing view with wildcard matches"); + createView("both-indices", "index-*"); + assertHitCount(searchView("both-indices"), indexInView1DocCount + indexInView2DocCount); + } +} diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index e2a738ac959a3..b18498de9581e 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -225,6 +225,7 @@ import org.opensearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.opensearch.action.admin.indices.validate.query.ValidateQueryAction; import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.TransportBulkAction; import org.opensearch.action.bulk.TransportShardBulkAction; @@ -410,6 +411,7 @@ import org.opensearch.rest.action.admin.indices.RestUpgradeAction; import org.opensearch.rest.action.admin.indices.RestUpgradeStatusAction; import org.opensearch.rest.action.admin.indices.RestValidateQueryAction; +import org.opensearch.rest.action.admin.indices.RestViewAction; import org.opensearch.rest.action.cat.AbstractCatAction; import org.opensearch.rest.action.cat.RestAliasAction; import org.opensearch.rest.action.cat.RestAllocationAction; @@ -724,6 +726,7 @@ public void reg // Views: actions.register(CreateViewAction.INSTANCE, CreateViewAction.TransportAction.class); + actions.register(SearchViewAction.INSTANCE, SearchViewAction.TransportAction.class); // Persistent tasks: actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class); @@ -919,6 +922,10 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestResolveIndexAction()); registerHandler.accept(new RestDataStreamsStatsAction()); + // View API + registerHandler.accept(new RestViewAction.CreateViewHandler()); + registerHandler.accept(new RestViewAction.SearchViewHandler()); + // CAT API registerHandler.accept(new RestAllocationAction()); registerHandler.accept(new RestCatSegmentReplicationAction()); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index 10d84bd832229..f4e8b1db1956a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; @@ -14,15 +15,22 @@ import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; -import org.opensearch.cluster.metadata.ViewService; +import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.inject.Inject; +import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.common.util.CollectionUtils; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -30,60 +38,34 @@ public class CreateViewAction extends ActionType { public static final CreateViewAction INSTANCE = new CreateViewAction(); - public static final String NAME = "cluster:views:create"; + public static final String NAME = "cluster:admin:views:create"; private CreateViewAction() { super(NAME, CreateViewAction.Response::new); } - - /** View target representation for create requests */ - public static class ViewTarget implements Writeable { - public final String indexPattern; - - public ViewTarget(final String indexPattern) { - this.indexPattern = indexPattern; - } - - public ViewTarget(final StreamInput in) throws IOException { - this.indexPattern = in.readString(); - } - - public String getIndexPattern() { - return indexPattern; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(indexPattern); - } - - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; - - if (Strings.isNullOrEmpty(indexPattern)) { - validationException = ValidateActions.addValidationError("index pattern cannot be empty or null", validationException); - } - - return validationException; - } - - } - /** * Request for Creating View */ + @ExperimentalApi public static class Request extends ClusterManagerNodeRequest { private final String name; private final String description; - private final List targets; + private final List targets; - public Request(final String name, final String description, final List targets) { + public Request(final String name, final String description, final List targets) { this.name = name; - this.description = description; + this.description = Objects.requireNonNullElse(description, ""); this.targets = targets; } + public Request(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.description = in.readString(); + this.targets = in.readList(Target::new); + } + public String getName() { return name; } @@ -92,34 +74,42 @@ public String getDescription() { return description; } - public List getTargets() { + public List getTargets() { return new ArrayList<>(targets); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request that = (Request) o; + return name.equals(that.name) + && description.equals(that.description) + && targets.equals(that.targets); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, targets); + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (Strings.isNullOrEmpty(name)) { validationException = ValidateActions.addValidationError("Name is cannot be empty or null", validationException); } - if (targets.isEmpty()) { + if (CollectionUtils.isEmpty(targets)) { validationException = ValidateActions.addValidationError("targets cannot be empty", validationException); - } - - for (final ViewTarget target : targets) { - validationException = target.validate(); + } else { + for (final Target target : targets) { + validationException = target.validate(); + } } return validationException; } - public Request(final StreamInput in) throws IOException { - super(in); - this.name = in.readString(); - this.description = in.readString(); - this.targets = in.readList(ViewTarget::new); - } - @Override public void writeTo(final StreamOutput out) throws IOException { super.writeTo(out); @@ -127,26 +117,121 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeString(description); out.writeList(targets); } + + /** View target representation for create requests */ + @ExperimentalApi + public static class Target implements Writeable { + public final String indexPattern; + + public Target(final String indexPattern) { + this.indexPattern = indexPattern; + } + + public Target(final StreamInput in) throws IOException { + this.indexPattern = in.readString(); + } + + public String getIndexPattern() { + return indexPattern; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Target that = (Target) o; + return indexPattern.equals(that.indexPattern); + } + + @Override + public int hashCode() { + return Objects.hash(indexPattern); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(indexPattern); + } + + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(indexPattern)) { + validationException = ValidateActions.addValidationError("index pattern cannot be empty or null", validationException); + } + + return validationException; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "target", + args -> new Target((String) args[0]) + ); + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.Target.INDEX_PATTERN_FIELD); + } + + public static Target fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "create_view_request", + args -> new Request((String) args[0], (String) args[1], (List) args[2]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), View.DESCRIPTION_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Target.fromXContent(p), View.TARGETS_FIELD); + } + + public static Request fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } } - /** Response after view is created */ - public static class Response extends ActionResponse { + /** Response for view creation */ + @ExperimentalApi + public static class Response extends ActionResponse implements ToXContentObject { - private final org.opensearch.cluster.metadata.View createdView; + private final View createdView; - public Response(final org.opensearch.cluster.metadata.View createdView) { + public Response(final View createdView) { this.createdView = createdView; } public Response(final StreamInput in) throws IOException { super(in); - this.createdView = new org.opensearch.cluster.metadata.View(in); + this.createdView = new View(in); } @Override public void writeTo(final StreamOutput out) throws IOException { this.createdView.writeTo(out); } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field("view", createdView); + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "create_view_response", + args -> new Response((View) args[0]) + ); + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), View.PARSER, new ParseField("view")); + } + + public static Response fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } } /** @@ -171,22 +256,22 @@ public TransportAction( @Override protected String executor() { - return ThreadPool.Names.SAME; + return ThreadPool.Names.MANAGEMENT; } @Override - protected Response read(StreamInput in) throws IOException { + protected Response read(final StreamInput in) throws IOException { return new Response(in); } @Override - protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) throws Exception { viewService.createView(request, listener); } @Override - protected ClusterBlockException checkBlock(Request request, ClusterState state) { + protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java new file mode 100644 index 0000000000000..b27fb6339418b --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java @@ -0,0 +1,134 @@ +package org.opensearch.action.admin.indices.view; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Function; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.HandledTransportAction; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.tasks.Task; +import org.opensearch.transport.TransportService; + +import static org.opensearch.action.ValidateActions.addValidationError; + +/** Action to create a view */ +public class SearchViewAction extends ActionType { + + public static final SearchViewAction INSTANCE = new SearchViewAction(); + public static final String NAME = "cluster:admin:views:search"; + + private SearchViewAction() { + super(NAME, SearchResponse::new); + } + + /** Given a search request, creates a ViewSearchRequest */ + public static Request createRequestWith(final String view, final SearchRequest searchRequest) + throws IOException { + final BytesStreamOutput savedSearchRequest = new BytesStreamOutput(); + searchRequest.writeTo(savedSearchRequest); + savedSearchRequest.writeString(view); + + final BytesStreamInput input = new BytesStreamInput(savedSearchRequest.bytes().toBytesRef().bytes); + return new Request(input); + } + /** + * Wraps the functionality of search requests and tailors for what is available + * when searching through views + */ + @ExperimentalApi + public static class Request extends SearchRequest { + + private final String view; + + public Request(final String view) { + super(); + this.view = view; + } + + public Request(final StreamInput in) throws IOException { + super(in); + view = in.readString(); + } + + public String getView() { + return view; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request that = (Request) o; + return view.equals(that.view) + && super.equals(that); + } + + @Override + public int hashCode() { + return Objects.hash(view, super.hashCode()); + } + + @Override + public ActionRequestValidationException validate() { + final Function unsupported = (String x) -> x + " is not supported when searching views"; + ActionRequestValidationException validationException = super.validate(); + + if (scroll() != null) { + validationException = addValidationError(unsupported.apply("Scroll"), validationException); + } + + // TODO: Filter out anything additional search features that are not supported + + if (Strings.isNullOrEmpty(view)) { + validationException = addValidationError("View is required", validationException); + } + + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(view); + } + + @Override + public String toString() { + return super.toString().replace("SearchRequest{", "SearchViewAction.Request{view=" + view + ","); + } + } + + /** + * Transport Action for searching a View + */ + public static class TransportAction extends HandledTransportAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ActionFilters actionFilters, + final ViewService viewService) { + super(NAME, transportService, actionFilters, Request::new); + this.viewService = viewService; + } + + @Override + protected void doExecute(final Task task, final Request request, final ActionListener listener) { + viewService.searchView(request, listener); + } + } +} diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java similarity index 61% rename from server/src/main/java/org/opensearch/cluster/metadata/ViewService.java rename to server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 4056e477f2226..4839244f8179c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -1,13 +1,19 @@ -package org.opensearch.cluster.metadata; +package org.opensearch.action.admin.indices.view; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; @@ -16,9 +22,11 @@ public class ViewService { private final static Logger LOG = LogManager.getLogger(ViewService.class); private final ClusterService clusterService; + private final NodeClient client; - public ViewService(final ClusterService clusterService) { + public ViewService(final ClusterService clusterService, NodeClient client) { this.clusterService = clusterService; + this.client = client; } public void createView(final CreateViewAction.Request request, final ActionListener listener) { @@ -51,4 +59,25 @@ public void clusterStateProcessed(final String source, final ClusterState oldSta } }); } + + public void searchView(final SearchViewAction.Request request, final ActionListener listener) { + final Optional optView = Optional.ofNullable(clusterService) + .map(ClusterService::state) + .map(ClusterState::metadata) + .map(m -> m.views()) + .map(views -> views.get(request.getView())); + + if (optView.isEmpty()) { + throw new ResourceNotFoundException("no such view [" + request.getView() + "]"); + } + final View view = optView.get(); + + final String[] indices = view.getTargets().stream() + .map(View.Target::getIndexPattern) + .collect(Collectors.toList()) + .toArray(new String[0]); + request.indices(indices); + + client.executeLocally(SearchAction.INSTANCE, request, listener); + } } diff --git a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java b/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java deleted file mode 100644 index d457039f2a0fe..0000000000000 --- a/server/src/main/java/org/opensearch/action/search/ViewSearchRequest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.action.search; - -import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.cluster.metadata.View; -import org.opensearch.common.annotation.ExperimentalApi; -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import java.io.IOException; -import java.util.Objects; -import java.util.function.Function; - -import static org.opensearch.action.ValidateActions.addValidationError; - -/** Wraps the functionality of search requests and tailors for what is available when searching through views - */ -@ExperimentalApi -public class ViewSearchRequest extends SearchRequest { - - public final View view; - - public ViewSearchRequest(final View view) { - super(); - this.view = view; - } - - public ViewSearchRequest(final StreamInput in) throws IOException { - super(in); - view = new View(in); - } - - @Override - public ActionRequestValidationException validate() { - final Function unsupported = (String x) -> x + " is not supported when searching views"; - ActionRequestValidationException validationException = super.validate(); - - if (scroll() != null) { - validationException = addValidationError(unsupported.apply("Scroll"), validationException); - } - - // TODO: Filter out anything additional search features that are not supported - - return validationException; - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - super.writeTo(out); - view.writeTo(out); - } - - @Override - public boolean equals(final Object o) { - return this.hashCode() == o.hashCode(); - } - - @Override - public int hashCode() { - return Objects.hash(view, super.hashCode()); - } - - @Override - public String toString() { - return super.toString().replace("SearchRequest{", "ViewSearchRequest{view=" + view + ","); - } -} diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index 20dab1caa36c4..1e27073e0450d 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -125,6 +125,9 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.Nullable; @@ -838,4 +841,16 @@ public interface IndicesAdminClient extends OpenSearchClient { * Resolves names and wildcard expressions to indices, aliases, and data streams */ ActionFuture resolveIndex(ResolveIndexAction.Request request); + + /** Create a view */ + void createView(CreateViewAction.Request request, ActionListener listener); + + /** Create a view */ + ActionFuture createView(CreateViewAction.Request request); + + /** Search a view */ + void searchView(final SearchViewAction.Request request, final ActionListener listener); + + /** Search a view */ + ActionFuture searchView(final SearchViewAction.Request request); } diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index 786bfa38bb19c..c554316389953 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -312,6 +312,8 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkRequestBuilder; @@ -2070,6 +2072,26 @@ public void resolveIndex(ResolveIndexAction.Request request, ActionListener resolveIndex(ResolveIndexAction.Request request) { return execute(ResolveIndexAction.INSTANCE, request); } + + @Override + public void createView(CreateViewAction.Request request, ActionListener listener) { + execute(CreateViewAction.INSTANCE, request); + } + + @Override + public ActionFuture createView(CreateViewAction.Request request) { + return execute(CreateViewAction.INSTANCE, request); + } + + @Override + public void searchView(SearchViewAction.Request request, ActionListener listener) { + execute(SearchViewAction.INSTANCE, request); + } + + @Override + public ActionFuture searchView(SearchViewAction.Request request) { + return execute(SearchViewAction.INSTANCE, request); + } } @Override diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index 69e49e7aec6eb..59dc86ea28ed6 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -1348,7 +1348,7 @@ public Builder views(final Map views) { public Builder put(final View view) { Objects.requireNonNull(view, "view cannot be null"); final var replacementViews = new HashMap<>(getViews()); - replacementViews.put(view.name, view); + replacementViews.put(view.getName(), view); return views(replacementViews); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index 8db65c6afaebe..80efd8172b797 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -24,15 +24,15 @@ import java.util.List; import java.util.Objects; -/** TODO */ +/** View of data in OpenSearch indices */ @ExperimentalApi public class View extends AbstractDiffable implements ToXContentObject { - public final String name; - public final String description; - public final long createdAt; - public final long modifiedAt; - public final List targets; + private final String name; + private final String description; + private final long createdAt; + private final long modifiedAt; + private final List targets; public View(final String name, final String description, final Long createdAt, final Long modifiedAt, final List targets) { this.name = Objects.requireNonNull(name, "Name must be provided"); @@ -46,15 +46,53 @@ public View(final StreamInput in) throws IOException { this(in.readString(), in.readOptionalString(), in.readVLong(), in.readVLong(), in.readList(Target::new)); } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getModifiedAt() { + return modifiedAt; + } + + public List getTargets() { + return targets; + } + public static Diff readDiffFrom(final StreamInput in) throws IOException { return readDiffFrom(View::new, in); } - /** TODO */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + View that = (View) o; + return name.equals(that.name) + && description.equals(that.description) + && createdAt == that.createdAt + && modifiedAt == that.modifiedAt + && targets.equals(that.targets); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, createdAt, modifiedAt, targets); + } + + /** The source of data used to project the view */ @ExperimentalApi public static class Target implements Writeable, ToXContentObject { - public final String indexPattern; + private final String indexPattern; public Target(final String indexPattern) { this.indexPattern = Objects.requireNonNull(indexPattern, "IndexPattern is required"); @@ -64,7 +102,24 @@ public Target(final StreamInput in) throws IOException { this(in.readString()); } - private static final ParseField INDEX_PATTERN_FIELD = new ParseField("indexPattern"); + public String getIndexPattern() { + return indexPattern; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Target that = (Target) o; + return indexPattern.equals(that.indexPattern); + } + + @Override + public int hashCode() { + return Objects.hash(indexPattern); + } + + public static final ParseField INDEX_PATTERN_FIELD = new ParseField("indexPattern"); @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { @@ -74,16 +129,16 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa return builder; } - private static final ConstructingObjectParser T_PARSER = new ConstructingObjectParser<>( + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "target", args -> new Target((String) args[0]) ); static { - T_PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_PATTERN_FIELD); } public static Target fromXContent(final XContentParser parser) throws IOException { - return T_PARSER.parse(parser, null); + return PARSER.parse(parser, null); } @Override @@ -92,14 +147,14 @@ public void writeTo(final StreamOutput out) throws IOException { } } - private static final ParseField NAME_FIELD = new ParseField("name"); - private static final ParseField DESCRIPTION_FIELD = new ParseField("description"); - private static final ParseField CREATED_AT_FIELD = new ParseField("createdAt"); - private static final ParseField MODIFIED_AT_FIELD = new ParseField("modifiedAt"); - private static final ParseField TARGETS_FIELD = new ParseField("targets"); + public static final ParseField NAME_FIELD = new ParseField("name"); + public static final ParseField DESCRIPTION_FIELD = new ParseField("description"); + public static final ParseField CREATED_AT_FIELD = new ParseField("createdAt"); + public static final ParseField MODIFIED_AT_FIELD = new ParseField("modifiedAt"); + public static final ParseField TARGETS_FIELD = new ParseField("targets"); @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "view", args -> new View((String) args[0], (String) args[1], (Long) args[2], (Long) args[3], (List) args[4]) ); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java index 78f9707f4b536..1f9311b714d1b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -29,6 +29,7 @@ import static org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata.MINIMMAL_SUPPORTED_VERSION; +/** View metadata */ public class ViewMetadata implements Metadata.Custom { public static final String TYPE = "view"; @@ -141,7 +142,7 @@ public static class Builder { private final Map views = new HashMap<>(); public Builder putDataStream(final View view) { - views.put(view.name, view); + views.put(view.getName(), view); return this; } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index d26a7deae9e77..287c674e9246e 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -44,6 +44,7 @@ import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; +import org.opensearch.action.admin.indices.view.ViewService; import org.opensearch.action.search.SearchExecutionStatsCollector; import org.opensearch.action.search.SearchPhaseController; import org.opensearch.action.search.SearchRequestSlowLog; @@ -72,7 +73,6 @@ import org.opensearch.cluster.metadata.MetadataIndexUpgradeService; import org.opensearch.cluster.metadata.SystemIndexMetadataUpgradeService; import org.opensearch.cluster.metadata.TemplateUpgradeService; -import org.opensearch.cluster.metadata.ViewService; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.routing.BatchedRerouteService; @@ -204,8 +204,6 @@ import org.opensearch.repositories.RepositoriesModule; import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; -import org.opensearch.rest.action.admin.indices.RestViewAction; -import org.opensearch.rest.action.admin.indices.RestViewSearchAction; import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptModule; @@ -864,7 +862,8 @@ protected Node( ); final ViewService viewService = new ViewService( - clusterService + clusterService, + client ); Collection pluginComponents = pluginsService.filterPlugins(Plugin.class) diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 44ace1fcb1f6f..bfc97c309155e 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -11,12 +11,17 @@ import joptsimple.internal.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.search.SearchAction; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.ValidationException; import org.opensearch.common.inject.Inject; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.XContentBuilder; @@ -27,10 +32,15 @@ import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; import org.opensearch.rest.RestResponse; +import org.opensearch.rest.action.RestCancellableNodeClient; +import org.opensearch.rest.action.RestStatusToXContentListener; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.rest.action.search.RestSearchAction; import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.function.IntConsumer; import java.util.stream.Collectors; import static org.opensearch.rest.RestRequest.Method.DELETE; @@ -38,142 +48,153 @@ import static org.opensearch.rest.RestRequest.Method.POST; import static org.opensearch.rest.RestRequest.Method.PUT; -/** TODO */ -public class RestViewAction extends BaseRestHandler { +/** All rest handlers for view actions */ +public class RestViewAction { private final static Logger LOG = LogManager.getLogger(RestViewAction.class); public static final String VIEW_ID = "view_id"; + public static final String VIEW_ID_PARAMETER = "{" + VIEW_ID + "}"; - private final ClusterService clusterService; + /** Handler for create view */ + public static class CreateViewHandler extends BaseRestHandler { - @Inject - public RestViewAction(final ClusterService clusterService) { - this.clusterService = clusterService; - } - - @Override - public List routes() { - final String viewIdParameter = "{" + VIEW_ID + "}"; - - return List.of( - new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster:views:list").build(), - new NamedRoute.Builder().path("/views").method(POST).uniqueName("cluster:views:create").build(), - new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster:views:get").build(), - new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster:views:delete").build(), - new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster:views:update").build() - ); - } - - @Override - public String getName() { - return "view_actions"; - } - - @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - if (!request.hasParam(VIEW_ID)) { - if (request.method() == RestRequest.Method.GET) { - return channel -> channel.sendResponse(handleGet(request, channel.newBuilder())); - } - - if (request.method() == RestRequest.Method.POST) { - return channel -> handlePost(request, channel); - } - - } else if (request.hasParam(VIEW_ID)) { - if (request.method() == RestRequest.Method.GET) { - return channel -> channel.sendResponse(handleSingleGet(request, channel.newBuilder())); - } - - if (request.method() == RestRequest.Method.PUT) { - return channel -> handleSinglePut(request); - } - - if (request.method() == RestRequest.Method.DELETE) { - return channel -> handleSingleDelete(request); - } + @Override + public List routes() { + return List.of(new NamedRoute.Builder().path("/views").method(POST).uniqueName(CreateViewAction.NAME).build()); } - return channel -> channel.sendResponse( - new BytesRestResponse(RestStatus.BAD_REQUEST, "Unable to process " + request.method() + " on this endpoint " + request.path()) - ); - } - - public RestResponse handleGet(final RestRequest r, final XContentBuilder builder) throws IOException { - final List views = Optional.ofNullable(clusterService.state().getMetadata()) - .map(m -> m.views()) - .map(v -> v.values()) - .map(v -> v.stream().collect(Collectors.toList())) - .orElse(List.of()); - - return new BytesRestResponse(RestStatus.OK, builder.startObject().field("views", views).endObject()); - } - - public RestResponse handlePost(final RestRequest r, final RestChannel channel) throws IOException { - final View inputView; - try (final XContentParser parser = r.contentParser()) { - inputView = View.fromXContent(parser); + @Override + public String getName() { + return CreateViewAction.NAME; } - final long currentTime = System.currentTimeMillis(); - final View view = new View(inputView.name, inputView.description, currentTime, currentTime, inputView.targets); - - clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(final ClusterState currentState) throws Exception { - return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) - .build(); - } - - @Override - public void onFailure(final String source, final Exception e) { - LOG.error("Unable to create view, due to {}", source, e); - channel.sendResponse( - new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.") - ); - } - - @Override - public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { - try { - channel.sendResponse( - new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject()) - ); - } catch (final IOException e) { - // TODO? - LOG.error(e); - } + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + try (final XContentParser parser = request.contentParser()) { + final CreateViewAction.Request createViewAction = CreateViewAction.Request.fromXContent(parser); + return channel -> client.admin().indices().createView(createViewAction, new RestToXContentListener<>(channel)); } - }); - // TODO: Handle CREATED vs UPDATED - return null; + } } - public RestResponse handleSingleGet(final RestRequest r, final XContentBuilder builder) throws IOException { - final String viewId = r.param(VIEW_ID); - - if (Strings.isNullOrEmpty(viewId)) { - return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + public static class SearchViewHandler extends BaseRestHandler { + @Override + public List routes() { + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search").method(GET).uniqueName("cluster:views:search").build(), + new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search").method(POST).uniqueName("cluster:views:search").build() + ); } - final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) - .map(m -> m.views()) - .map(views -> views.get(viewId)); - - if (view.isEmpty()) { - return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + @Override + public String getName() { + return "view_search_action"; } - return new BytesRestResponse(RestStatus.OK, builder.startObject().value(view).endObject()); - } + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_ID); + + final SearchViewAction.Request viewSearchRequest = new SearchViewAction.Request(viewId); + final IntConsumer setSize = size -> viewSearchRequest.source().size(size); + + request.withContentOrSourceParamParserOrNull( + parser -> RestSearchAction.parseSearchRequest( + viewSearchRequest, + request, + parser, + client.getNamedWriteableRegistry(), + setSize + ) + ); + + final ValidationException validationResult = viewSearchRequest.validate(); + if (validationResult != null) { + throw validationResult; + } - public RestResponse handleSinglePut(final RestRequest r) { - return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); + return channel -> { + final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + cancelClient.execute(SearchAction.INSTANCE, viewSearchRequest, new RestStatusToXContentListener<>(channel)); + }; + } } - public RestResponse handleSingleDelete(final RestRequest r) { - return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); - } + // public List routes() { + + // return List.of( + // new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster:views:list").build(), + // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster:views:get").build(), + // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster:views:delete").build(), + // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster:views:update").build() + // ); + // } + + // public RestResponse handlePost(final RestRequest r, final RestChannel channel) throws IOException { + // final View inputView; + // try (final XContentParser parser = r.contentParser()) { + // inputView = View.fromXContent(parser); + // } + + // final long currentTime = System.currentTimeMillis(); + // final View view = new View(inputView.name, inputView.description, currentTime, currentTime, inputView.targets); + + // clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { + // @Override + // public ClusterState execute(final ClusterState currentState) throws Exception { + // return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) + // .build(); + // } + + // @Override + // public void onFailure(final String source, final Exception e) { + // LOG.error("Unable to create view, due to {}", source, e); + // channel.sendResponse( + // new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.") + // ); + // } + + // @Override + // public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + // try { + // channel.sendResponse( + // new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject()) + // ); + // } catch (final IOException e) { + // // TODO? + // LOG.error(e); + // } + // } + // }); + // // TODO: Handle CREATED vs UPDATED + // return null; + // } + + // public RestResponse handleSingleGet(final RestRequest r, final XContentBuilder builder) throws IOException { + // final String viewId = r.param(VIEW_ID); + + // if (Strings.isNullOrEmpty(viewId)) { + // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + // } + + // final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) + // .map(m -> m.views()) + // .map(views -> views.get(viewId)); + + // if (view.isEmpty()) { + // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + // } + + // return new BytesRestResponse(RestStatus.OK, builder.startObject().value(view).endObject()); + // } + + // public RestResponse handleSinglePut(final RestRequest r) { + // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); + // } + + // public RestResponse handleSingleDelete(final RestRequest r) { + // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); + // } } diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java deleted file mode 100644 index bdda2971ac38a..0000000000000 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewSearchAction.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.rest.action.admin.indices; - -import joptsimple.internal.Strings; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.search.SearchAction; -import org.opensearch.action.search.ViewSearchRequest; -import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.metadata.View; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.inject.Inject; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.BytesRestResponse; -import org.opensearch.rest.NamedRoute; -import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.RestCancellableNodeClient; -import org.opensearch.rest.action.RestStatusToXContentListener; -import org.opensearch.rest.action.search.RestSearchAction; - -import java.io.IOException; -import java.util.List; -import java.util.Optional; -import java.util.function.IntConsumer; -import java.util.stream.Collectors; - -import static org.opensearch.rest.RestRequest.Method.GET; -import static org.opensearch.rest.RestRequest.Method.POST; - -/** TODO */ -public class RestViewSearchAction extends BaseRestHandler { - - private final static Logger LOG = LogManager.getLogger(RestViewSearchAction.class); - - private static final String VIEW_ID = "view_id"; - - private final ClusterService clusterService; - - @Inject - public RestViewSearchAction(final ClusterService clusterService) { - this.clusterService = clusterService; - } - - @Override - public List routes() { - final String viewIdParameter = "{" + VIEW_ID + "}"; - - return List.of( - new NamedRoute.Builder().path("/views/" + viewIdParameter + "/_search").method(GET).uniqueName("cluster:views:search").build(), - new NamedRoute.Builder().path("/views/" + viewIdParameter + "/_search").method(POST).uniqueName("cluster:views:search").build() - ); - } - - @Override - public String getName() { - return "view_search_action"; - } - - @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final String viewId = request.param(VIEW_ID); - return channel -> { - - if (Strings.isNullOrEmpty(viewId)) { - channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "")); - } - - final Optional optView = Optional.ofNullable(clusterService.state().getMetadata()) - .map(m -> m.views()) - .map(views -> views.get(viewId)); - - if (optView.isEmpty()) { - channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "")); - } - final View view = optView.get(); - - final ViewSearchRequest viewSearchRequest = new ViewSearchRequest(view); - final IntConsumer setSize = size -> viewSearchRequest.source().size(size); - - request.withContentOrSourceParamParserOrNull( - parser -> RestSearchAction.parseSearchRequest( - viewSearchRequest, - request, - parser, - client.getNamedWriteableRegistry(), - setSize - ) - ); - - // TODO: Only allow operations that are supported - - final String[] indices = view.targets.stream() - .map(target -> target.indexPattern) - .collect(Collectors.toList()) - .toArray(new String[0]); - viewSearchRequest.indices(indices); - - // TODO: Look into resource leak on cancelClient? Note; is already leaking in - // server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java - final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); - cancelClient.execute(SearchAction.INSTANCE, viewSearchRequest, new RestStatusToXContentListener<>(channel)); - }; - } -} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java new file mode 100644 index 0000000000000..146b6e68ddef6 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java @@ -0,0 +1,51 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import java.util.List; + +public class CreateViewTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return CreateViewAction.Request::new; + } + + @Override + protected CreateViewAction.Request createTestInstance() { + return new CreateViewAction.Request(randomAlphaOfLength(8), randomAlphaOfLength(8), + randomList(5, () -> new CreateViewAction.Request.Target(randomAlphaOfLength(8)))); + } + + public void testValidateRequest() { + final CreateViewAction.Request request = new CreateViewAction.Request("my-view", "this is a description", + List.of(new CreateViewAction.Request.Target("my-indices-*"))); + assertNull(request.validate()); + } + + public void testValidateRequestWithoutName() { + final CreateViewAction.Request request = new CreateViewAction.Request("", null, null); + ActionRequestValidationException e = request.validate(); + assertNotNull(e); + assertThat(e.validationErrors().size(), equalTo(1)); + assertThat(e.validationErrors().get(0), containsString("name is missing")); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java new file mode 100644 index 0000000000000..2e2eb6a2ab5d2 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java @@ -0,0 +1,55 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.util.List; + +public class SearchViewTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return SearchViewAction.Request::new; + } + + @Override + protected SearchViewAction.Request createTestInstance() { + try { + return SearchViewAction.createRequestWith(randomAlphaOfLength(8), new SearchRequest()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + public void testValidateRequest() throws IOException { + final SearchViewAction.Request request = SearchViewAction.createRequestWith("my-view", new SearchRequest()); + assertNull(request.validate()); + } + + public void testValidateRequestWithoutName() { + final SearchViewAction.Request request = new SearchViewAction.Request((String)null); + ActionRequestValidationException e = request.validate(); + assertNotNull(e); + assertThat(e.validationErrors().size(), equalTo(1)); + assertThat(e.validationErrors().get(0), containsString("View is required")); + } + +} diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java index c1184ddeca915..9dc57f519aa43 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java @@ -14,19 +14,13 @@ package org.opensearch.cluster.metadata; import org.opensearch.cluster.metadata.View.Target; -import org.opensearch.common.UUIDs; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.index.Index; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import java.util.Locale; -import static org.opensearch.cluster.DataStreamTestHelper.createTimestampField; -import static org.opensearch.cluster.metadata.DataStream.getDefaultBackingIndexName; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; @@ -35,17 +29,13 @@ public class ViewTests extends AbstractSerializingTestCase { private static List randomTargets() { int numTargets = randomIntBetween(0, 128); - List targets = new ArrayList<>(numTargets); - for (int i = 0; i < numTargets; i++) { - targets.add(new Target(randomAlphaOfLength(10).toLowerCase(Locale.ROOT)); - } - return targets; + return randomList(numTargets, () -> new View.Target(randomAlphaOfLength(8))); } private static View randomInstance() { final List targets = randomTargets(); - final String viewName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); - final String description = randomAlphaOfLength(100).toLowerCase(Locale.ROOT); + final String viewName = randomAlphaOfLength(10); + final String description = randomAlphaOfLength(100); return new View(viewName, description, Math.abs(randomLong()), Math.abs(randomLong()), targets); } @@ -77,7 +67,7 @@ public void testNullTargets() { } public void testNullTargetIndexPattern() { - final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View.Target(null)); + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View.Target((String)null)); assertThat(npe.getMessage(), equalTo("IndexPattern is required")); } @@ -85,10 +75,10 @@ public void testNullTargetIndexPattern() { public void testDefaultValues() { final View view = new View("myName", null, null, null, List.of()); - assertThat(view.name, equalTo("myName")); - assertThat(view.description, equalTo(null)); - assertThat(view.createdAt, equalTo(-1L)); - assertThat(view.modifiedAt, equalTo(-1L)); - assertThat(view.targets, empty()); + assertThat(view.getName(), equalTo("myName")); + assertThat(view.getDescription(), equalTo(null)); + assertThat(view.getCreatedAt(), equalTo(-1L)); + assertThat(view.getModifiedAt(), equalTo(-1L)); + assertThat(view.getTargets(), empty()); } } From 2832b24ca12fd192f7e41ee63ed5dbe751d07387 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 5 Feb 2024 08:49:57 -0600 Subject: [PATCH 17/55] Spotless and compile issue Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 19 ++- .../admin/indices/view/CreateViewAction.java | 26 ++-- .../admin/indices/view/SearchViewAction.java | 22 ++- .../admin/indices/view/ViewService.java | 11 +- .../org/opensearch/cluster/metadata/View.java | 3 +- .../main/java/org/opensearch/node/Node.java | 5 +- .../action/admin/indices/RestViewAction.java | 141 ++++++++---------- .../admin/indices/view/CreateViewTests.java | 18 ++- .../admin/indices/view/SearchViewTests.java | 7 +- .../cluster/metadata/ViewTests.java | 2 +- 10 files changed, 121 insertions(+), 133 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index ac706491478d1..2da1f5d187e2d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -8,11 +8,6 @@ package org.opensearch.action.admin.indices.view; -import static org.hamcrest.Matchers.is; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; - -import java.util.List; - import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.index.IndexNotFoundException; @@ -20,7 +15,11 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; -import org.opensearch.test.junit.annotations.TestLogging; + +import java.util.List; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, numDataNodes = 2) public class ViewIT extends OpenSearchIntegTestCase { @@ -40,7 +39,11 @@ private int createIndexWithDocs(final String indexName) throws Exception { } private CreateViewAction.Response createView(final String name, final String indexPattern) throws Exception { - final CreateViewAction.Request request = new CreateViewAction.Request(name, null, List.of(new CreateViewAction.Request.Target(indexPattern))); + final CreateViewAction.Request request = new CreateViewAction.Request( + name, + null, + List.of(new CreateViewAction.Request.Target(indexPattern)) + ); final CreateViewAction.Response response = client().admin().indices().createView(request).actionGet(); performRemoteStoreTestAction(); return response; @@ -48,7 +51,7 @@ private CreateViewAction.Response createView(final String name, final String ind private SearchResponse searchView(final String viewName) throws Exception { final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); - final SearchResponse response = client().searchView(request).actionGet(); + final SearchResponse response = client().admin().indices().searchView(request).actionGet(); return response; } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index f4e8b1db1956a..c47eb9ea5432d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -1,10 +1,5 @@ package org.opensearch.action.admin.indices.view; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; import org.opensearch.action.ValidateActions; @@ -34,6 +29,11 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + /** Action to create a view */ public class CreateViewAction extends ActionType { @@ -83,11 +83,9 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request that = (Request) o; - return name.equals(that.name) - && description.equals(that.description) - && targets.equals(that.targets); + return name.equals(that.name) && description.equals(that.description) && targets.equals(that.targets); } - + @Override public int hashCode() { return Objects.hash(name, description, targets); @@ -142,7 +140,7 @@ public boolean equals(Object o) { Target that = (Target) o; return indexPattern.equals(that.indexPattern); } - + @Override public int hashCode() { return Objects.hash(indexPattern); @@ -170,7 +168,7 @@ public ActionRequestValidationException validate() { static { PARSER.declareString(ConstructingObjectParser.constructorArg(), View.Target.INDEX_PATTERN_FIELD); } - + public static Target fromXContent(final XContentParser parser) throws IOException { return PARSER.parse(parser, null); } @@ -187,7 +185,7 @@ public static Target fromXContent(final XContentParser parser) throws IOExceptio PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), View.DESCRIPTION_FIELD); PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Target.fromXContent(p), View.TARGETS_FIELD); } - + public static Request fromXContent(final XContentParser parser) throws IOException { return PARSER.parse(parser, null); } @@ -197,7 +195,7 @@ public static Request fromXContent(final XContentParser parser) throws IOExcepti @ExperimentalApi public static class Response extends ActionResponse implements ToXContentObject { - private final View createdView; + private final View createdView; public Response(final View createdView) { this.createdView = createdView; @@ -220,7 +218,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.endObject(); return builder; } - + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "create_view_response", args -> new Response((View) args[0]) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java index b27fb6339418b..b089d04751a63 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java @@ -1,9 +1,5 @@ package org.opensearch.action.admin.indices.view; -import java.io.IOException; -import java.util.Objects; -import java.util.function.Function; - import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; import org.opensearch.action.search.SearchRequest; @@ -21,6 +17,10 @@ import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.Objects; +import java.util.function.Function; + import static org.opensearch.action.ValidateActions.addValidationError; /** Action to create a view */ @@ -34,8 +34,7 @@ private SearchViewAction() { } /** Given a search request, creates a ViewSearchRequest */ - public static Request createRequestWith(final String view, final SearchRequest searchRequest) - throws IOException { + public static Request createRequestWith(final String view, final SearchRequest searchRequest) throws IOException { final BytesStreamOutput savedSearchRequest = new BytesStreamOutput(); searchRequest.writeTo(savedSearchRequest); savedSearchRequest.writeString(view); @@ -43,6 +42,7 @@ public static Request createRequestWith(final String view, final SearchRequest s final BytesStreamInput input = new BytesStreamInput(savedSearchRequest.bytes().toBytesRef().bytes); return new Request(input); } + /** * Wraps the functionality of search requests and tailors for what is available * when searching through views @@ -71,10 +71,9 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request that = (Request) o; - return view.equals(that.view) - && super.equals(that); + return view.equals(that.view) && super.equals(that); } - + @Override public int hashCode() { return Objects.hash(view, super.hashCode()); @@ -118,10 +117,7 @@ public static class TransportAction extends HandledTransportAction pluginComponents = pluginsService.filterPlugins(Plugin.class) .stream() diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index bfc97c309155e..126348e71c971 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -8,30 +8,17 @@ package org.opensearch.rest.action.admin.indices; -import joptsimple.internal.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.admin.indices.view.CreateViewAction; import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.action.search.SearchAction; import org.opensearch.client.node.NodeClient; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.ClusterStateUpdateTask; -import org.opensearch.cluster.metadata.Metadata; -import org.opensearch.cluster.metadata.View; -import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.ValidationException; -import org.opensearch.common.inject.Inject; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.BaseRestHandler; -import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.NamedRoute; -import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestCancellableNodeClient; import org.opensearch.rest.action.RestStatusToXContentListener; import org.opensearch.rest.action.RestToXContentListener; @@ -39,14 +26,10 @@ import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.function.IntConsumer; -import java.util.stream.Collectors; -import static org.opensearch.rest.RestRequest.Method.DELETE; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; -import static org.opensearch.rest.RestRequest.Method.PUT; /** All rest handlers for view actions */ public class RestViewAction { @@ -82,8 +65,14 @@ public static class SearchViewHandler extends BaseRestHandler { @Override public List routes() { return List.of( - new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search").method(GET).uniqueName("cluster:views:search").build(), - new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search").method(POST).uniqueName("cluster:views:search").build() + new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search") + .method(GET) + .uniqueName("cluster:views:search") + .build(), + new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search") + .method(POST) + .uniqueName("cluster:views:search") + .build() ); } @@ -114,7 +103,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC throw validationResult; } - return channel -> { + return channel -> { final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); cancelClient.execute(SearchAction.INSTANCE, viewSearchRequest, new RestStatusToXContentListener<>(channel)); }; @@ -123,78 +112,78 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC // public List routes() { - // return List.of( - // new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster:views:list").build(), - // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster:views:get").build(), - // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster:views:delete").build(), - // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster:views:update").build() - // ); + // return List.of( + // new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster:views:list").build(), + // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster:views:get").build(), + // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster:views:delete").build(), + // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster:views:update").build() + // ); // } // public RestResponse handlePost(final RestRequest r, final RestChannel channel) throws IOException { - // final View inputView; - // try (final XContentParser parser = r.contentParser()) { - // inputView = View.fromXContent(parser); - // } - - // final long currentTime = System.currentTimeMillis(); - // final View view = new View(inputView.name, inputView.description, currentTime, currentTime, inputView.targets); - - // clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { - // @Override - // public ClusterState execute(final ClusterState currentState) throws Exception { - // return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) - // .build(); - // } - - // @Override - // public void onFailure(final String source, final Exception e) { - // LOG.error("Unable to create view, due to {}", source, e); - // channel.sendResponse( - // new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.") - // ); - // } - - // @Override - // public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { - // try { - // channel.sendResponse( - // new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject()) - // ); - // } catch (final IOException e) { - // // TODO? - // LOG.error(e); - // } - // } - // }); - // // TODO: Handle CREATED vs UPDATED - // return null; + // final View inputView; + // try (final XContentParser parser = r.contentParser()) { + // inputView = View.fromXContent(parser); + // } + + // final long currentTime = System.currentTimeMillis(); + // final View view = new View(inputView.name, inputView.description, currentTime, currentTime, inputView.targets); + + // clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { + // @Override + // public ClusterState execute(final ClusterState currentState) throws Exception { + // return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) + // .build(); + // } + + // @Override + // public void onFailure(final String source, final Exception e) { + // LOG.error("Unable to create view, due to {}", source, e); + // channel.sendResponse( + // new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.") + // ); + // } + + // @Override + // public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + // try { + // channel.sendResponse( + // new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject()) + // ); + // } catch (final IOException e) { + // // TODO? + // LOG.error(e); + // } + // } + // }); + // // TODO: Handle CREATED vs UPDATED + // return null; // } // public RestResponse handleSingleGet(final RestRequest r, final XContentBuilder builder) throws IOException { - // final String viewId = r.param(VIEW_ID); + // final String viewId = r.param(VIEW_ID); - // if (Strings.isNullOrEmpty(viewId)) { - // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); - // } + // if (Strings.isNullOrEmpty(viewId)) { + // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + // } - // final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) - // .map(m -> m.views()) - // .map(views -> views.get(viewId)); + // final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) + // .map(m -> m.views()) + // .map(views -> views.get(viewId)); - // if (view.isEmpty()) { - // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); - // } + // if (view.isEmpty()) { + // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); + // } - // return new BytesRestResponse(RestStatus.OK, builder.startObject().value(view).endObject()); + // return new BytesRestResponse(RestStatus.OK, builder.startObject().value(view).endObject()); // } // public RestResponse handleSinglePut(final RestRequest r) { - // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); + // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); // } // public RestResponse handleSingleDelete(final RestRequest r) { - // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); + // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); // } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java index 146b6e68ddef6..3879da72243af 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java @@ -16,11 +16,11 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; +import java.util.List; + import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import java.util.List; - public class CreateViewTests extends AbstractWireSerializingTestCase { @Override @@ -30,13 +30,19 @@ protected Writeable.Reader instanceReader() { @Override protected CreateViewAction.Request createTestInstance() { - return new CreateViewAction.Request(randomAlphaOfLength(8), randomAlphaOfLength(8), - randomList(5, () -> new CreateViewAction.Request.Target(randomAlphaOfLength(8)))); + return new CreateViewAction.Request( + randomAlphaOfLength(8), + randomAlphaOfLength(8), + randomList(5, () -> new CreateViewAction.Request.Target(randomAlphaOfLength(8))) + ); } public void testValidateRequest() { - final CreateViewAction.Request request = new CreateViewAction.Request("my-view", "this is a description", - List.of(new CreateViewAction.Request.Target("my-indices-*"))); + final CreateViewAction.Request request = new CreateViewAction.Request( + "my-view", + "this is a description", + List.of(new CreateViewAction.Request.Target("my-indices-*")) + ); assertNull(request.validate()); } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java index 2e2eb6a2ab5d2..1e56676a10e29 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java @@ -17,12 +17,11 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; +import java.io.IOException; + import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import java.io.IOException; -import java.util.List; - public class SearchViewTests extends AbstractWireSerializingTestCase { @Override @@ -45,7 +44,7 @@ public void testValidateRequest() throws IOException { } public void testValidateRequestWithoutName() { - final SearchViewAction.Request request = new SearchViewAction.Request((String)null); + final SearchViewAction.Request request = new SearchViewAction.Request((String) null); ActionRequestValidationException e = request.validate(); assertNotNull(e); assertThat(e.validationErrors().size(), equalTo(1)); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java index 9dc57f519aa43..e417dbd1b33a2 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java @@ -67,7 +67,7 @@ public void testNullTargets() { } public void testNullTargetIndexPattern() { - final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View.Target((String)null)); + final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View.Target((String) null)); assertThat(npe.getMessage(), equalTo("IndexPattern is required")); } From db00415391d624ec05214431d545bb69ac93f514 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 5 Feb 2024 09:31:44 -0600 Subject: [PATCH 18/55] Clean up license headers Signed-off-by: Peter Nied Signed-off-by: Peter Nied --- .../admin/indices/view/CreateViewAction.java | 8 + .../admin/indices/view/SearchViewAction.java | 8 + .../admin/indices/view/ViewService.java | 8 + .../opensearch/index/view/package-info.java | 10 -- .../org/opensearch/index/view/views-design.md | 168 ------------------ .../action/admin/indices/RestViewAction.java | 2 + .../admin/indices/view/CreateViewTests.java | 4 - .../admin/indices/view/SearchViewTests.java | 4 - .../cluster/metadata/ViewTests.java | 5 - 9 files changed, 26 insertions(+), 191 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/index/view/package-info.java delete mode 100644 server/src/main/java/org/opensearch/index/view/views-design.md diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index c47eb9ea5432d..ea563986d154e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -1,3 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + package org.opensearch.action.admin.indices.view; import org.opensearch.action.ActionRequestValidationException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java index b089d04751a63..2e475eaecd5d2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java @@ -1,3 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + package org.opensearch.action.admin.indices.view; import org.opensearch.action.ActionRequestValidationException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 29ec6b3ba29aa..9b7a3a38e061e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -1,3 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + package org.opensearch.action.admin.indices.view; import org.apache.logging.log4j.LogManager; diff --git a/server/src/main/java/org/opensearch/index/view/package-info.java b/server/src/main/java/org/opensearch/index/view/package-info.java deleted file mode 100644 index bb65723bdd5cd..0000000000000 --- a/server/src/main/java/org/opensearch/index/view/package-info.java +++ /dev/null @@ -1,10 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/** Core classes responsible for handling all view operations */ -package org.opensearch.index.view; diff --git a/server/src/main/java/org/opensearch/index/view/views-design.md b/server/src/main/java/org/opensearch/index/view/views-design.md deleted file mode 100644 index 4bd5d9585c5c2..0000000000000 --- a/server/src/main/java/org/opensearch/index/view/views-design.md +++ /dev/null @@ -1,168 +0,0 @@ -# Views - -Views define how searches are performed against indices on a cluster, uniform data access that is configured separately from the queries. - -## Design - -### View data - -Views create a mapping to the resources that hold information to be searched over in a consistent manner. This abstraction allows for indirection with the backing indices, so they might be changed without callers being impacted. This can also be used to simplify the security model - searches over views do not require permissions to the backing indices only permissions to the view itself. - -```mermaid -classDiagram - class View { - +String name - +String description - +long createdAt - +long modifiedAt - +List targets - +toXContent(XContentBuilder, Params) XContentBuilder - +writeTo(StreamOutput) void - } - class Target { - +String indexPattern - +toXContent(XContentBuilder, Params) XContentBuilder - +writeTo(StreamOutput) void - } - class StreamOutput - class XContentBuilder - - View -- Target : contains - View -- StreamOutput : writes to - View -- XContentBuilder : outputs to - Target -- StreamOutput : writes to - Target -- XContentBuilder : outputs to -``` - -### View persistence - -Views are long lived objects in OpenSearch, all operations on them should be fully committed before responding to the caller. Views are intentionally created for user scenarios following a similar creation cadence to indices. - -Committed implies that the updates are synchronized across all nodes in a cluster. The Cluster Metadata Store is already available and allows for acknowledging that changes have been applied to all nodes. While this data could be stored in a new purpose built index, index data replication has delays and ensuring synchronization is non-trivial to implement as is seen in the Security plugins [1]. - -- [1] https://github.com/opensearch-project/security/issues/3275 - -```mermaid -sequenceDiagram - participant Client - participant HTTP_Request as ActionHandler - participant Cluster_Metadata as Cluster Metadata Store - participant Data_Store as Indices - - Client->>HTTP_Request: View List/Get/Update/Create/Delete
/views or /views/{view_id} - HTTP_Request->>Cluster_Metadata: Query Views - alt Update/Create/Delete - Cluster_Metadata->>Cluster_Metadata: Refresh Cluster - end - Cluster_Metadata-->>HTTP_Request: Return - HTTP_Request-->>Client: Return - - Client->>HTTP_Request: Search View
/views/{view_id}/search - HTTP_Request->>Cluster_Metadata: Query Views - Cluster_Metadata-->>HTTP_Request: Return - HTTP_Request->>HTTP_Request: Rewrite Search Request - HTTP_Request->>HTTP_Request: Validate Search Request - HTTP_Request->>Data_Store: Search indices - Data_Store-->>HTTP_Request: Return - HTTP_Request-->>Client: Return -``` -## Appendix - -### Local Testing - -``` -curl localhost:9200/abc/_doc \ - -XPOST \ - --header "Content-Type: application/json" \ - --data '{"foo":"bar"}' \ - -curl localhost:9200/views \ - -XPOST \ - --header "Content-Type: application/json" \ - --data '{"name":"hi", "createdAt": -1, "modifiedAt": -1, "targets":[]}' \ - -v - -curl localhost:9200/views \ - -XPOST \ - --header "Content-Type: application/json" \ - --data '{"name":"hi", "createdAt": -1, "modifiedAt": -1, "targets":[{"indexPattern":"abc"}]}' \ - -v - - -curl localhost:9200/views/hi/_search -``` - -### v0 View Data Model - -``` -VIEW MODEL -{ - name: STRING, // [Optional] Friendly name resolves to ID - id: STRING, // Non-mutatable identifier - description: STRING, // [Optional] Description of the view - created: DATE, // Creation time of the view - modified: DATE // Last modified time of the view - query: QUERY, // enforced query - filter: QUERY, // P2 enforced query after transformations - targets: [ - { - indexPattern: STRING, // No wildcard/aliases! - // P2 Allow wildcard/aliases query parameter - query: QUERY, // enforced query specific for this target - filter: QUERY, // P2 enforced query specific after transformations - documentTransformer: SCRIPT // P2 Convert the results in some way - } - ], - documentTransformer: SCRIPT // P2 Convert the results in some way -} -``` - -#### View Operations - -| Method | Path | -| - | - | -| POST | /views | -| GET | /views/{view_id} | -| PUT | /views/{view_id} | -| PATCH | /views/{view_id} | -| DELETE | /views/{view_id} | - -#### Enumerate Views - -| Method | Path | -| - | - | -| GET | /views | - -#### Perform a Search on a view -| Method | Path | -| - | - | -| GET | /views/{view_id}/_search | -| POST | /views/{view_id}/_search | - -#### Search Views // P2? -| Method | Path | -| - | - | -| GET | /views/_search | -| POST | /views/_search | - -#### Mapping // P2? Need to understand the utility / impact of not having this -| Method | Path | -| - | - | -| GET | /views/{view_id}/_mappings | -| PUT | /views/{view_id}/_mappings | -| PATCH | /views/{view_id}/_mappings | - - -*Results do not include any fields '_', how to protect leaking data?* - -#### Response on Create/Enumerate/Search - -views: [ - { - name: STRING, - id: STRING, - description: STRING, - created: DATE, - modified: DATE - } -] diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 126348e71c971..486a7e9da9706 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -110,6 +110,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } } + // TODO: Replace and reorganize this layout + // public List routes() { // return List.of( diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java index 3879da72243af..49d13a260ad3e 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java @@ -5,10 +5,6 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ package org.opensearch.action.admin.indices.view; diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java index 1e56676a10e29..e07104cbb6ad6 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java @@ -5,10 +5,6 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ package org.opensearch.action.admin.indices.view; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java index e417dbd1b33a2..145d3277bbf14 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java @@ -6,11 +6,6 @@ * compatible open source license. */ -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.cluster.metadata; import org.opensearch.cluster.metadata.View.Target; From 6a947e4f2428b132243f61d5e6f52466b12a1e45 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 5 Feb 2024 20:13:56 +0000 Subject: [PATCH 19/55] Fix logger usage miss Signed-off-by: Peter Nied --- .../org/opensearch/action/admin/indices/view/ViewService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 9b7a3a38e061e..4f87e2570d524 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -55,7 +55,7 @@ public ClusterState execute(final ClusterState currentState) throws Exception { @Override public void onFailure(final String source, final Exception e) { - LOG.error("Unable to create view, due to {}", source, e); + LOG.error("Unable to create view, in source " + source, e); listener.onFailure(e); } From 6156eb92dbb35671fcb28785fade7197877d6edd Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 5 Feb 2024 23:23:00 +0000 Subject: [PATCH 20/55] Before adding all action types Signed-off-by: Peter Nied --- .../admin/indices/view/CreateViewAction.java | 5 +++-- .../admin/indices/view/DeleteViewAction.java | 20 +++++++++++++++++++ .../admin/indices/view/GetViewAction.java | 20 +++++++++++++++++++ .../indices/view/ListViewNamesAction.java | 20 +++++++++++++++++++ .../admin/indices/view/SearchViewAction.java | 6 ++++-- .../admin/indices/view/UpdateViewAction.java | 20 +++++++++++++++++++ .../admin/indices/view/ViewService.java | 2 ++ .../java/org/opensearch/client/Client.java | 7 +++++++ .../opensearch/client/IndicesAdminClient.java | 8 -------- .../cluster/metadata/ViewMetadata.java | 2 ++ .../action/admin/indices/RestViewAction.java | 11 +++++----- .../transport/TransportService.java | 3 ++- .../admin/indices/view/CreateViewTests.java | 9 ++++----- 13 files changed, 110 insertions(+), 23 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index ea563986d154e..b8647f62fdf06 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -43,10 +43,11 @@ import java.util.Objects; /** Action to create a view */ +@ExperimentalApi public class CreateViewAction extends ActionType { public static final CreateViewAction INSTANCE = new CreateViewAction(); - public static final String NAME = "cluster:admin:views:create"; + public static final String NAME = "cluster:admin/views/create"; private CreateViewAction() { super(NAME, CreateViewAction.Response::new); @@ -103,7 +104,7 @@ public int hashCode() { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (Strings.isNullOrEmpty(name)) { - validationException = ValidateActions.addValidationError("Name is cannot be empty or null", validationException); + validationException = ValidateActions.addValidationError("name cannot be empty or null", validationException); } if (CollectionUtils.isEmpty(targets)) { validationException = ValidateActions.addValidationError("targets cannot be empty", validationException); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java new file mode 100644 index 0000000000000..dd96a92b55be7 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** Action to delete a view */ +@ExperimentalApi +public class DeleteViewAction { + + public static final DeleteViewAction INSTANCE = new DeleteViewAction(); + public static final String NAME = "cluster:admin/views/delete"; + +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java new file mode 100644 index 0000000000000..584ca50deaee3 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** Action to get a view */ +@ExperimentalApi +public class GetViewAction { + + public static final GetViewAction INSTANCE = new GetViewAction(); + public static final String NAME = "views:data/read/get"; + +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java new file mode 100644 index 0000000000000..e793f7900ae68 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** Action to list a view names */ +@ExperimentalApi +public class ListViewNamesAction { + + public static final ListViewNamesAction INSTANCE = new ListViewNamesAction(); + public static final String NAME = "views:data/read/list"; + +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java index 2e475eaecd5d2..25c0906cc2ffd 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java @@ -32,10 +32,11 @@ import static org.opensearch.action.ValidateActions.addValidationError; /** Action to create a view */ +@ExperimentalApi public class SearchViewAction extends ActionType { public static final SearchViewAction INSTANCE = new SearchViewAction(); - public static final String NAME = "cluster:admin:views:search"; + public static final String NAME = "views:data/read/search"; private SearchViewAction() { super(NAME, SearchResponse::new); @@ -96,7 +97,8 @@ public ActionRequestValidationException validate() { validationException = addValidationError(unsupported.apply("Scroll"), validationException); } - // TODO: Filter out anything additional search features that are not supported + // TODO: Filter out any additional search features that are not supported. + // Required before removing @ExperimentalApi annotations. if (Strings.isNullOrEmpty(view)) { validationException = addValidationError("View is required", validationException); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java new file mode 100644 index 0000000000000..a559d6b88d589 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.common.annotation.ExperimentalApi; + +/** Action to update a view */ +@ExperimentalApi +public class UpdateViewAction { + + public static final UpdateViewAction INSTANCE = new UpdateViewAction(); + public static final String NAME = "cluster:admin/views/update"; + +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 4f87e2570d524..48e5cd76b0c83 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -19,6 +19,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.action.ActionListener; import java.util.List; @@ -26,6 +27,7 @@ import java.util.stream.Collectors; /** Service to interact with views, create, retrieve, update, and delete */ +@ExperimentalApi public class ViewService { private final static Logger LOG = LogManager.getLogger(ViewService.class); diff --git a/server/src/main/java/org/opensearch/client/Client.java b/server/src/main/java/org/opensearch/client/Client.java index f4ae383249f61..d7679d0ae1e3a 100644 --- a/server/src/main/java/org/opensearch/client/Client.java +++ b/server/src/main/java/org/opensearch/client/Client.java @@ -34,6 +34,7 @@ import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.PitSegmentsRequest; +import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkRequestBuilder; import org.opensearch.action.bulk.BulkResponse; @@ -466,6 +467,12 @@ public interface Client extends OpenSearchClient, Releasable { */ void fieldCaps(FieldCapabilitiesRequest request, ActionListener listener); + /** Search a view */ + void searchView(final SearchViewAction.Request request, final ActionListener listener); + + /** Search a view */ + ActionFuture searchView(final SearchViewAction.Request request); + /** * Returns this clients settings */ diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index 1e27073e0450d..ec7f5b1ae5131 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -126,8 +126,6 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.opensearch.action.admin.indices.view.CreateViewAction; -import org.opensearch.action.admin.indices.view.SearchViewAction; -import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.Nullable; @@ -847,10 +845,4 @@ public interface IndicesAdminClient extends OpenSearchClient { /** Create a view */ ActionFuture createView(CreateViewAction.Request request); - - /** Search a view */ - void searchView(final SearchViewAction.Request request, final ActionListener listener); - - /** Search a view */ - ActionFuture searchView(final SearchViewAction.Request request); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java index 1f9311b714d1b..dac11d55fddaa 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -12,6 +12,7 @@ import org.opensearch.cluster.Diff; import org.opensearch.cluster.DiffableUtils; import org.opensearch.cluster.NamedDiff; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; @@ -30,6 +31,7 @@ import static org.opensearch.cluster.metadata.ComposableIndexTemplateMetadata.MINIMMAL_SUPPORTED_VERSION; /** View metadata */ +@ExperimentalApi public class ViewMetadata implements Metadata.Custom { public static final String TYPE = "view"; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 486a7e9da9706..b8018936f0b89 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -12,9 +12,9 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.view.CreateViewAction; import org.opensearch.action.admin.indices.view.SearchViewAction; -import org.opensearch.action.search.SearchAction; import org.opensearch.client.node.NodeClient; import org.opensearch.common.ValidationException; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.NamedRoute; @@ -32,6 +32,7 @@ import static org.opensearch.rest.RestRequest.Method.POST; /** All rest handlers for view actions */ +@ExperimentalApi public class RestViewAction { private final static Logger LOG = LogManager.getLogger(RestViewAction.class); @@ -67,18 +68,18 @@ public List routes() { return List.of( new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search") .method(GET) - .uniqueName("cluster:views:search") + .uniqueName(SearchViewAction.NAME) .build(), new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search") .method(POST) - .uniqueName("cluster:views:search") + .uniqueName(SearchViewAction.NAME) .build() ); } @Override public String getName() { - return "view_search_action"; + return SearchViewAction.NAME; } @Override @@ -105,7 +106,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> { final RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); - cancelClient.execute(SearchAction.INSTANCE, viewSearchRequest, new RestStatusToXContentListener<>(channel)); + cancelClient.execute(SearchViewAction.INSTANCE, viewSearchRequest, new RestStatusToXContentListener<>(channel)); }; } } diff --git a/server/src/main/java/org/opensearch/transport/TransportService.java b/server/src/main/java/org/opensearch/transport/TransportService.java index a1697b1898eeb..c6db9a7c7598d 100644 --- a/server/src/main/java/org/opensearch/transport/TransportService.java +++ b/server/src/main/java/org/opensearch/transport/TransportService.java @@ -1114,7 +1114,8 @@ public TransportAddress[] addressesFromString(String address) throws UnknownHost "cluster:admin", "cluster:monitor", "cluster:internal", - "internal:" + "internal:", + "views:" ) ) ); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java index 49d13a260ad3e..7b4d29d7fb839 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java @@ -14,8 +14,7 @@ import java.util.List; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.contains; public class CreateViewTests extends AbstractWireSerializingTestCase { @@ -39,15 +38,15 @@ public void testValidateRequest() { "this is a description", List.of(new CreateViewAction.Request.Target("my-indices-*")) ); + assertNull(request.validate()); } public void testValidateRequestWithoutName() { final CreateViewAction.Request request = new CreateViewAction.Request("", null, null); ActionRequestValidationException e = request.validate(); - assertNotNull(e); - assertThat(e.validationErrors().size(), equalTo(1)); - assertThat(e.validationErrors().get(0), containsString("name is missing")); + + assertThat(e.validationErrors(), contains("name cannot be empty or null", "targets cannot be empty")); } } From a7dbd934b708c7da94cdb3d787c25ef749e2657b Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 5 Feb 2024 23:23:24 +0000 Subject: [PATCH 21/55] Expand action types Signed-off-by: Peter Nied --- .../admin/indices/view/CreateViewAction.java | 53 +----- .../admin/indices/view/DeleteViewAction.java | 138 +++++++++++++- .../admin/indices/view/GetViewAction.java | 171 ++++++++++++++++++ .../indices/view/ListViewNamesAction.java | 56 ++++++ .../admin/indices/view/UpdateViewAction.java | 70 ++++++- .../admin/indices/view/ViewService.java | 111 ++++++++++-- .../client/support/AbstractClient.java | 20 +- .../action/admin/indices/RestViewAction.java | 27 +++ 8 files changed, 570 insertions(+), 76 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index b8647f62fdf06..395e3307218c0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -44,13 +44,13 @@ /** Action to create a view */ @ExperimentalApi -public class CreateViewAction extends ActionType { +public class CreateViewAction extends ActionType { public static final CreateViewAction INSTANCE = new CreateViewAction(); public static final String NAME = "cluster:admin/views/create"; private CreateViewAction() { - super(NAME, CreateViewAction.Response::new); + super(NAME, GetViewAction.Response::new); } /** @@ -200,51 +200,10 @@ public static Request fromXContent(final XContentParser parser) throws IOExcepti } } - /** Response for view creation */ - @ExperimentalApi - public static class Response extends ActionResponse implements ToXContentObject { - - private final View createdView; - - public Response(final View createdView) { - this.createdView = createdView; - } - - public Response(final StreamInput in) throws IOException { - super(in); - this.createdView = new View(in); - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - this.createdView.writeTo(out); - } - - @Override - public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject(); - builder.field("view", createdView); - builder.endObject(); - return builder; - } - - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "create_view_response", - args -> new Response((View) args[0]) - ); - static { - PARSER.declareObject(ConstructingObjectParser.constructorArg(), View.PARSER, new ParseField("view")); - } - - public static Response fromXContent(final XContentParser parser) throws IOException { - return PARSER.parse(parser, null); - } - } - /** * Transport Action for creating a View */ - public static class TransportAction extends TransportClusterManagerNodeAction { + public static class TransportAction extends TransportClusterManagerNodeAction { private final ViewService viewService; @@ -267,12 +226,12 @@ protected String executor() { } @Override - protected Response read(final StreamInput in) throws IOException { - return new Response(in); + protected GetViewAction.Response read(final StreamInput in) throws IOException { + return new GetViewAction.Response(in); } @Override - protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) + protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) throws Exception { viewService.createView(request, listener); } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java index dd96a92b55be7..d2266b4ebd96b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java @@ -8,13 +8,149 @@ package org.opensearch.action.admin.indices.view; +import static org.mockito.Mockito.description; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.ValidateActions; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable.Reader; +import org.opensearch.core.common.util.CollectionUtils; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; /** Action to delete a view */ +@SuppressWarnings("deprecation") @ExperimentalApi -public class DeleteViewAction { +public class DeleteViewAction extends ActionType { public static final DeleteViewAction INSTANCE = new DeleteViewAction(); public static final String NAME = "cluster:admin/views/delete"; + public DeleteViewAction() { + super(NAME, AcknowledgedResponse::new); + } + + @ExperimentalApi + public static class Request extends ClusterManagerNodeRequest { + private final String name; + + public Request(final String name) { + this.name = name; + } + + public Request(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request that = (Request) o; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = ValidateActions.addValidationError("name cannot be empty or null", validationException); + } + + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "delete_view_request", + args -> new Request((String) args[0]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.NAME_FIELD); + } + + public static Request fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + /** + * Transport Action for deleting a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected AcknowledgedResponse read(final StreamInput in) throws IOException { + return new AcknowledgedResponse(in); + } + + @Override + protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) + throws Exception { + viewService.deleteView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java index 584ca50deaee3..435c4e3f380f9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java @@ -8,7 +8,36 @@ package org.opensearch.action.admin.indices.view; +import java.io.IOException; +import java.util.Objects; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ValidateActions; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.ParseField; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.ToXContent.Params; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; /** Action to get a view */ @ExperimentalApi @@ -17,4 +46,146 @@ public class GetViewAction { public static final GetViewAction INSTANCE = new GetViewAction(); public static final String NAME = "views:data/read/get"; + @ExperimentalApi + public static class Request extends ClusterManagerNodeRequest { + private final String name; + + public Request(final String name) { + this.name = name; + } + + public Request(final StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request that = (Request) o; + return name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = ValidateActions.addValidationError("name cannot be empty or null", validationException); + } + + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + } + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "get_view_request", + args -> new Request((String) args[0]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), View.NAME_FIELD); + } + + public static Request fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + /** Response with a view */ + @ExperimentalApi + public static class Response extends ActionResponse implements ToXContentObject { + + private final View view; + + public Response(final View view) { + this.view = view; + } + + public Response(final StreamInput in) throws IOException { + super(in); + this.view = new View(in); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + this.view.writeTo(out); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field("view", view); + builder.endObject(); + return builder; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "view_response", + args -> new Response((View) args[0]) + ); + static { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), View.PARSER, new ParseField("view")); + } + + public static Response fromXContent(final XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + } + + /** + * Transport Action for getting a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected Response read(final StreamInput in) throws IOException { + return new Response(in); + } + + @Override + protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) + throws Exception { + viewService.getView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java index e793f7900ae68..464fbf9991174 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java @@ -8,7 +8,21 @@ package org.opensearch.action.admin.indices.view; +import java.io.IOException; + +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; /** Action to list a view names */ @ExperimentalApi @@ -17,4 +31,46 @@ public class ListViewNamesAction { public static final ListViewNamesAction INSTANCE = new ListViewNamesAction(); public static final String NAME = "views:data/read/list"; + /** + * Transport Action for getting a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected Response read(final StreamInput in) throws IOException { + return new Response(in); + } + + @Override + protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) + throws Exception { + viewService.getView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } + } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java index a559d6b88d589..28b34da0de6d4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java @@ -8,13 +8,81 @@ package org.opensearch.action.admin.indices.view; +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; +import org.opensearch.action.ValidateActions; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; +import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.block.ClusterBlockException; +import org.opensearch.cluster.block.ClusterBlockLevel; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.inject.Inject; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.util.CollectionUtils; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; /** Action to update a view */ @ExperimentalApi -public class UpdateViewAction { +public class UpdateViewAction extends ActionType { public static final UpdateViewAction INSTANCE = new UpdateViewAction(); public static final String NAME = "cluster:admin/views/update"; + public UpdateViewAction() { + super(NAME, GetViewAction.Response::new); + } + + /** + * Transport Action for updating a View + */ + public static class TransportAction extends TransportClusterManagerNodeAction { + + private final ViewService viewService; + + @Inject + public TransportAction( + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver, + final ViewService viewService + ) { + super(NAME, transportService, clusterService, threadPool, actionFilters, CreateViewAction.Request::new, indexNameExpressionResolver); + this.viewService = viewService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected GetViewAction.Response read(final StreamInput in) throws IOException { + return new GetViewAction.Response(in); + } + + @Override + protected void clusterManagerOperation(final CreateViewAction.Request request, final ClusterState state, final ActionListener listener) + throws Exception { + viewService.updateView(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(final CreateViewAction.Request request, final ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } + } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 48e5cd76b0c83..f72ade33978e4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -11,8 +11,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.admin.indices.view.DeleteViewAction.Request; +import org.opensearch.action.admin.indices.view.GetViewAction.Response; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; @@ -23,7 +26,10 @@ import org.opensearch.core.action.ActionListener; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.LongSupplier; +import java.util.function.Supplier; import java.util.stream.Collectors; /** Service to interact with views, create, retrieve, update, and delete */ @@ -33,14 +39,16 @@ public class ViewService { private final static Logger LOG = LogManager.getLogger(ViewService.class); private final ClusterService clusterService; private final NodeClient client; + private LongSupplier timeProvider; - public ViewService(final ClusterService clusterService, NodeClient client) { + public ViewService(final ClusterService clusterService, final NodeClient client, final LongSupplier timeProvider) { this.clusterService = clusterService; this.client = client; + this.timeProvider = Optional.ofNullable(timeProvider).orElse(System::currentTimeMillis); } - public void createView(final CreateViewAction.Request request, final ActionListener listener) { - final long currentTime = System.currentTimeMillis(); + public void createView(final CreateViewAction.Request request, final ActionListener listener) { + final long currentTime = timeProvider.getAsLong(); final List targets = request.getTargets() .stream() @@ -48,39 +56,66 @@ public void createView(final CreateViewAction.Request request, final ActionListe .collect(Collectors.toList()); final View view = new View(request.getName(), request.getDescription(), currentTime, currentTime, targets); - clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { + createOrUpdateView(Operation.CreateView, view, listener); + } + + public void updateView(final CreateViewAction.Request request, final ActionListener listener) { + final View originalView = getViewOrThrowException(request.getName()); + + final long currentTime = timeProvider.getAsLong(); + final List targets = request.getTargets() + .stream() + .map(target -> new View.Target(target.getIndexPattern())) + .collect(Collectors.toList()); + final View updatedView = new View(request.getName(), request.getDescription(), originalView.getCreatedAt(), currentTime, targets); + + createOrUpdateView(Operation.UpdateView, updatedView, listener); + } + + public void deleteView(final DeleteViewAction.Request request, final ActionListener listener) { + getViewOrThrowException(request.getName()); + + clusterService.submitStateUpdateTask("delete_view_task", new ClusterStateUpdateTask() { @Override public ClusterState execute(final ClusterState currentState) throws Exception { - return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) + return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).removeView(request.getName())) .build(); } @Override public void onFailure(final String source, final Exception e) { - LOG.error("Unable to create view, in source " + source, e); + LOG.error("Unable to delete view, from " + source, e); listener.onFailure(e); } @Override public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { - final View createdView = newState.getMetadata().views().get(request.getName()); - final CreateViewAction.Response response = new CreateViewAction.Response(createdView); - listener.onResponse(response); + listener.onResponse(new AcknowledgedResponse(true)); } }); } - public void searchView(final SearchViewAction.Request request, final ActionListener listener) { - final Optional optView = Optional.ofNullable(clusterService) + public void getView(final GetViewAction.Request request, final ActionListener listener) { + final View view = getViewOrThrowException(request.getName()); + + listener.onResponse(new GetViewAction.Response(view)); + } + + public void listViewNames(final ActionListener> listener) { + final List viewNames = Optional.ofNullable(clusterService) .map(ClusterService::state) .map(ClusterState::metadata) - .map(m -> m.views()) - .map(views -> views.get(request.getView())); + .map(Metadata::views) + .map(Map::keySet) + .orElseThrow() + .stream() + .collect(Collectors.toList()); - if (optView.isEmpty()) { - throw new ResourceNotFoundException("no such view [" + request.getView() + "]"); - } - final View view = optView.get(); + listener.onResponse(viewNames); + } + + public void searchView(final SearchViewAction.Request request, final ActionListener listener) { + final View view = getViewOrThrowException(request.getView()); final String[] indices = view.getTargets() .stream() @@ -91,4 +126,46 @@ public void searchView(final SearchViewAction.Request request, final ActionListe client.executeLocally(SearchAction.INSTANCE, request, listener); } + + private View getViewOrThrowException(final String viewName) { + return Optional.ofNullable(clusterService) + .map(ClusterService::state) + .map(ClusterState::metadata) + .map(m -> m.views()) + .map(views -> views.get(viewName)) + .orElseThrow(() -> new ResourceNotFoundException("no such view [" + viewName + "]")); + } + + private static enum Operation { + CreateView("create"), + UpdateView("update"); + + private final String name; + private Operation(final String name) { + this.name = name; + } + } + + private void createOrUpdateView(final Operation operation, final View view, final ActionListener listener) { + clusterService.submitStateUpdateTask(operation.name + "_view_task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(final ClusterState currentState) throws Exception { + return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) + .build(); + } + + @Override + public void onFailure(final String source, final Exception e) { + LOG.error("Unable to " + operation.name + " view, from " + source, e); + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + final View createdView = newState.getMetadata().views().get(view.getName()); + final GetViewAction.Response response = new GetViewAction.Response(createdView); + listener.onResponse(response); + } + }); + } } diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index c554316389953..c3e5fab4ae294 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -744,6 +744,16 @@ public FieldCapabilitiesRequestBuilder prepareFieldCaps(String... indices) { return new FieldCapabilitiesRequestBuilder(this, FieldCapabilitiesAction.INSTANCE, indices); } + @Override + public void searchView(SearchViewAction.Request request, ActionListener listener) { + execute(SearchViewAction.INSTANCE, request); + } + + @Override + public ActionFuture searchView(SearchViewAction.Request request) { + return execute(SearchViewAction.INSTANCE, request); + } + static class Admin implements AdminClient { private final ClusterAdmin clusterAdmin; @@ -2082,16 +2092,6 @@ public void createView(CreateViewAction.Request request, ActionListener createView(CreateViewAction.Request request) { return execute(CreateViewAction.INSTANCE, request); } - - @Override - public void searchView(SearchViewAction.Request request, ActionListener listener) { - execute(SearchViewAction.INSTANCE, request); - } - - @Override - public ActionFuture searchView(SearchViewAction.Request request) { - return execute(SearchViewAction.INSTANCE, request); - } } @Override diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index b8018936f0b89..1edbf18f72d3a 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.client.node.NodeClient; import org.opensearch.common.ValidationException; @@ -28,6 +29,7 @@ import java.util.List; import java.util.function.IntConsumer; +import static org.opensearch.rest.RestRequest.Method.DELETE; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; @@ -62,6 +64,31 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No } } + /** Handler for create view */ + public static class DeleteViewHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(DELETE).uniqueName(DeleteViewAction.NAME).build()); + } + + @Override + public String getName() { + return DeleteViewAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_ID); + + try (final XContentParser parser = request.contentParser()) { + final CreateViewAction.Request createViewAction = CreateViewAction.Request.fromXContent(parser); + return channel -> client.admin().indices().createView(createViewAction, new RestToXContentListener<>(channel)); + } + } + } + + /** Handler for search view */ public static class SearchViewHandler extends BaseRestHandler { @Override public List routes() { From f12efaa45dcd33b39463b5696d89212f9b004c5e Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 6 Feb 2024 03:13:38 +0000 Subject: [PATCH 22/55] wire up all actions Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 37 ++++++- .../admin/indices/view/CreateViewAction.java | 11 +- .../admin/indices/view/DeleteViewAction.java | 24 ++--- .../admin/indices/view/GetViewAction.java | 16 ++- .../indices/view/ListViewNamesAction.java | 102 +++++++++++------- .../admin/indices/view/UpdateViewAction.java | 57 +++++++--- .../admin/indices/view/ViewService.java | 14 +-- .../java/org/opensearch/client/Client.java | 7 ++ .../opensearch/client/IndicesAdminClient.java | 24 ++++- .../client/support/AbstractClient.java | 52 ++++++++- .../cluster/metadata/ViewMetadata.java | 1 + .../main/java/org/opensearch/node/Node.java | 2 +- .../action/admin/indices/RestViewAction.java | 81 +++++++++++++- 13 files changed, 336 insertions(+), 92 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index 2da1f5d187e2d..eb594204c1206 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -19,6 +19,7 @@ import java.util.List; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, numDataNodes = 2) @@ -38,17 +39,22 @@ private int createIndexWithDocs(final String indexName) throws Exception { return numOfDocs; } - private CreateViewAction.Response createView(final String name, final String indexPattern) throws Exception { + private GetViewAction.Response createView(final String name, final String indexPattern) throws Exception { final CreateViewAction.Request request = new CreateViewAction.Request( name, null, List.of(new CreateViewAction.Request.Target(indexPattern)) ); - final CreateViewAction.Response response = client().admin().indices().createView(request).actionGet(); + final GetViewAction.Response response = client().admin().indices().createView(request).actionGet(); performRemoteStoreTestAction(); return response; } + private void deleteView(final String name) { + client().admin().indices().deleteView(new DeleteViewAction.Request(name)).actionGet(); + performRemoteStoreTestAction(); + } + private SearchResponse searchView(final String viewName) throws Exception { final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); final SearchResponse response = client().admin().indices().searchView(request).actionGet(); @@ -77,4 +83,31 @@ public void testBasicOperations() throws Exception { createView("both-indices", "index-*"); assertHitCount(searchView("both-indices"), indexInView1DocCount + indexInView2DocCount); } + + public void testListViewNames() throws Exception { + logger.info("Create a single view"); + createView("view1", "*"); + final List viewNames1 = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + + assertThat(viewNames1, contains("view1")); + + logger.info("Create a second view"); + createView("view2", "*"); + final List viewNames2 = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + + assertThat(viewNames1, contains("view1", "view2")); + + logger.info("Delete a view"); + deleteView("view1"); + final List viewNamesAfterDelete = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + + assertThat(viewNames1, contains("view2")); + + logger.info("Delete a view"); + deleteView("view1"); + final List viewNamesAfterDelete = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + + assertThat(viewNames1, contains("view2")); + + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index 395e3307218c0..546698026a9de 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -22,17 +22,13 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.inject.Inject; -import org.opensearch.core.ParseField; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.xcontent.ConstructingObjectParser; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -231,8 +227,11 @@ protected GetViewAction.Response read(final StreamInput in) throws IOException { } @Override - protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) - throws Exception { + protected void clusterManagerOperation( + final Request request, + final ClusterState state, + final ActionListener listener + ) throws Exception { viewService.createView(request, listener); } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java index d2266b4ebd96b..9b0adebcfaee4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java @@ -8,12 +8,6 @@ package org.opensearch.action.admin.indices.view; -import static org.mockito.Mockito.description; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; - import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; import org.opensearch.action.ValidateActions; @@ -33,18 +27,19 @@ import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable.Reader; -import org.opensearch.core.common.util.CollectionUtils; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.Objects; + /** Action to delete a view */ @SuppressWarnings("deprecation") @ExperimentalApi -public class DeleteViewAction extends ActionType { - +public class DeleteViewAction extends ActionType { + public static final DeleteViewAction INSTANCE = new DeleteViewAction(); public static final String NAME = "cluster:admin/views/delete"; @@ -52,6 +47,7 @@ public DeleteViewAction() { super(NAME, AcknowledgedResponse::new); } + /** Request for delete view */ @ExperimentalApi public static class Request extends ClusterManagerNodeRequest { private final String name; @@ -97,6 +93,7 @@ public void writeTo(final StreamOutput out) throws IOException { super.writeTo(out); out.writeString(name); } + @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "delete_view_request", @@ -143,8 +140,11 @@ protected AcknowledgedResponse read(final StreamInput in) throws IOException { } @Override - protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) - throws Exception { + protected void clusterManagerOperation( + final Request request, + final ClusterState state, + final ActionListener listener + ) throws Exception { viewService.deleteView(request, listener); } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java index 435c4e3f380f9..73a9ff1006aba 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java @@ -8,15 +8,12 @@ package org.opensearch.action.admin.indices.view; -import java.io.IOException; -import java.util.Objects; - import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; import org.opensearch.action.ValidateActions; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; @@ -39,13 +36,21 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.Objects; + /** Action to get a view */ @ExperimentalApi -public class GetViewAction { +public class GetViewAction extends ActionType { public static final GetViewAction INSTANCE = new GetViewAction(); public static final String NAME = "views:data/read/get"; + public GetViewAction() { + super(NAME, GetViewAction.Response::new); + } + + /** Request for get view */ @ExperimentalApi public static class Request extends ClusterManagerNodeRequest { private final String name; @@ -91,6 +96,7 @@ public void writeTo(final StreamOutput out) throws IOException { super.writeTo(out); out.writeString(name); } + @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "get_view_request", diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java index 464fbf9991174..b2c9ac1fe857c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java @@ -8,69 +8,99 @@ package org.opensearch.action.admin.indices.view; -import java.io.IOException; - +import org.opensearch.action.ActionRequest; +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.action.ActionType; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.cluster.ClusterState; -import org.opensearch.cluster.block.ClusterBlockException; -import org.opensearch.cluster.block.ClusterBlockLevel; -import org.opensearch.cluster.metadata.IndexNameExpressionResolver; -import org.opensearch.cluster.service.ClusterService; +import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.threadpool.ThreadPool; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.List; + /** Action to list a view names */ @ExperimentalApi -public class ListViewNamesAction { +public class ListViewNamesAction extends ActionType { public static final ListViewNamesAction INSTANCE = new ListViewNamesAction(); public static final String NAME = "views:data/read/list"; - /** - * Transport Action for getting a View - */ - public static class TransportAction extends TransportClusterManagerNodeAction { + public ListViewNamesAction() { + super(NAME, ListViewNamesAction.Response::new); + } - private final ViewService viewService; + /** Request for list view names */ + @ExperimentalApi + public static class Request extends ActionRequest { + public Request() {} - @Inject - public TransportAction( - final TransportService transportService, - final ClusterService clusterService, - final ThreadPool threadPool, - final ActionFilters actionFilters, - final IndexNameExpressionResolver indexNameExpressionResolver, - final ViewService viewService - ) { - super(NAME, transportService, clusterService, threadPool, actionFilters, Request::new, indexNameExpressionResolver); - this.viewService = viewService; - } + public Request(final StreamInput in) {} @Override - protected String executor() { - return ThreadPool.Names.MANAGEMENT; + public ActionRequestValidationException validate() { + return null; + } + } + + /** Response for list view names */ + @ExperimentalApi + public static class Response extends ActionResponse implements ToXContentObject { + + private final List views; + + public Response(final List views) { + this.views = views; + } + + public Response(final StreamInput in) throws IOException { + views = in.readStringList(); + } + + public List getViewNames() { + return views; } @Override - protected Response read(final StreamInput in) throws IOException { - return new Response(in); + public void writeTo(final StreamOutput out) throws IOException { + out.writeStringCollection(views); } @Override - protected void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) - throws Exception { - viewService.getView(request, listener); + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field("views", views); + builder.endObject(); + return builder; + } + } + + /** + * Transport Action for getting a View + */ + public static class TransportAction extends HandledTransportAction { + + private final ViewService viewService; + + @Inject + public TransportAction(final TransportService transportService, final ActionFilters actionFilters, final ViewService viewService) { + super(NAME, transportService, actionFilters, Request::new); + this.viewService = viewService; } @Override - protected ClusterBlockException checkBlock(final Request request, final ClusterState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + protected void doExecute(Task task, Request request, ActionListener listener) { + viewService.listViewNames(listener); } + } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java index 28b34da0de6d4..9182684c73a0b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/UpdateViewAction.java @@ -8,30 +8,27 @@ package org.opensearch.action.admin.indices.view; -import java.io.IOException; -import java.util.List; -import java.util.Objects; - -import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; -import org.opensearch.action.ValidateActions; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; -import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.util.CollectionUtils; +import org.opensearch.core.xcontent.ConstructingObjectParser; +import org.opensearch.core.xcontent.XContentParser; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; +import java.io.IOException; +import java.util.List; + /** Action to update a view */ @ExperimentalApi public class UpdateViewAction extends ActionType { @@ -43,9 +40,34 @@ public UpdateViewAction() { super(NAME, GetViewAction.Response::new); } + /** Request for update view */ + @ExperimentalApi + public static class Request { + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "create_view_request", + false, + (args, viewName) -> new CreateViewAction.Request(viewName, (String) args[0], (List) args[1]) + ); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), View.DESCRIPTION_FIELD); + PARSER.declareObjectArray( + ConstructingObjectParser.constructorArg(), + (p, c) -> CreateViewAction.Request.Target.fromXContent(p), + View.TARGETS_FIELD + ); + } + + public static CreateViewAction.Request fromXContent(final XContentParser parser, final String viewName) throws IOException { + return PARSER.parse(parser, viewName); + } + } + /** * Transport Action for updating a View */ + @ExperimentalApi public static class TransportAction extends TransportClusterManagerNodeAction { private final ViewService viewService; @@ -59,7 +81,15 @@ public TransportAction( final IndexNameExpressionResolver indexNameExpressionResolver, final ViewService viewService ) { - super(NAME, transportService, clusterService, threadPool, actionFilters, CreateViewAction.Request::new, indexNameExpressionResolver); + super( + NAME, + transportService, + clusterService, + threadPool, + actionFilters, + CreateViewAction.Request::new, + indexNameExpressionResolver + ); this.viewService = viewService; } @@ -74,8 +104,11 @@ protected GetViewAction.Response read(final StreamInput in) throws IOException { } @Override - protected void clusterManagerOperation(final CreateViewAction.Request request, final ClusterState state, final ActionListener listener) - throws Exception { + protected void clusterManagerOperation( + final CreateViewAction.Request request, + final ClusterState state, + final ActionListener listener + ) throws Exception { viewService.updateView(request, listener); } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index f72ade33978e4..366292b3b0fc4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -11,8 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.admin.indices.view.DeleteViewAction.Request; -import org.opensearch.action.admin.indices.view.GetViewAction.Response; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -29,7 +27,6 @@ import java.util.Map; import java.util.Optional; import java.util.function.LongSupplier; -import java.util.function.Supplier; import java.util.stream.Collectors; /** Service to interact with views, create, retrieve, update, and delete */ @@ -72,14 +69,16 @@ public void updateView(final CreateViewAction.Request request, final ActionListe createOrUpdateView(Operation.UpdateView, updatedView, listener); } + @SuppressWarnings("deprecation") public void deleteView(final DeleteViewAction.Request request, final ActionListener listener) { getViewOrThrowException(request.getName()); clusterService.submitStateUpdateTask("delete_view_task", new ClusterStateUpdateTask() { @Override public ClusterState execute(final ClusterState currentState) throws Exception { - return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).removeView(request.getName())) - .build(); + return new ClusterState.Builder(clusterService.state()).metadata( + Metadata.builder(currentState.metadata()).removeView(request.getName()) + ).build(); } @Override @@ -101,7 +100,7 @@ public void getView(final GetViewAction.Request request, final ActionListener> listener) { + public void listViewNames(final ActionListener listener) { final List viewNames = Optional.ofNullable(clusterService) .map(ClusterService::state) .map(ClusterState::metadata) @@ -111,7 +110,7 @@ public void listViewNames(final ActionListener> listener) { .stream() .collect(Collectors.toList()); - listener.onResponse(viewNames); + listener.onResponse(new ListViewNamesAction.Response(viewNames)); } public void searchView(final SearchViewAction.Request request, final ActionListener listener) { @@ -141,6 +140,7 @@ private static enum Operation { UpdateView("update"); private final String name; + private Operation(final String name) { this.name = name; } diff --git a/server/src/main/java/org/opensearch/client/Client.java b/server/src/main/java/org/opensearch/client/Client.java index d7679d0ae1e3a..322b435bdf35c 100644 --- a/server/src/main/java/org/opensearch/client/Client.java +++ b/server/src/main/java/org/opensearch/client/Client.java @@ -34,6 +34,7 @@ import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.PitSegmentsRequest; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkRequestBuilder; @@ -473,6 +474,12 @@ public interface Client extends OpenSearchClient, Releasable { /** Search a view */ ActionFuture searchView(final SearchViewAction.Request request); + /** List all view names */ + void listViewNames(final ListViewNamesAction.Request request, ActionListener listener); + + /** List all view names */ + ActionFuture listViewNames(final ListViewNamesAction.Request request); + /** * Returns this clients settings */ diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index ec7f5b1ae5131..c1a75b02aaf95 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -126,6 +126,8 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.Nullable; @@ -841,8 +843,26 @@ public interface IndicesAdminClient extends OpenSearchClient { ActionFuture resolveIndex(ResolveIndexAction.Request request); /** Create a view */ - void createView(CreateViewAction.Request request, ActionListener listener); + void createView(CreateViewAction.Request request, ActionListener listener); /** Create a view */ - ActionFuture createView(CreateViewAction.Request request); + ActionFuture createView(CreateViewAction.Request request); + + /** Gets a view */ + void getView(GetViewAction.Request request, ActionListener listener); + + /** Gets a view */ + ActionFuture getView(GetViewAction.Request request); + + /** Create a view */ + void deleteView(DeleteViewAction.Request request, ActionListener listener); + + /** Create a view */ + ActionFuture deleteView(DeleteViewAction.Request request); + + /** Create a view */ + void updateView(CreateViewAction.Request request, ActionListener listener); + + /** Create a view */ + ActionFuture updateView(CreateViewAction.Request request); } diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index c3e5fab4ae294..6c6049f04231b 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -313,7 +313,11 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.admin.indices.view.UpdateViewAction; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkRequestBuilder; @@ -745,15 +749,25 @@ public FieldCapabilitiesRequestBuilder prepareFieldCaps(String... indices) { } @Override - public void searchView(SearchViewAction.Request request, ActionListener listener) { + public void searchView(final SearchViewAction.Request request, final ActionListener listener) { execute(SearchViewAction.INSTANCE, request); } @Override - public ActionFuture searchView(SearchViewAction.Request request) { + public ActionFuture searchView(final SearchViewAction.Request request) { return execute(SearchViewAction.INSTANCE, request); } + @Override + public void listViewNames(final ListViewNamesAction.Request request, ActionListener listener) { + execute(ListViewNamesAction.INSTANCE, request, listener); + } + + @Override + public ActionFuture listViewNames(final ListViewNamesAction.Request request) { + return execute(ListViewNamesAction.INSTANCE, request); + } + static class Admin implements AdminClient { private final ClusterAdmin clusterAdmin; @@ -2084,14 +2098,44 @@ public ActionFuture resolveIndex(ResolveIndexAction } @Override - public void createView(CreateViewAction.Request request, ActionListener listener) { + public void createView(CreateViewAction.Request request, ActionListener listener) { execute(CreateViewAction.INSTANCE, request); } @Override - public ActionFuture createView(CreateViewAction.Request request) { + public ActionFuture createView(CreateViewAction.Request request) { return execute(CreateViewAction.INSTANCE, request); } + + /** Gets a view */ + public void getView(GetViewAction.Request request, ActionListener listener) { + execute(GetViewAction.INSTANCE, request, listener); + } + + /** Gets a view */ + public ActionFuture getView(GetViewAction.Request request) { + return execute(GetViewAction.INSTANCE, request); + } + + /** Create a view */ + public void deleteView(DeleteViewAction.Request request, ActionListener listener) { + execute(DeleteViewAction.INSTANCE, request, listener); + } + + /** Create a view */ + public ActionFuture deleteView(DeleteViewAction.Request request) { + return execute(DeleteViewAction.INSTANCE, request); + } + + /** Create a view */ + public void updateView(CreateViewAction.Request request, ActionListener listener) { + execute(UpdateViewAction.INSTANCE, request, listener); + } + + /** Create a view */ + public ActionFuture updateView(CreateViewAction.Request request) { + return execute(UpdateViewAction.INSTANCE, request); + } } @Override diff --git a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java index dac11d55fddaa..a89068078be58 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/ViewMetadata.java @@ -139,6 +139,7 @@ public String toString() { /** * Builder of view metadata. */ + @ExperimentalApi public static class Builder { private final Map views = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 94a20df67c6cd..64f38d8c5f57b 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -861,7 +861,7 @@ protected Node( metadataCreateIndexService ); - final ViewService viewService = new ViewService(clusterService, client); + final ViewService viewService = new ViewService(clusterService, client, null); Collection pluginComponents = pluginsService.filterPlugins(Plugin.class) .stream() diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 1edbf18f72d3a..de9d5d884ccad 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -12,7 +12,10 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.view.CreateViewAction; import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.admin.indices.view.UpdateViewAction; import org.opensearch.client.node.NodeClient; import org.opensearch.common.ValidationException; import org.opensearch.common.annotation.ExperimentalApi; @@ -32,6 +35,7 @@ import static org.opensearch.rest.RestRequest.Method.DELETE; import static org.opensearch.rest.RestRequest.Method.GET; import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.rest.RestRequest.Method.PUT; /** All rest handlers for view actions */ @ExperimentalApi @@ -64,17 +68,43 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No } } - /** Handler for create view */ + /** Handler for delete view */ public static class DeleteViewHandler extends BaseRestHandler { @Override public List routes() { - return List.of(new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(DELETE).uniqueName(DeleteViewAction.NAME).build()); + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(DELETE).uniqueName(DeleteViewAction.NAME).build() + ); } @Override public String getName() { - return DeleteViewAction.NAME; + return CreateViewAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_ID); + + final DeleteViewAction.Request deleteRequest = new DeleteViewAction.Request(viewId); + return channel -> client.admin().indices().deleteView(deleteRequest, new RestToXContentListener<>(channel)); + } + } + + /** Handler for update view */ + public static class UpdateViewHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(PUT).uniqueName(UpdateViewAction.NAME).build() + ); + } + + @Override + public String getName() { + return UpdateViewAction.NAME; } @Override @@ -82,12 +112,53 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No final String viewId = request.param(VIEW_ID); try (final XContentParser parser = request.contentParser()) { - final CreateViewAction.Request createViewAction = CreateViewAction.Request.fromXContent(parser); - return channel -> client.admin().indices().createView(createViewAction, new RestToXContentListener<>(channel)); + final CreateViewAction.Request updateRequest = UpdateViewAction.Request.fromXContent(parser, viewId); + return channel -> client.admin().indices().updateView(updateRequest, new RestToXContentListener<>(channel)); } } } + /** Handler for get view */ + public static class GetViewHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(GET).uniqueName(GetViewAction.NAME).build()); + } + + @Override + public String getName() { + return GetViewAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final String viewId = request.param(VIEW_ID); + + final GetViewAction.Request getRequest = new GetViewAction.Request(viewId); + return channel -> client.admin().indices().getView(getRequest, new RestToXContentListener<>(channel)); + } + } + + /** Handler for get view */ + public static class ListViewNamesHandler extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new NamedRoute.Builder().path("/views/").method(GET).uniqueName(ListViewNamesAction.NAME).build()); + } + + @Override + public String getName() { + return ListViewNamesAction.NAME; + } + + @Override + protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + return channel -> client.listViewNames(new ListViewNamesAction.Request(), new RestToXContentListener<>(channel)); + } + } + /** Handler for search view */ public static class SearchViewHandler extends BaseRestHandler { @Override From 135acba6203452f6048b87447db0beb24e94e41d Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 6 Feb 2024 03:26:37 +0000 Subject: [PATCH 23/55] Functional integration test Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 14 +++++++------- .../java/org/opensearch/action/ActionModule.java | 12 ++++++++++++ .../rest/action/admin/indices/RestViewAction.java | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index eb594204c1206..bb51931d96a7a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -57,7 +57,7 @@ private void deleteView(final String name) { private SearchResponse searchView(final String viewName) throws Exception { final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); - final SearchResponse response = client().admin().indices().searchView(request).actionGet(); + final SearchResponse response = client().searchView(request).actionGet(); return response; } @@ -95,19 +95,19 @@ public void testListViewNames() throws Exception { createView("view2", "*"); final List viewNames2 = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); - assertThat(viewNames1, contains("view1", "view2")); + assertThat(viewNames2, contains("view1", "view2")); logger.info("Delete a view"); deleteView("view1"); final List viewNamesAfterDelete = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); - assertThat(viewNames1, contains("view2")); + assertThat(viewNamesAfterDelete, contains("view2")); - logger.info("Delete a view"); - deleteView("view1"); - final List viewNamesAfterDelete = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + logger.info("Update a view"); + client().admin().indices().updateView(new CreateViewAction.Request("view2", "newDescription", List.of())); + final List viewNamesAfterUpdate = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); - assertThat(viewNames1, contains("view2")); + assertThat(viewNamesAfterUpdate, contains("view2")); } } diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java index b18498de9581e..b19bf9590f43b 100644 --- a/server/src/main/java/org/opensearch/action/ActionModule.java +++ b/server/src/main/java/org/opensearch/action/ActionModule.java @@ -225,7 +225,11 @@ import org.opensearch.action.admin.indices.validate.query.TransportValidateQueryAction; import org.opensearch.action.admin.indices.validate.query.ValidateQueryAction; import org.opensearch.action.admin.indices.view.CreateViewAction; +import org.opensearch.action.admin.indices.view.DeleteViewAction; +import org.opensearch.action.admin.indices.view.GetViewAction; +import org.opensearch.action.admin.indices.view.ListViewNamesAction; import org.opensearch.action.admin.indices.view.SearchViewAction; +import org.opensearch.action.admin.indices.view.UpdateViewAction; import org.opensearch.action.bulk.BulkAction; import org.opensearch.action.bulk.TransportBulkAction; import org.opensearch.action.bulk.TransportShardBulkAction; @@ -726,6 +730,10 @@ public void reg // Views: actions.register(CreateViewAction.INSTANCE, CreateViewAction.TransportAction.class); + actions.register(DeleteViewAction.INSTANCE, DeleteViewAction.TransportAction.class); + actions.register(GetViewAction.INSTANCE, GetViewAction.TransportAction.class); + actions.register(UpdateViewAction.INSTANCE, UpdateViewAction.TransportAction.class); + actions.register(ListViewNamesAction.INSTANCE, ListViewNamesAction.TransportAction.class); actions.register(SearchViewAction.INSTANCE, SearchViewAction.TransportAction.class); // Persistent tasks: @@ -924,7 +932,11 @@ public void initRestHandlers(Supplier nodesInCluster) { // View API registerHandler.accept(new RestViewAction.CreateViewHandler()); + registerHandler.accept(new RestViewAction.DeleteViewHandler()); + registerHandler.accept(new RestViewAction.GetViewHandler()); + registerHandler.accept(new RestViewAction.UpdateViewHandler()); registerHandler.accept(new RestViewAction.SearchViewHandler()); + registerHandler.accept(new RestViewAction.ListViewNamesHandler()); // CAT API registerHandler.accept(new RestAllocationAction()); diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index de9d5d884ccad..9f9fbe13779c8 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -80,7 +80,7 @@ public List routes() { @Override public String getName() { - return CreateViewAction.NAME; + return DeleteViewAction.NAME; } @Override From ac456dae5c3e168fdbc7764b8ff3655ec02640e6 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 6 Feb 2024 17:49:07 +0000 Subject: [PATCH 24/55] refactor how view names are listed in tests Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index bb51931d96a7a..b46047e28ed5b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -55,6 +55,10 @@ private void deleteView(final String name) { performRemoteStoreTestAction(); } + private List listViews() { + return client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + } + private SearchResponse searchView(final String viewName) throws Exception { final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); final SearchResponse response = client().searchView(request).actionGet(); @@ -87,27 +91,26 @@ public void testBasicOperations() throws Exception { public void testListViewNames() throws Exception { logger.info("Create a single view"); createView("view1", "*"); - final List viewNames1 = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + final List viewNames1 = listViews(); assertThat(viewNames1, contains("view1")); logger.info("Create a second view"); createView("view2", "*"); - final List viewNames2 = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + final List viewNames2 = listViews(); assertThat(viewNames2, contains("view1", "view2")); logger.info("Delete a view"); deleteView("view1"); - final List viewNamesAfterDelete = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + final List viewNamesAfterDelete = listViews(); assertThat(viewNamesAfterDelete, contains("view2")); logger.info("Update a view"); client().admin().indices().updateView(new CreateViewAction.Request("view2", "newDescription", List.of())); - final List viewNamesAfterUpdate = client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + final List viewNamesAfterUpdate = listViews(); assertThat(viewNamesAfterUpdate, contains("view2")); - } } From 327d158e0d862d4d49045526baf0a42e1ac1a279 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 6 Feb 2024 22:19:21 +0000 Subject: [PATCH 25/55] More tests Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 11 +- .../admin/indices/view/ViewService.java | 4 +- .../admin/indices/view/ViewServiceTest.java | 178 ++++++++++++++++++ 3 files changed, 186 insertions(+), 7 deletions(-) create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index b46047e28ed5b..6596d3aff2634 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -8,6 +8,7 @@ package org.opensearch.action.admin.indices.view; +import org.hamcrest.MatcherAssert; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.index.IndexNotFoundException; @@ -77,7 +78,7 @@ public void testBasicOperations() throws Exception { logger.info("Testing view with no matches"); createView("no-matches", "this-pattern-will-match-nothing"); final IndexNotFoundException ex = assertThrows(IndexNotFoundException.class, () -> searchView("no-matches")); - assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]")); + MatcherAssert.assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]")); logger.info("Testing view with exact index match"); createView("only-index-1", "index-1"); @@ -93,24 +94,24 @@ public void testListViewNames() throws Exception { createView("view1", "*"); final List viewNames1 = listViews(); - assertThat(viewNames1, contains("view1")); + MatcherAssert.assertThat(viewNames1, contains("view1")); logger.info("Create a second view"); createView("view2", "*"); final List viewNames2 = listViews(); - assertThat(viewNames2, contains("view1", "view2")); + MatcherAssert.assertThat(viewNames2, contains("view1", "view2")); logger.info("Delete a view"); deleteView("view1"); final List viewNamesAfterDelete = listViews(); - assertThat(viewNamesAfterDelete, contains("view2")); + MatcherAssert.assertThat(viewNamesAfterDelete, contains("view2")); logger.info("Update a view"); client().admin().indices().updateView(new CreateViewAction.Request("view2", "newDescription", List.of())); final List viewNamesAfterUpdate = listViews(); - assertThat(viewNamesAfterUpdate, contains("view2")); + MatcherAssert.assertThat(viewNamesAfterUpdate, contains("view2")); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 366292b3b0fc4..484d7a158a7fa 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -126,7 +126,7 @@ public void searchView(final SearchViewAction.Request request, final ActionListe client.executeLocally(SearchAction.INSTANCE, request, listener); } - private View getViewOrThrowException(final String viewName) { + View getViewOrThrowException(final String viewName) { return Optional.ofNullable(clusterService) .map(ClusterService::state) .map(ClusterState::metadata) @@ -135,7 +135,7 @@ private View getViewOrThrowException(final String viewName) { .orElseThrow(() -> new ResourceNotFoundException("no such view [" + viewName + "]")); } - private static enum Operation { + private enum Operation { CreateView("create"), UpdateView("update"); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java new file mode 100644 index 0000000000000..f17f2f7166de9 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.action.search.SearchAction; +import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.View; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.core.action.ActionListener; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + + +public class ViewServiceTest { + + private final View.Target typicalTarget = new View.Target("my-index-*"); + private final View typicalView = new View("actualView", "actual description", -1L, -1L, List.of(typicalTarget)); + + private ClusterService clusterService; + private NodeClient nodeClient; + private AtomicLong currentTime = new AtomicLong(0); + private LongSupplier timeProvider = currentTime::longValue; + private ViewService viewService; + + @Before + public void before() { + clusterService = mock(ClusterService.class); + nodeClient = mock(NodeClient.class); + timeProvider = mock(LongSupplier.class); + doAnswer(invocation -> currentTime.get()).when(timeProvider).getAsLong(); + viewService = spy(new ViewService(clusterService, nodeClient, timeProvider)); + } + + @After + public void after() { + verifyNoMoreInteractions(timeProvider, clusterService, nodeClient); + } + + @Test + public void createView() { + final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.createView(request, listener); + + verify(clusterService).submitStateUpdateTask(eq("create_view_task"), any()); + verify(timeProvider).getAsLong(); + } + + @Test + public void updateView() { + final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.updateView(request, listener); + + verify(clusterService).submitStateUpdateTask(eq("update_view_task"), any()); + verify(timeProvider).getAsLong(); + } + + @Test + public void updateView_doesNotExist() { + final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); + final var listener = mock(ActionListener.class); + doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); + + final Exception ex = assertThrows(ResourceNotFoundException.class, () -> viewService.updateView(request, listener)); + assertThat(ex.getMessage(), equalTo("abc")); + } + + @Test + public void deleteView() { + final var request = new DeleteViewAction.Request("viewName"); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.deleteView(request, listener); + + verify(clusterService).submitStateUpdateTask(eq("delete_view_task"), any()); + } + + @Test + public void deleteView_doesNotExist() { + final var request = new DeleteViewAction.Request("viewName"); + final var listener = mock(ActionListener.class); + doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); + + final ResourceNotFoundException ex = assertThrows(ResourceNotFoundException.class, () -> viewService.deleteView(request, listener)); + + assertThat(ex.getMessage(), equalTo("abc")); + } + + @Test + public void getView() { + final var request = new GetViewAction.Request("viewName"); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.getView(request, listener); + + verify(listener).onResponse(any()); + } + + @Test + public void getView_doesNotExist() { + final var request = new GetViewAction.Request("viewName"); + final var listener = mock(ActionListener.class); + doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); + + final ResourceNotFoundException ex = assertThrows(ResourceNotFoundException.class, () -> viewService.getView(request, listener)); + + assertThat(ex.getMessage(), equalTo("abc")); + } + + @Test + public void listViewNames() { + final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")) + .metadata(new Metadata.Builder().views(Map.of(typicalView.getName(), typicalView)).build()) + .build(); + final var listener = mock(ActionListener.class); + when(clusterService.state()).thenReturn(clusterState); + + viewService.listViewNames(listener); + + verify(clusterService).state(); + verify(listener).onResponse(any()); + } + + @Test + public void listViewNames_noViews() { + final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")).build(); + final var listener = mock(ActionListener.class); + when(clusterService.state()).thenReturn(clusterState); + + viewService.listViewNames(listener); + + verify(clusterService).state(); + verify(listener).onResponse(any()); + } + + @Test + public void searchView() { + final var request = spy(new SearchViewAction.Request("view")); + final var listener = mock(ActionListener.class); + setGetViewOrThrowExceptionToReturnTypicalView(); + + viewService.searchView(request, listener); + + verify(nodeClient).executeLocally(eq(SearchAction.INSTANCE), any(), any(ActionListener.class)); + verify(request).indices(typicalTarget.getIndexPattern()); + } + + private void setGetViewOrThrowExceptionToReturnTypicalView() { + doAnswer(invocation -> typicalView).when(viewService).getViewOrThrowException(anyString()); + } +} From b95056ae2e67d03358dbba8a1ca0b035c2dee300 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 6 Feb 2024 22:38:39 +0000 Subject: [PATCH 26/55] Following up on PR feedback Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 2 +- .../action/admin/indices/RestViewAction.java | 137 ++++++------------ .../admin/indices/view/ViewServiceTest.java | 38 +++-- 3 files changed, 68 insertions(+), 109 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index 6596d3aff2634..9c7b550a32ec5 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -8,7 +8,6 @@ package org.opensearch.action.admin.indices.view; -import org.hamcrest.MatcherAssert; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.index.IndexNotFoundException; @@ -16,6 +15,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; +import org.hamcrest.MatcherAssert; import java.util.List; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 9f9fbe13779c8..013930113ec7d 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -8,8 +8,6 @@ package org.opensearch.rest.action.admin.indices; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.view.CreateViewAction; import org.opensearch.action.admin.indices.view.DeleteViewAction; import org.opensearch.action.admin.indices.view.GetViewAction; @@ -41,12 +39,11 @@ @ExperimentalApi public class RestViewAction { - private final static Logger LOG = LogManager.getLogger(RestViewAction.class); - - public static final String VIEW_ID = "view_id"; - public static final String VIEW_ID_PARAMETER = "{" + VIEW_ID + "}"; + public static final String VIEW_NAME = "view_name"; + public static final String VIEW_NAME_PARAMETER = "{" + VIEW_NAME + "}"; /** Handler for create view */ + @ExperimentalApi public static class CreateViewHandler extends BaseRestHandler { @Override @@ -63,18 +60,25 @@ public String getName() { protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { try (final XContentParser parser = request.contentParser()) { final CreateViewAction.Request createViewAction = CreateViewAction.Request.fromXContent(parser); + + final ValidationException validationResult = createViewAction.validate(); + if (validationResult != null) { + throw validationResult; + } + return channel -> client.admin().indices().createView(createViewAction, new RestToXContentListener<>(channel)); } } } /** Handler for delete view */ + @ExperimentalApi public static class DeleteViewHandler extends BaseRestHandler { @Override public List routes() { return List.of( - new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(DELETE).uniqueName(DeleteViewAction.NAME).build() + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER).method(DELETE).uniqueName(DeleteViewAction.NAME).build() ); } @@ -85,20 +89,27 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final String viewId = request.param(VIEW_ID); + final String viewId = request.param(VIEW_NAME); final DeleteViewAction.Request deleteRequest = new DeleteViewAction.Request(viewId); + + final ValidationException validationResult = deleteRequest.validate(); + if (validationResult != null) { + throw validationResult; + } + return channel -> client.admin().indices().deleteView(deleteRequest, new RestToXContentListener<>(channel)); } } /** Handler for update view */ + @ExperimentalApi public static class UpdateViewHandler extends BaseRestHandler { @Override public List routes() { return List.of( - new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(PUT).uniqueName(UpdateViewAction.NAME).build() + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER).method(PUT).uniqueName(UpdateViewAction.NAME).build() ); } @@ -109,21 +120,30 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final String viewId = request.param(VIEW_ID); + final String viewId = request.param(VIEW_NAME); try (final XContentParser parser = request.contentParser()) { final CreateViewAction.Request updateRequest = UpdateViewAction.Request.fromXContent(parser, viewId); + + final ValidationException validationResult = updateRequest.validate(); + if (validationResult != null) { + throw validationResult; + } + return channel -> client.admin().indices().updateView(updateRequest, new RestToXContentListener<>(channel)); } } } /** Handler for get view */ + @ExperimentalApi public static class GetViewHandler extends BaseRestHandler { @Override public List routes() { - return List.of(new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER).method(GET).uniqueName(GetViewAction.NAME).build()); + return List.of( + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER).method(GET).uniqueName(GetViewAction.NAME).build() + ); } @Override @@ -133,14 +153,21 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final String viewId = request.param(VIEW_ID); + final String viewId = request.param(VIEW_NAME); final GetViewAction.Request getRequest = new GetViewAction.Request(viewId); + + final ValidationException validationResult = getRequest.validate(); + if (validationResult != null) { + throw validationResult; + } + return channel -> client.admin().indices().getView(getRequest, new RestToXContentListener<>(channel)); } } /** Handler for get view */ + @ExperimentalApi public static class ListViewNamesHandler extends BaseRestHandler { @Override @@ -160,15 +187,16 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No } /** Handler for search view */ + @ExperimentalApi public static class SearchViewHandler extends BaseRestHandler { @Override public List routes() { return List.of( - new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search") + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER + "/_search") .method(GET) .uniqueName(SearchViewAction.NAME) .build(), - new NamedRoute.Builder().path("/views/" + VIEW_ID_PARAMETER + "/_search") + new NamedRoute.Builder().path("/views/" + VIEW_NAME_PARAMETER + "/_search") .method(POST) .uniqueName(SearchViewAction.NAME) .build() @@ -182,7 +210,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final String viewId = request.param(VIEW_ID); + final String viewId = request.param(VIEW_NAME); final SearchViewAction.Request viewSearchRequest = new SearchViewAction.Request(viewId); final IntConsumer setSize = size -> viewSearchRequest.source().size(size); @@ -208,83 +236,4 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC }; } } - - // TODO: Replace and reorganize this layout - - // public List routes() { - - // return List.of( - // new NamedRoute.Builder().path("/views").method(GET).uniqueName("cluster:views:list").build(), - // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(GET).uniqueName("cluster:views:get").build(), - // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(DELETE).uniqueName("cluster:views:delete").build(), - // new NamedRoute.Builder().path("/views/" + viewIdParameter).method(PUT).uniqueName("cluster:views:update").build() - // ); - // } - - // public RestResponse handlePost(final RestRequest r, final RestChannel channel) throws IOException { - // final View inputView; - // try (final XContentParser parser = r.contentParser()) { - // inputView = View.fromXContent(parser); - // } - - // final long currentTime = System.currentTimeMillis(); - // final View view = new View(inputView.name, inputView.description, currentTime, currentTime, inputView.targets); - - // clusterService.submitStateUpdateTask("create_view_task", new ClusterStateUpdateTask() { - // @Override - // public ClusterState execute(final ClusterState currentState) throws Exception { - // return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) - // .build(); - // } - - // @Override - // public void onFailure(final String source, final Exception e) { - // LOG.error("Unable to create view, due to {}", source, e); - // channel.sendResponse( - // new BytesRestResponse(RestStatus.INTERNAL_SERVER_ERROR, "Unknown error occurred, see the log for details.") - // ); - // } - - // @Override - // public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { - // try { - // channel.sendResponse( - // new BytesRestResponse(RestStatus.CREATED, channel.newBuilder().startObject().field(view.name, view).endObject()) - // ); - // } catch (final IOException e) { - // // TODO? - // LOG.error(e); - // } - // } - // }); - // // TODO: Handle CREATED vs UPDATED - // return null; - // } - - // public RestResponse handleSingleGet(final RestRequest r, final XContentBuilder builder) throws IOException { - // final String viewId = r.param(VIEW_ID); - - // if (Strings.isNullOrEmpty(viewId)) { - // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); - // } - - // final Optional view = Optional.ofNullable(clusterService.state().getMetadata()) - // .map(m -> m.views()) - // .map(views -> views.get(viewId)); - - // if (view.isEmpty()) { - // return new BytesRestResponse(RestStatus.NOT_FOUND, ""); - // } - - // return new BytesRestResponse(RestStatus.OK, builder.startObject().value(view).endObject()); - // } - - // public RestResponse handleSinglePut(final RestRequest r) { - // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); - // } - - // public RestResponse handleSingleDelete(final RestRequest r) { - // return new BytesRestResponse(RestStatus.NOT_IMPLEMENTED, ""); - // } - } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java index f17f2f7166de9..5053891a96915 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -8,9 +8,6 @@ package org.opensearch.action.admin.indices.view; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import org.opensearch.ResourceNotFoundException; import org.opensearch.action.search.SearchAction; import org.opensearch.client.node.NodeClient; @@ -20,6 +17,10 @@ import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; +import org.hamcrest.MatcherAssert; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.util.List; import java.util.Map; @@ -27,10 +28,19 @@ import java.util.function.LongSupplier; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; - - +import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@SuppressWarnings("unchecked") public class ViewServiceTest { private final View.Target typicalTarget = new View.Target("my-index-*"); @@ -38,7 +48,7 @@ public class ViewServiceTest { private ClusterService clusterService; private NodeClient nodeClient; - private AtomicLong currentTime = new AtomicLong(0); + private final AtomicLong currentTime = new AtomicLong(0); private LongSupplier timeProvider = currentTime::longValue; private ViewService viewService; @@ -87,7 +97,7 @@ public void updateView_doesNotExist() { doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); final Exception ex = assertThrows(ResourceNotFoundException.class, () -> viewService.updateView(request, listener)); - assertThat(ex.getMessage(), equalTo("abc")); + MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); } @Test @@ -109,7 +119,7 @@ public void deleteView_doesNotExist() { final ResourceNotFoundException ex = assertThrows(ResourceNotFoundException.class, () -> viewService.deleteView(request, listener)); - assertThat(ex.getMessage(), equalTo("abc")); + MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); } @Test @@ -131,14 +141,14 @@ public void getView_doesNotExist() { final ResourceNotFoundException ex = assertThrows(ResourceNotFoundException.class, () -> viewService.getView(request, listener)); - assertThat(ex.getMessage(), equalTo("abc")); + MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); } @Test public void listViewNames() { - final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")) - .metadata(new Metadata.Builder().views(Map.of(typicalView.getName(), typicalView)).build()) - .build(); + final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")).metadata( + new Metadata.Builder().views(Map.of(typicalView.getName(), typicalView)).build() + ).build(); final var listener = mock(ActionListener.class); when(clusterService.state()).thenReturn(clusterState); From ea78012a81f51e8c21bb54a321f74e38b03a2297 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 6 Feb 2024 22:43:25 +0000 Subject: [PATCH 27/55] Add changelog entry Signed-off-by: Peter Nied --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bec5e0a6845a9..7a8511e6f0c5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow changing number of replicas of searchable snapshot index ([#11317](https://github.com/opensearch-project/OpenSearch/pull/11317)) - Adding slf4j license header to LoggerMessageFormat.java ([#11069](https://github.com/opensearch-project/OpenSearch/pull/11069)) - [BWC and API enforcement] Introduce checks for enforcing the API restrictions ([#11175](https://github.com/opensearch-project/OpenSearch/pull/11175)) +- Projected Views ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957)) ### Dependencies - Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276)) From 01655d0d9d9d352f249a648d16e92876b3b8a96d Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 8 Feb 2024 20:55:50 +0000 Subject: [PATCH 28/55] Fix build warning about test annotations Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewServiceTest.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java index 5053891a96915..3f76d59f45d90 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -66,7 +66,6 @@ public void after() { verifyNoMoreInteractions(timeProvider, clusterService, nodeClient); } - @Test public void createView() { final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); final var listener = mock(ActionListener.class); @@ -78,7 +77,6 @@ public void createView() { verify(timeProvider).getAsLong(); } - @Test public void updateView() { final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); final var listener = mock(ActionListener.class); @@ -90,7 +88,6 @@ public void updateView() { verify(timeProvider).getAsLong(); } - @Test public void updateView_doesNotExist() { final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); final var listener = mock(ActionListener.class); @@ -100,7 +97,6 @@ public void updateView_doesNotExist() { MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); } - @Test public void deleteView() { final var request = new DeleteViewAction.Request("viewName"); final var listener = mock(ActionListener.class); @@ -111,7 +107,6 @@ public void deleteView() { verify(clusterService).submitStateUpdateTask(eq("delete_view_task"), any()); } - @Test public void deleteView_doesNotExist() { final var request = new DeleteViewAction.Request("viewName"); final var listener = mock(ActionListener.class); @@ -122,7 +117,6 @@ public void deleteView_doesNotExist() { MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); } - @Test public void getView() { final var request = new GetViewAction.Request("viewName"); final var listener = mock(ActionListener.class); @@ -133,7 +127,6 @@ public void getView() { verify(listener).onResponse(any()); } - @Test public void getView_doesNotExist() { final var request = new GetViewAction.Request("viewName"); final var listener = mock(ActionListener.class); @@ -144,7 +137,6 @@ public void getView_doesNotExist() { MatcherAssert.assertThat(ex.getMessage(), equalTo("abc")); } - @Test public void listViewNames() { final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")).metadata( new Metadata.Builder().views(Map.of(typicalView.getName(), typicalView)).build() @@ -158,7 +150,6 @@ public void listViewNames() { verify(listener).onResponse(any()); } - @Test public void listViewNames_noViews() { final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")).build(); final var listener = mock(ActionListener.class); @@ -170,7 +161,6 @@ public void listViewNames_noViews() { verify(listener).onResponse(any()); } - @Test public void searchView() { final var request = spy(new SearchViewAction.Request("view")); final var listener = mock(ActionListener.class); From 292a322ffd9d3d296f023161ed9c6620f260be68 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 9 Feb 2024 21:39:50 +0000 Subject: [PATCH 29/55] Round out integration tests Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 138 ++++++++++-------- .../admin/indices/view/ViewTestBase.java | 74 ++++++++++ .../admin/indices/view/GetViewAction.java | 4 + .../admin/indices/view/ViewService.java | 2 +- .../opensearch/client/IndicesAdminClient.java | 12 +- .../admin/indices/view/ViewServiceTest.java | 1 - 6 files changed, 163 insertions(+), 68 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index 9c7b550a32ec5..9630c484d27db 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -8,11 +8,8 @@ package org.opensearch.action.admin.indices.view; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.index.IndexNotFoundException; -import org.opensearch.test.BackgroundIndexer; -import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.ResourceNotFoundException; +import org.opensearch.cluster.metadata.View; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; import org.hamcrest.MatcherAssert; @@ -21,52 +18,94 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; @ClusterScope(scope = Scope.TEST, numDataNodes = 2) -public class ViewIT extends OpenSearchIntegTestCase { +public class ViewIT extends ViewTestBase { - private int createIndexWithDocs(final String indexName) throws Exception { - createIndex(indexName); - ensureGreen(indexName); + public void testCreateView() throws Exception { + final String viewName = "test-view"; + final String indexPattern = "test-index-*"; - final int numOfDocs = scaledRandomIntBetween(0, 200); - try (final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) { - waitForDocs(numOfDocs, indexer); - } + logger.info("Testing createView with valid parameters"); + final View view = createView(viewName, indexPattern).getView(); + MatcherAssert.assertThat(view.getName(), is(viewName)); + MatcherAssert.assertThat(view.getTargets().size(), is(1)); + MatcherAssert.assertThat(view.getTargets().get(0).getIndexPattern(), is(indexPattern)); - refresh(indexName); - assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs); - return numOfDocs; + logger.info("Testing createView with existing view name"); + final Exception ex = assertThrows(ResourceNotFoundException.class, () -> createView(viewName, "new-pattern")); + MatcherAssert.assertThat(ex.getMessage(), is("View [test-view] already exists")); } - private GetViewAction.Response createView(final String name, final String indexPattern) throws Exception { - final CreateViewAction.Request request = new CreateViewAction.Request( - name, - null, - List.of(new CreateViewAction.Request.Target(indexPattern)) - ); - final GetViewAction.Response response = client().admin().indices().createView(request).actionGet(); - performRemoteStoreTestAction(); - return response; + public void testGetView() throws Exception { + final String viewName = "existing-view"; + + logger.info("Testing getView with existing view"); + createView(viewName, "index-*"); + final View view = getView(viewName).getView(); + MatcherAssert.assertThat(view.getName(), is(viewName)); + + logger.info("Testing getView with non-existent view"); + final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> getView("non-existent")); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); } - private void deleteView(final String name) { - client().admin().indices().deleteView(new DeleteViewAction.Request(name)).actionGet(); - performRemoteStoreTestAction(); + public void testDeleteView() throws Exception { + final String viewName = "deleted-view"; + createView(viewName, "index-*"); + + logger.info("Testing deleteView with existing view"); + deleteView(viewName); + final Exception whenDeletedEx = assertThrows(ResourceNotFoundException.class, () -> getView(viewName)); + MatcherAssert.assertThat(whenDeletedEx.getMessage(), is("View [deleted-view] does not exist")); + + logger.info("Testing deleteView with non-existent view"); + final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> deleteView("non-existent")); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); } - private List listViews() { - return client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + public void testUpdateView() throws Exception { + final String viewName = "updatable-view"; + final View originalView = createView(viewName, "index-old-*").getView(); + + logger.info("Testing updateView with existing view"); + final View updatedView = updateView(viewName, "new description", "index-new-*").getView(); + + MatcherAssert.assertThat(updatedView, not(is(originalView))); + MatcherAssert.assertThat(updatedView.getDescription(), is("new description")); + MatcherAssert.assertThat(updatedView.getTargets(), hasSize(1)); + MatcherAssert.assertThat(updatedView.getTargets().get(0).getIndexPattern(), is("index-new-*")); + + logger.info("Testing updateView with non-existent view"); + final Exception whenNeverExistedEx = assertThrows( + ResourceNotFoundException.class, + () -> updateView("non-existent", null, "index-*") + ); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); } - private SearchResponse searchView(final String viewName) throws Exception { - final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); - final SearchResponse response = client().searchView(request).actionGet(); - return response; + public void testListViewNames() throws Exception { + final String view1 = "view1"; + final String view2 = "view2"; + createView(view1, "index-1-*"); + createView(view2, "index-2-*"); + + logger.info("Testing listViewNames"); + final List views = listViewNames(); + MatcherAssert.assertThat(views, containsInAnyOrder(view1, view2)); + + logger.info("Testing listViewNames after deleting a view"); + deleteView(view1); + final List viewsAfterDeletion = listViewNames(); + MatcherAssert.assertThat(viewsAfterDeletion, not(contains(view1))); + MatcherAssert.assertThat(viewsAfterDeletion, contains(view2)); } - public void testBasicOperations() throws Exception { + public void testSearchOperations() throws Exception { final String indexInView1 = "index-1"; final String indexInView2 = "index-2"; final String indexNotInView = "another-index-1"; @@ -77,7 +116,7 @@ public void testBasicOperations() throws Exception { logger.info("Testing view with no matches"); createView("no-matches", "this-pattern-will-match-nothing"); - final IndexNotFoundException ex = assertThrows(IndexNotFoundException.class, () -> searchView("no-matches")); + final Exception ex = assertThrows(ResourceNotFoundException.class, () -> searchView("no-matches")); MatcherAssert.assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]")); logger.info("Testing view with exact index match"); @@ -87,31 +126,10 @@ public void testBasicOperations() throws Exception { logger.info("Testing view with wildcard matches"); createView("both-indices", "index-*"); assertHitCount(searchView("both-indices"), indexInView1DocCount + indexInView2DocCount); - } - - public void testListViewNames() throws Exception { - logger.info("Create a single view"); - createView("view1", "*"); - final List viewNames1 = listViews(); - - MatcherAssert.assertThat(viewNames1, contains("view1")); - - logger.info("Create a second view"); - createView("view2", "*"); - final List viewNames2 = listViews(); - - MatcherAssert.assertThat(viewNames2, contains("view1", "view2")); - - logger.info("Delete a view"); - deleteView("view1"); - final List viewNamesAfterDelete = listViews(); - - MatcherAssert.assertThat(viewNamesAfterDelete, contains("view2")); - logger.info("Update a view"); - client().admin().indices().updateView(new CreateViewAction.Request("view2", "newDescription", List.of())); - final List viewNamesAfterUpdate = listViews(); + logger.info("Testing searchView with non-existent view"); + final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> searchView("non-existent")); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); - MatcherAssert.assertThat(viewNamesAfterUpdate, contains("view2")); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java new file mode 100644 index 0000000000000..2d039f63f3dc2 --- /dev/null +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java @@ -0,0 +1,74 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; + +public abstract class ViewTestBase extends OpenSearchIntegTestCase { + + protected int createIndexWithDocs(final String indexName) throws Exception { + createIndex(indexName); + ensureGreen(indexName); + + final int numOfDocs = scaledRandomIntBetween(0, 200); + try (final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) { + waitForDocs(numOfDocs, indexer); + } + + refresh(indexName); + assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs); + return numOfDocs; + } + + protected GetViewAction.Response createView(final String name, final String indexPattern) throws Exception { + final CreateViewAction.Request request = new CreateViewAction.Request( + name, + null, + List.of(new CreateViewAction.Request.Target(indexPattern)) + ); + return client().admin().indices().createView(request).actionGet(); + } + + protected GetViewAction.Response getView(final String name) { + return client().admin().indices().getView(new GetViewAction.Request(name)).actionGet(); + + } + + protected void deleteView(final String name) { + client().admin().indices().deleteView(new DeleteViewAction.Request(name)).actionGet(); + performRemoteStoreTestAction(); + } + + protected List listViewNames() { + return client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames(); + } + + protected SearchResponse searchView(final String viewName) throws Exception { + final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); + final SearchResponse response = client().searchView(request).actionGet(); + return response; + } + + protected GetViewAction.Response updateView(final String name, final String description, final String indexPattern) { + final CreateViewAction.Request request = new CreateViewAction.Request( + name, + description, + List.of(new CreateViewAction.Request.Target(indexPattern)) + ); + final GetViewAction.Response response = client().admin().indices().updateView(request).actionGet(); + return response; + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java index 73a9ff1006aba..330646837b9e2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java @@ -127,6 +127,10 @@ public Response(final StreamInput in) throws IOException { this.view = new View(in); } + public View getView() { + return view; + } + @Override public void writeTo(final StreamOutput out) throws IOException { this.view.writeTo(out); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 484d7a158a7fa..3eb983b236b57 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -132,7 +132,7 @@ View getViewOrThrowException(final String viewName) { .map(ClusterState::metadata) .map(m -> m.views()) .map(views -> views.get(viewName)) - .orElseThrow(() -> new ResourceNotFoundException("no such view [" + viewName + "]")); + .orElseThrow(() -> new ResourceNotFoundException("View [" + viewName + "] does not exist")); } private enum Operation { diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index c1a75b02aaf95..588584cd8a280 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -848,21 +848,21 @@ public interface IndicesAdminClient extends OpenSearchClient { /** Create a view */ ActionFuture createView(CreateViewAction.Request request); - /** Gets a view */ + /** Get the details of a view */ void getView(GetViewAction.Request request, ActionListener listener); - /** Gets a view */ + /** Get the details of a view */ ActionFuture getView(GetViewAction.Request request); - /** Create a view */ + /** Delete a view */ void deleteView(DeleteViewAction.Request request, ActionListener listener); - /** Create a view */ + /** Delete a view */ ActionFuture deleteView(DeleteViewAction.Request request); - /** Create a view */ + /** Update a view */ void updateView(CreateViewAction.Request request, ActionListener listener); - /** Create a view */ + /** Update a view */ ActionFuture updateView(CreateViewAction.Request request); } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java index 3f76d59f45d90..93fa70b2981ac 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -20,7 +20,6 @@ import org.hamcrest.MatcherAssert; import org.junit.After; import org.junit.Before; -import org.junit.Test; import java.util.List; import java.util.Map; From 07bcfe0be616ddf1ba93e7c12bfe2ee78584ad44 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 9 Feb 2024 22:18:50 +0000 Subject: [PATCH 30/55] Add test cases Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 56 +++++++++++-------- .../admin/indices/view/DeleteViewAction.java | 2 +- .../admin/indices/view/GetViewAction.java | 15 ++++- .../indices/view/ListViewNamesAction.java | 27 +++++++++ ...Tests.java => CreateViewRequestTests.java} | 10 ++-- .../indices/view/DeleteViewRequestTests.java | 44 +++++++++++++++ .../indices/view/GetViewResponseTests.java | 34 +++++++++++ .../view/ListViewNamesRequestTests.java | 35 ++++++++++++ .../view/ListViewNamesResponseTests.java | 25 +++++++++ ...Tests.java => SearchViewRequestTests.java} | 14 +++-- ...ServiceTest.java => ViewServiceTests.java} | 41 +++++++++----- 11 files changed, 254 insertions(+), 49 deletions(-) rename server/src/test/java/org/opensearch/action/admin/indices/view/{CreateViewTests.java => CreateViewRequestTests.java} (75%) create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java create mode 100644 server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java rename server/src/test/java/org/opensearch/action/admin/indices/view/{SearchViewTests.java => SearchViewRequestTests.java} (73%) rename server/src/test/java/org/opensearch/action/admin/indices/view/{ViewServiceTest.java => ViewServiceTests.java} (81%) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index 9630c484d27db..aa0eca75d1a7c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -27,8 +27,8 @@ public class ViewIT extends ViewTestBase { public void testCreateView() throws Exception { - final String viewName = "test-view"; - final String indexPattern = "test-index-*"; + final String viewName = randomAlphaOfLength(8); + final String indexPattern = randomAlphaOfLength(8); logger.info("Testing createView with valid parameters"); final View view = createView(viewName, indexPattern).getView(); @@ -37,58 +37,66 @@ public void testCreateView() throws Exception { MatcherAssert.assertThat(view.getTargets().get(0).getIndexPattern(), is(indexPattern)); logger.info("Testing createView with existing view name"); - final Exception ex = assertThrows(ResourceNotFoundException.class, () -> createView(viewName, "new-pattern")); - MatcherAssert.assertThat(ex.getMessage(), is("View [test-view] already exists")); + final Exception ex = assertThrows(ResourceNotFoundException.class, () -> createView(viewName, randomAlphaOfLength(8))); + MatcherAssert.assertThat(ex.getMessage(), is("View [" + viewName + "] already exists")); } public void testGetView() throws Exception { - final String viewName = "existing-view"; + final String viewName = randomAlphaOfLength(8); + createView(viewName, randomAlphaOfLength(8)); - logger.info("Testing getView with existing view"); - createView(viewName, "index-*"); final View view = getView(viewName).getView(); MatcherAssert.assertThat(view.getName(), is(viewName)); logger.info("Testing getView with non-existent view"); - final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> getView("non-existent")); - MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); + final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> getView(nonExistentView)); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } public void testDeleteView() throws Exception { - final String viewName = "deleted-view"; - createView(viewName, "index-*"); + final String viewName = randomAlphaOfLength(8); + createView(viewName, randomAlphaOfLength(8)); logger.info("Testing deleteView with existing view"); deleteView(viewName); final Exception whenDeletedEx = assertThrows(ResourceNotFoundException.class, () -> getView(viewName)); - MatcherAssert.assertThat(whenDeletedEx.getMessage(), is("View [deleted-view] does not exist")); + MatcherAssert.assertThat(whenDeletedEx.getMessage(), is("View [" + viewName + "] does not exist")); logger.info("Testing deleteView with non-existent view"); - final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> deleteView("non-existent")); - MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); + final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> deleteView(nonExistentView)); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } public void testUpdateView() throws Exception { - final String viewName = "updatable-view"; - final View originalView = createView(viewName, "index-old-*").getView(); + final String viewName = randomAlphaOfLength(8); + final String originalIndexPattern = randomAlphaOfLength(8); + final View originalView = createView(viewName, originalIndexPattern).getView(); logger.info("Testing updateView with existing view"); - final View updatedView = updateView(viewName, "new description", "index-new-*").getView(); + final String newDescription = randomAlphaOfLength(20); + final String newIndexPattern = "newPattern-" + originalIndexPattern; + final View updatedView = updateView(viewName, newDescription, newIndexPattern).getView(); MatcherAssert.assertThat(updatedView, not(is(originalView))); - MatcherAssert.assertThat(updatedView.getDescription(), is("new description")); + MatcherAssert.assertThat(updatedView.getDescription(), is(newDescription)); MatcherAssert.assertThat(updatedView.getTargets(), hasSize(1)); - MatcherAssert.assertThat(updatedView.getTargets().get(0).getIndexPattern(), is("index-new-*")); + MatcherAssert.assertThat(updatedView.getTargets().get(0).getIndexPattern(), is(newIndexPattern)); logger.info("Testing updateView with non-existent view"); + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); final Exception whenNeverExistedEx = assertThrows( ResourceNotFoundException.class, - () -> updateView("non-existent", null, "index-*") + () -> updateView(nonExistentView, null, "index-*") ); - MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } public void testListViewNames() throws Exception { + logger.info("Testing listViewNames when no views have been created"); + MatcherAssert.assertThat(listViewNames(), is(List.of())); + final String view1 = "view1"; final String view2 = "view2"; createView(view1, "index-1-*"); @@ -128,8 +136,8 @@ public void testSearchOperations() throws Exception { assertHitCount(searchView("both-indices"), indexInView1DocCount + indexInView2DocCount); logger.info("Testing searchView with non-existent view"); - final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> searchView("non-existent")); - MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [non-existent] does not exist")); - + final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); + final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> searchView(nonExistentView)); + MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java index 9b0adebcfaee4..abb3c3f4db5f6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java @@ -69,7 +69,7 @@ public String getName() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Request that = (Request) o; + final Request that = (Request) o; return name.equals(that.name); } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java index 330646837b9e2..762eea965c8c1 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/GetViewAction.java @@ -72,7 +72,7 @@ public String getName() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Request that = (Request) o; + final Request that = (Request) o; return name.equals(that.name); } @@ -131,6 +131,19 @@ public View getView() { return view; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Response that = (Response) o; + return getView().equals(that.getView()); + } + + @Override + public int hashCode() { + return Objects.hash(getView()); + } + @Override public void writeTo(final StreamOutput out) throws IOException { this.view.writeTo(out); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java index b2c9ac1fe857c..8341d102a10ea 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; /** Action to list a view names */ @ExperimentalApi @@ -45,6 +46,19 @@ public Request() {} public Request(final StreamInput in) {} + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Request that = (Request) o; + return true; + } + + @Override + public int hashCode() { + return 1; + } + @Override public ActionRequestValidationException validate() { return null; @@ -69,6 +83,19 @@ public List getViewNames() { return views; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final Request that = (Request) o; + return views.equals(that.views); + } + + @Override + public int hashCode() { + return Objects.hash(views); + } + @Override public void writeTo(final StreamOutput out) throws IOException { out.writeStringCollection(views); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java similarity index 75% rename from server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java rename to server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java index 7b4d29d7fb839..edb9f398debbe 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java @@ -11,12 +11,14 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; import java.util.List; import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.nullValue; -public class CreateViewTests extends AbstractWireSerializingTestCase { +public class CreateViewRequestTests extends AbstractWireSerializingTestCase { @Override protected Writeable.Reader instanceReader() { @@ -39,14 +41,14 @@ public void testValidateRequest() { List.of(new CreateViewAction.Request.Target("my-indices-*")) ); - assertNull(request.validate()); + MatcherAssert.assertThat(request.validate(), nullValue()); } public void testValidateRequestWithoutName() { final CreateViewAction.Request request = new CreateViewAction.Request("", null, null); - ActionRequestValidationException e = request.validate(); + final ActionRequestValidationException e = request.validate(); - assertThat(e.validationErrors(), contains("name cannot be empty or null", "targets cannot be empty")); + MatcherAssert.assertThat(e.validationErrors(), contains("name cannot be empty or null", "targets cannot be empty")); } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java new file mode 100644 index 0000000000000..29305e3dfb92f --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/DeleteViewRequestTests.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.action.ActionRequestValidationException; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.nullValue; + +public class DeleteViewRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return DeleteViewAction.Request::new; + } + + @Override + protected DeleteViewAction.Request createTestInstance() { + return new DeleteViewAction.Request(randomAlphaOfLength(8)); + } + + public void testValidateRequest() { + final DeleteViewAction.Request request = new DeleteViewAction.Request("my-view"); + + MatcherAssert.assertThat(request.validate(), nullValue()); + } + + public void testValidateRequestWithoutName() { + final DeleteViewAction.Request request = new DeleteViewAction.Request(""); + final ActionRequestValidationException e = request.validate(); + + MatcherAssert.assertThat(e.validationErrors(), contains("name cannot be empty or null")); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java new file mode 100644 index 0000000000000..4dd075597a804 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java @@ -0,0 +1,34 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.cluster.metadata.View; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; + +public class GetViewResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return GetViewAction.Response::new; + } + + @Override + protected GetViewAction.Response createTestInstance() { + return new GetViewAction.Response( + new View( + randomAlphaOfLength(8), + randomAlphaOfLength(8), + randomLong(), + randomLong(), + randomList(5, () -> new View.Target(randomAlphaOfLength(8))) + ) + ); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java new file mode 100644 index 0000000000000..80a2827d158bb --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesRequestTests.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; + +import static org.hamcrest.Matchers.nullValue; + +public class ListViewNamesRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return ListViewNamesAction.Request::new; + } + + @Override + protected ListViewNamesAction.Request createTestInstance() { + return new ListViewNamesAction.Request(); + } + + public void testValidateRequest() { + final ListViewNamesAction.Request request = new ListViewNamesAction.Request(); + + MatcherAssert.assertThat(request.validate(), nullValue()); + } + +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java new file mode 100644 index 0000000000000..ee8409fe3c805 --- /dev/null +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ListViewNamesResponseTests.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; + +public class ListViewNamesResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return ListViewNamesAction.Response::new; + } + + @Override + protected ListViewNamesAction.Response createTestInstance() { + return new ListViewNamesAction.Response(randomList(5, () -> randomAlphaOfLength(8))); + } +} diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java similarity index 73% rename from server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java rename to server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java index e07104cbb6ad6..c6d2fcc19563e 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java @@ -12,13 +12,15 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.MatcherAssert; import java.io.IOException; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; -public class SearchViewTests extends AbstractWireSerializingTestCase { +public class SearchViewRequestTests extends AbstractWireSerializingTestCase { @Override protected Writeable.Reader instanceReader() { @@ -36,15 +38,15 @@ protected SearchViewAction.Request createTestInstance() { public void testValidateRequest() throws IOException { final SearchViewAction.Request request = SearchViewAction.createRequestWith("my-view", new SearchRequest()); - assertNull(request.validate()); + MatcherAssert.assertThat(request.validate(), nullValue()); } public void testValidateRequestWithoutName() { final SearchViewAction.Request request = new SearchViewAction.Request((String) null); - ActionRequestValidationException e = request.validate(); - assertNotNull(e); - assertThat(e.validationErrors().size(), equalTo(1)); - assertThat(e.validationErrors().get(0), containsString("View is required")); + final ActionRequestValidationException e = request.validate(); + + MatcherAssert.assertThat(e.validationErrors().size(), equalTo(1)); + MatcherAssert.assertThat(e.validationErrors().get(0), containsString("View is required")); } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTests.java similarity index 81% rename from server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java rename to server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTests.java index 93fa70b2981ac..45b48f5f9dec5 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTests.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongSupplier; +import static org.opensearch.test.OpenSearchTestCase.randomAlphaOfLength; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.any; @@ -40,10 +41,16 @@ import static org.mockito.Mockito.when; @SuppressWarnings("unchecked") -public class ViewServiceTest { +public class ViewServiceTests { - private final View.Target typicalTarget = new View.Target("my-index-*"); - private final View typicalView = new View("actualView", "actual description", -1L, -1L, List.of(typicalTarget)); + private final View.Target typicalTarget = new View.Target(randomAlphaOfLength(8)); + private final View typicalView = new View( + "view-" + randomAlphaOfLength(8), + "description " + randomAlphaOfLength(20), + -1L, + -1L, + List.of(typicalTarget) + ); private ClusterService clusterService; private NodeClient nodeClient; @@ -65,8 +72,16 @@ public void after() { verifyNoMoreInteractions(timeProvider, clusterService, nodeClient); } + private CreateViewAction.Request createTypicalViewRequest() { + return new CreateViewAction.Request( + randomAlphaOfLength(8), + randomAlphaOfLength(20), + List.of(new CreateViewAction.Request.Target(randomAlphaOfLength(8))) + ); + } + public void createView() { - final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); + final var request = createTypicalViewRequest(); final var listener = mock(ActionListener.class); setGetViewOrThrowExceptionToReturnTypicalView(); @@ -77,7 +92,7 @@ public void createView() { } public void updateView() { - final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); + final var request = createTypicalViewRequest(); final var listener = mock(ActionListener.class); setGetViewOrThrowExceptionToReturnTypicalView(); @@ -88,7 +103,7 @@ public void updateView() { } public void updateView_doesNotExist() { - final var request = new CreateViewAction.Request("a", "b", List.of(new CreateViewAction.Request.Target("my-index-*"))); + final var request = createTypicalViewRequest(); final var listener = mock(ActionListener.class); doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); @@ -97,7 +112,7 @@ public void updateView_doesNotExist() { } public void deleteView() { - final var request = new DeleteViewAction.Request("viewName"); + final var request = new DeleteViewAction.Request(randomAlphaOfLength(8)); final var listener = mock(ActionListener.class); setGetViewOrThrowExceptionToReturnTypicalView(); @@ -107,7 +122,7 @@ public void deleteView() { } public void deleteView_doesNotExist() { - final var request = new DeleteViewAction.Request("viewName"); + final var request = new DeleteViewAction.Request(randomAlphaOfLength(8)); final var listener = mock(ActionListener.class); doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); @@ -117,7 +132,7 @@ public void deleteView_doesNotExist() { } public void getView() { - final var request = new GetViewAction.Request("viewName"); + final var request = new GetViewAction.Request(randomAlphaOfLength(8)); final var listener = mock(ActionListener.class); setGetViewOrThrowExceptionToReturnTypicalView(); @@ -127,7 +142,7 @@ public void getView() { } public void getView_doesNotExist() { - final var request = new GetViewAction.Request("viewName"); + final var request = new GetViewAction.Request(randomAlphaOfLength(8)); final var listener = mock(ActionListener.class); doThrow(new ResourceNotFoundException("abc")).when(viewService).getViewOrThrowException(anyString()); @@ -137,7 +152,7 @@ public void getView_doesNotExist() { } public void listViewNames() { - final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")).metadata( + final var clusterState = new ClusterState.Builder(new ClusterName(randomAlphaOfLength(8))).metadata( new Metadata.Builder().views(Map.of(typicalView.getName(), typicalView)).build() ).build(); final var listener = mock(ActionListener.class); @@ -150,7 +165,7 @@ public void listViewNames() { } public void listViewNames_noViews() { - final var clusterState = new ClusterState.Builder(new ClusterName("MyCluster")).build(); + final var clusterState = new ClusterState.Builder(new ClusterName(randomAlphaOfLength(8))).build(); final var listener = mock(ActionListener.class); when(clusterService.state()).thenReturn(clusterState); @@ -161,7 +176,7 @@ public void listViewNames_noViews() { } public void searchView() { - final var request = spy(new SearchViewAction.Request("view")); + final var request = spy(new SearchViewAction.Request(randomAlphaOfLength(8))); final var listener = mock(ActionListener.class); setGetViewOrThrowExceptionToReturnTypicalView(); From e7f36b0bb5e4fa57f583b2e5f240cc3c91df552c Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Fri, 9 Feb 2024 23:03:13 +0000 Subject: [PATCH 31/55] Fix compile issue Signed-off-by: Peter Nied --- .../action/admin/indices/view/ListViewNamesAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java index 8341d102a10ea..eac0b1d5558ca 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ListViewNamesAction.java @@ -87,7 +87,7 @@ public List getViewNames() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - final Request that = (Request) o; + final Response that = (Response) o; return views.equals(that.views); } From 65fa8e7e869577f6c4798fdbdcb529977d81cf39 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 12 Feb 2024 16:29:52 +0000 Subject: [PATCH 32/55] Rename test class Signed-off-by: Peter Nied --- .../view/{ViewServiceTests.java => ViewServiceTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename server/src/test/java/org/opensearch/action/admin/indices/view/{ViewServiceTests.java => ViewServiceTest.java} (99%) diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java similarity index 99% rename from server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTests.java rename to server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java index 45b48f5f9dec5..076bcdf4e1eec 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -41,7 +41,7 @@ import static org.mockito.Mockito.when; @SuppressWarnings("unchecked") -public class ViewServiceTests { +public class ViewServiceTest { private final View.Target typicalTarget = new View.Target(randomAlphaOfLength(8)); private final View typicalView = new View( From 43272377801d88be3338c8b966fc88e5cb504139 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 12 Feb 2024 22:39:32 +0000 Subject: [PATCH 33/55] PR feedback pass Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewService.java | 17 +++++++---------- .../org/opensearch/cluster/metadata/View.java | 6 +++--- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 3eb983b236b57..c5ea69239bc5b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -23,6 +23,7 @@ import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.action.ActionListener; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -36,7 +37,7 @@ public class ViewService { private final static Logger LOG = LogManager.getLogger(ViewService.class); private final ClusterService clusterService; private final NodeClient client; - private LongSupplier timeProvider; + private final LongSupplier timeProvider; public ViewService(final ClusterService clusterService, final NodeClient client, final LongSupplier timeProvider) { this.clusterService = clusterService; @@ -69,7 +70,6 @@ public void updateView(final CreateViewAction.Request request, final ActionListe createOrUpdateView(Operation.UpdateView, updatedView, listener); } - @SuppressWarnings("deprecation") public void deleteView(final DeleteViewAction.Request request, final ActionListener listener) { getViewOrThrowException(request.getName()); @@ -101,14 +101,12 @@ public void getView(final GetViewAction.Request request, final ActionListener listener) { - final List viewNames = Optional.ofNullable(clusterService) + final List viewNames = new ArrayList<>(Optional.ofNullable(clusterService) .map(ClusterService::state) .map(ClusterState::metadata) .map(Metadata::views) .map(Map::keySet) - .orElseThrow() - .stream() - .collect(Collectors.toList()); + .orElseThrow()); listener.onResponse(new ListViewNamesAction.Response(viewNames)); } @@ -119,8 +117,7 @@ public void searchView(final SearchViewAction.Request request, final ActionListe final String[] indices = view.getTargets() .stream() .map(View.Target::getIndexPattern) - .collect(Collectors.toList()) - .toArray(new String[0]); + .toArray(String[]::new); request.indices(indices); client.executeLocally(SearchAction.INSTANCE, request, listener); @@ -130,7 +127,7 @@ View getViewOrThrowException(final String viewName) { return Optional.ofNullable(clusterService) .map(ClusterService::state) .map(ClusterState::metadata) - .map(m -> m.views()) + .map(Metadata::views) .map(views -> views.get(viewName)) .orElseThrow(() -> new ResourceNotFoundException("View [" + viewName + "] does not exist")); } @@ -141,7 +138,7 @@ private enum Operation { private final String name; - private Operation(final String name) { + Operation(final String name) { this.name = name; } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index 52b180ffa0265..09a2d09e7f7b5 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -43,7 +43,7 @@ public View(final String name, final String description, final Long createdAt, f } public View(final StreamInput in) throws IOException { - this(in.readString(), in.readOptionalString(), in.readVLong(), in.readVLong(), in.readList(Target::new)); + this(in.readString(), in.readOptionalString(), in.readZLong(), in.readZLong(), in.readList(Target::new)); } public String getName() { @@ -186,8 +186,8 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa public void writeTo(final StreamOutput out) throws IOException { out.writeString(name); out.writeOptionalString(description); - out.writeVLong(createdAt); - out.writeVLong(modifiedAt); + out.writeZLong(createdAt); + out.writeZLong(modifiedAt); out.writeList(targets); } } From 5beefbbe55cb1f0132810a91da9f910ddafe5159 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Mon, 12 Feb 2024 22:58:00 +0000 Subject: [PATCH 34/55] Spotless Signed-off-by: Peter Nied --- .../admin/indices/view/ViewService.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index c5ea69239bc5b..734453cb2223f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -101,12 +101,14 @@ public void getView(final GetViewAction.Request request, final ActionListener listener) { - final List viewNames = new ArrayList<>(Optional.ofNullable(clusterService) - .map(ClusterService::state) - .map(ClusterState::metadata) - .map(Metadata::views) - .map(Map::keySet) - .orElseThrow()); + final List viewNames = new ArrayList<>( + Optional.ofNullable(clusterService) + .map(ClusterService::state) + .map(ClusterState::metadata) + .map(Metadata::views) + .map(Map::keySet) + .orElseThrow() + ); listener.onResponse(new ListViewNamesAction.Response(viewNames)); } @@ -114,10 +116,7 @@ public void listViewNames(final ActionListener lis public void searchView(final SearchViewAction.Request request, final ActionListener listener) { final View view = getViewOrThrowException(request.getView()); - final String[] indices = view.getTargets() - .stream() - .map(View.Target::getIndexPattern) - .toArray(String[]::new); + final String[] indices = view.getTargets().stream().map(View.Target::getIndexPattern).toArray(String[]::new); request.indices(indices); client.executeLocally(SearchAction.INSTANCE, request, listener); From edd890264e5eb0e816945ffe01a9b7a8f784ab81 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 06:59:56 -0600 Subject: [PATCH 35/55] More logging information Signed-off-by: Peter Nied --- .../src/main/java/org/opensearch/test/OpenSearchTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 96bffcf2d3692..0efd4d79a857a 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -639,7 +639,7 @@ protected static void checkStaticState(boolean afterClass) throws Exception { try { // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the // StatusData instances to Strings as otherwise their toString output is useless - assertThat( + assertThat(statusData.stream().map(status -> status.getLevel() + ": " + status.getMessage() + (status.getThrowable() == null ? "" : "\r\n\r\nTrowable:\r\n" + status.getThrowable().toString())).collect(Collectors.joining("\r\n")), statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()), empty() ); From ba385e4ac5c6ebc170acae994cdbb29cb88b378d Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 07:26:45 -0600 Subject: [PATCH 36/55] Fix spotless Signed-off-by: Peter Nied --- .../java/org/opensearch/test/OpenSearchTestCase.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 0efd4d79a857a..90032d261b4ac 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -639,7 +639,15 @@ protected static void checkStaticState(boolean afterClass) throws Exception { try { // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the // StatusData instances to Strings as otherwise their toString output is useless - assertThat(statusData.stream().map(status -> status.getLevel() + ": " + status.getMessage() + (status.getThrowable() == null ? "" : "\r\n\r\nTrowable:\r\n" + status.getThrowable().toString())).collect(Collectors.joining("\r\n")), + assertThat( + statusData.stream() + .map( + status -> status.getLevel() + + ": " + + status.getMessage() + + (status.getThrowable() == null ? "" : "\r\n\r\nTrowable:\r\n" + status.getThrowable().toString()) + ) + .collect(Collectors.joining("\r\n")), statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()), empty() ); From 0663818c98d90b00cc6edc9aae957d9275af8b56 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 08:37:00 -0600 Subject: [PATCH 37/55] More detailed logging from checkStaticState Signed-off-by: Peter Nied --- .../opensearch/test/OpenSearchTestCase.java | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 90032d261b4ac..4f757f1fb1e25 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -145,6 +145,8 @@ import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.math.BigInteger; import java.net.InetAddress; import java.net.UnknownHostException; @@ -170,6 +172,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import java.util.function.Predicate; import java.util.function.Supplier; @@ -639,15 +642,36 @@ protected static void checkStaticState(boolean afterClass) throws Exception { try { // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the // StatusData instances to Strings as otherwise their toString output is useless + + final Function statusToString = (statusData) -> { + try (final StringWriter sw = new StringWriter(); final PrintWriter pw = new PrintWriter(sw)) { + + pw.print(statusData.getLevel()); + pw.print(":"); + pw.print(statusData.getMessage().getFormattedMessage()); + + if (statusData.getStackTraceElement() != null) { + final var messageSource = statusData.getStackTraceElement(); + + pw.println(); + pw.println("Source:"); + pw.println(messageSource.getFileName() + "@" + messageSource.getLineNumber()); + } + + if (statusData.getThrowable() != null) { + pw.println(); + pw.println("Throwable:"); + pw.println(statusData.getThrowable().toString()); + statusData.getThrowable().printStackTrace(pw); + } + return sw.toString(); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + }; + assertThat( - statusData.stream() - .map( - status -> status.getLevel() - + ": " - + status.getMessage() - + (status.getThrowable() == null ? "" : "\r\n\r\nTrowable:\r\n" + status.getThrowable().toString()) - ) - .collect(Collectors.joining("\r\n")), + statusData.stream().map(statusToString::apply).collect(Collectors.joining("\r\n")), statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()), empty() ); From 7f28330201c6bc1cbd925842ccbf975fc370b32c Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 09:18:43 -0600 Subject: [PATCH 38/55] Detact metadataFilesEligibleToDelete from its source collection Signed-off-by: Peter Nied --- .../opensearch/index/store/RemoteSegmentStoreDirectory.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index dab99fd25b192..49db4ff71456f 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -745,10 +745,10 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException return; } - List metadataFilesEligibleToDelete = sortedMetadataFileList.subList( + List metadataFilesEligibleToDelete = new ArrayList<>(sortedMetadataFileList.subList( lastNMetadataFilesToKeep, sortedMetadataFileList.size() - ); + )); Set allLockFiles; try { allLockFiles = ((RemoteStoreMetadataLockManager) mdLockManager).fetchLockedMetadataFiles(MetadataFilenameUtils.METADATA_PREFIX); @@ -764,7 +764,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException logger.debug( "metadataFilesEligibleToDelete={} metadataFilesToBeDeleted={}", metadataFilesEligibleToDelete, - metadataFilesEligibleToDelete + metadataFilesToBeDeleted ); Map activeSegmentFilesMetadataMap = new HashMap<>(); From 8cb9f55490d72ba4474f0bd97ead04298ef88439 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 10:01:26 -0600 Subject: [PATCH 39/55] Fix spotless issue Signed-off-by: Peter Nied --- .../index/store/RemoteSegmentStoreDirectory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 49db4ff71456f..bfab9f8c18aa2 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -45,6 +45,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -745,10 +746,9 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException return; } - List metadataFilesEligibleToDelete = new ArrayList<>(sortedMetadataFileList.subList( - lastNMetadataFilesToKeep, - sortedMetadataFileList.size() - )); + List metadataFilesEligibleToDelete = new ArrayList<>( + sortedMetadataFileList.subList(lastNMetadataFilesToKeep, sortedMetadataFileList.size()) + ); Set allLockFiles; try { allLockFiles = ((RemoteStoreMetadataLockManager) mdLockManager).fetchLockedMetadataFiles(MetadataFilenameUtils.METADATA_PREFIX); From 9d0e1b00b669c44a247a116604e266006b8173ad Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 20:57:28 +0000 Subject: [PATCH 40/55] Breakout exceptions into their own clases Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 19 +++++----- .../opensearch/OpenSearchServerException.java | 36 ++++++++++++++++--- .../view/ViewAlreadyExistsException.java | 28 +++++++++++++++ .../indices/view/ViewNotFoundException.java | 28 +++++++++++++++ .../admin/indices/view/ViewService.java | 14 +++++--- 5 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index aa0eca75d1a7c..e9de79109e641 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -8,8 +8,8 @@ package org.opensearch.action.admin.indices.view; -import org.opensearch.ResourceNotFoundException; import org.opensearch.cluster.metadata.View; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope; import org.opensearch.test.OpenSearchIntegTestCase.Scope; import org.hamcrest.MatcherAssert; @@ -37,7 +37,7 @@ public void testCreateView() throws Exception { MatcherAssert.assertThat(view.getTargets().get(0).getIndexPattern(), is(indexPattern)); logger.info("Testing createView with existing view name"); - final Exception ex = assertThrows(ResourceNotFoundException.class, () -> createView(viewName, randomAlphaOfLength(8))); + final Exception ex = assertThrows(ViewAlreadyExistsException.class, () -> createView(viewName, randomAlphaOfLength(8))); MatcherAssert.assertThat(ex.getMessage(), is("View [" + viewName + "] already exists")); } @@ -50,7 +50,7 @@ public void testGetView() throws Exception { logger.info("Testing getView with non-existent view"); final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); - final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> getView(nonExistentView)); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> getView(nonExistentView)); MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } @@ -60,12 +60,12 @@ public void testDeleteView() throws Exception { logger.info("Testing deleteView with existing view"); deleteView(viewName); - final Exception whenDeletedEx = assertThrows(ResourceNotFoundException.class, () -> getView(viewName)); + final Exception whenDeletedEx = assertThrows(ViewNotFoundException.class, () -> getView(viewName)); MatcherAssert.assertThat(whenDeletedEx.getMessage(), is("View [" + viewName + "] does not exist")); logger.info("Testing deleteView with non-existent view"); final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); - final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> deleteView(nonExistentView)); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> deleteView(nonExistentView)); MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } @@ -86,10 +86,7 @@ public void testUpdateView() throws Exception { logger.info("Testing updateView with non-existent view"); final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); - final Exception whenNeverExistedEx = assertThrows( - ResourceNotFoundException.class, - () -> updateView(nonExistentView, null, "index-*") - ); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> updateView(nonExistentView, null, "index-*")); MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } @@ -124,7 +121,7 @@ public void testSearchOperations() throws Exception { logger.info("Testing view with no matches"); createView("no-matches", "this-pattern-will-match-nothing"); - final Exception ex = assertThrows(ResourceNotFoundException.class, () -> searchView("no-matches")); + final Exception ex = assertThrows(IndexNotFoundException.class, () -> searchView("no-matches")); MatcherAssert.assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]")); logger.info("Testing view with exact index match"); @@ -137,7 +134,7 @@ public void testSearchOperations() throws Exception { logger.info("Testing searchView with non-existent view"); final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); - final Exception whenNeverExistedEx = assertThrows(ResourceNotFoundException.class, () -> searchView(nonExistentView)); + final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> searchView(nonExistentView)); MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist")); } } diff --git a/server/src/main/java/org/opensearch/OpenSearchServerException.java b/server/src/main/java/org/opensearch/OpenSearchServerException.java index 39c22e60f038a..c5a5ce12b238c 100644 --- a/server/src/main/java/org/opensearch/OpenSearchServerException.java +++ b/server/src/main/java/org/opensearch/OpenSearchServerException.java @@ -8,13 +8,11 @@ package org.opensearch; -import org.opensearch.core.index.snapshots.IndexShardSnapshotException; -import org.opensearch.crypto.CryptoRegistryException; - import static org.opensearch.OpenSearchException.OpenSearchExceptionHandle; import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle; import static org.opensearch.OpenSearchException.UNKNOWN_VERSION_ADDED; import static org.opensearch.Version.V_2_10_0; +import static org.opensearch.Version.V_2_13_0; import static org.opensearch.Version.V_2_1_0; import static org.opensearch.Version.V_2_4_0; import static org.opensearch.Version.V_2_5_0; @@ -678,7 +676,12 @@ public static void registerExceptions() { ) ); registerExceptionHandle( - new OpenSearchExceptionHandle(IndexShardSnapshotException.class, IndexShardSnapshotException::new, 98, UNKNOWN_VERSION_ADDED) + new OpenSearchExceptionHandle( + org.opensearch.core.index.snapshots.IndexShardSnapshotException.class, + org.opensearch.core.index.snapshots.IndexShardSnapshotException::new, + 98, + UNKNOWN_VERSION_ADDED + ) ); registerExceptionHandle( new OpenSearchExceptionHandle( @@ -1174,7 +1177,30 @@ public static void registerExceptions() { V_2_7_0 ) ); - registerExceptionHandle(new OpenSearchExceptionHandle(CryptoRegistryException.class, CryptoRegistryException::new, 171, V_2_10_0)); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.crypto.CryptoRegistryException.class, + org.opensearch.crypto.CryptoRegistryException::new, + 171, + V_2_10_0 + ) + ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.action.admin.indices.view.ViewNotFoundException.class, + org.opensearch.action.admin.indices.view.ViewNotFoundException::new, + 172, + V_2_13_0 + ) + ); + registerExceptionHandle( + new OpenSearchExceptionHandle( + org.opensearch.action.admin.indices.view.ViewAlreadyExistsException.class, + org.opensearch.action.admin.indices.view.ViewAlreadyExistsException::new, + 173, + V_2_13_0 + ) + ); registerExceptionHandle( new OpenSearchExceptionHandle( org.opensearch.cluster.block.IndexCreateBlockException.class, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java new file mode 100644 index 0000000000000..90a69158286b4 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewAlreadyExistsException.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.ResourceAlreadyExistsException; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** Exception thrown when a view already exists */ +@ExperimentalApi +public class ViewAlreadyExistsException extends ResourceAlreadyExistsException { + + public ViewAlreadyExistsException(final String viewName) { + super("View [{}] already exists", viewName); + } + + public ViewAlreadyExistsException(final StreamInput in) throws IOException { + super(in); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java new file mode 100644 index 0000000000000..3a90e6b0bc791 --- /dev/null +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewNotFoundException.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.admin.indices.view; + +import org.opensearch.ResourceNotFoundException; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** Exception thrown when a view is not found */ +@ExperimentalApi +public class ViewNotFoundException extends ResourceNotFoundException { + + public ViewNotFoundException(final String viewName) { + super("View [{}] does not exist", viewName); + } + + public ViewNotFoundException(final StreamInput in) throws IOException { + super(in); + } +} diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 734453cb2223f..0fd5b7c12eeec 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.ResourceNotFoundException; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -128,17 +127,19 @@ View getViewOrThrowException(final String viewName) { .map(ClusterState::metadata) .map(Metadata::views) .map(views -> views.get(viewName)) - .orElseThrow(() -> new ResourceNotFoundException("View [" + viewName + "] does not exist")); + .orElseThrow(() -> new ViewNotFoundException(viewName)); } private enum Operation { - CreateView("create"), - UpdateView("update"); + CreateView("create", false), + UpdateView("update", true); private final String name; + private final boolean allowOverriding; - Operation(final String name) { + Operation(final String name, final boolean allowOverriding) { this.name = name; + this.allowOverriding = allowOverriding; } } @@ -146,6 +147,9 @@ private void createOrUpdateView(final Operation operation, final View view, fina clusterService.submitStateUpdateTask(operation.name + "_view_task", new ClusterStateUpdateTask() { @Override public ClusterState execute(final ClusterState currentState) throws Exception { + if (!operation.allowOverriding && currentState.metadata().views().containsKey(view.getName())) { + throw new ViewAlreadyExistsException(view.getName()); + } return new ClusterState.Builder(clusterService.state()).metadata(Metadata.builder(currentState.metadata()).put(view)) .build(); } From e58acdba8c1da785aeb2a093aaa540a69012ea5c Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 20:59:05 +0000 Subject: [PATCH 41/55] Clean up logger output message Signed-off-by: Peter Nied --- .../src/main/java/org/opensearch/test/OpenSearchTestCase.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 4f757f1fb1e25..aac3fca9e1e16 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -652,16 +652,12 @@ protected static void checkStaticState(boolean afterClass) throws Exception { if (statusData.getStackTraceElement() != null) { final var messageSource = statusData.getStackTraceElement(); - - pw.println(); pw.println("Source:"); pw.println(messageSource.getFileName() + "@" + messageSource.getLineNumber()); } if (statusData.getThrowable() != null) { - pw.println(); pw.println("Throwable:"); - pw.println(statusData.getThrowable().toString()); statusData.getThrowable().printStackTrace(pw); } return sw.toString(); From 6e9dffb15c5d10a8e71bbbf7400317538ef08f6d Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 21:01:42 +0000 Subject: [PATCH 42/55] Revert changes to NamedRoute Signed-off-by: Peter Nied --- server/src/main/java/org/opensearch/rest/NamedRoute.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/rest/NamedRoute.java b/server/src/main/java/org/opensearch/rest/NamedRoute.java index b477cd8571416..109f688a4924e 100644 --- a/server/src/main/java/org/opensearch/rest/NamedRoute.java +++ b/server/src/main/java/org/opensearch/rest/NamedRoute.java @@ -144,8 +144,7 @@ private NamedRoute(Builder builder) { "Invalid route name specified. The route name may include the following characters" + " 'a-z', 'A-Z', '0-9', ':', '/', '*', '_' and be less than " + MAX_LENGTH_OF_ACTION_NAME - + " characters, " - + builder.uniqueName + + " characters" ); } this.uniqueName = builder.uniqueName; From cba189e11c38dd8c807857ca12838944e681d77b Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 22:50:28 +0000 Subject: [PATCH 43/55] Revert changes to SearchRequest and follow pipeline model Signed-off-by: Peter Nied --- .../action/admin/indices/view/SearchViewAction.java | 4 ++-- .../org/opensearch/action/search/SearchRequest.java | 10 +++++----- .../rest/action/admin/indices/RestViewAction.java | 3 ++- .../admin/indices/view/SearchViewRequestTests.java | 2 +- .../action/admin/indices/view/ViewServiceTest.java | 3 ++- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java index 25c0906cc2ffd..953be210b0825 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java @@ -61,8 +61,8 @@ public static class Request extends SearchRequest { private final String view; - public Request(final String view) { - super(); + public Request(final String view, final SearchRequest searchRequest) { + super(searchRequest); this.view = view; } diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java index b166e17dee12d..f738c182c06da 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java @@ -82,13 +82,13 @@ public class SearchRequest extends ActionRequest implements IndicesRequest.Repla private static final long DEFAULT_ABSOLUTE_START_MILLIS = -1; - protected final String localClusterAlias; - protected final long absoluteStartMillis; - protected final boolean finalReduce; + private final String localClusterAlias; + private final long absoluteStartMillis; + private final boolean finalReduce; private SearchType searchType = SearchType.DEFAULT; - protected String[] indices = Strings.EMPTY_ARRAY; + private String[] indices = Strings.EMPTY_ARRAY; @Nullable private String routing; @@ -189,7 +189,7 @@ static SearchRequest subSearchRequest( return new SearchRequest(originalSearchRequest, indices, clusterAlias, absoluteStartMillis, finalReduce); } - protected SearchRequest( + private SearchRequest( SearchRequest searchRequest, String[] indices, String localClusterAlias, diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java index 013930113ec7d..47be439a97fc4 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestViewAction.java @@ -14,6 +14,7 @@ import org.opensearch.action.admin.indices.view.ListViewNamesAction; import org.opensearch.action.admin.indices.view.SearchViewAction; import org.opensearch.action.admin.indices.view.UpdateViewAction; +import org.opensearch.action.search.SearchRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.common.ValidationException; import org.opensearch.common.annotation.ExperimentalApi; @@ -212,7 +213,7 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final String viewId = request.param(VIEW_NAME); - final SearchViewAction.Request viewSearchRequest = new SearchViewAction.Request(viewId); + final SearchViewAction.Request viewSearchRequest = new SearchViewAction.Request(viewId, new SearchRequest()); final IntConsumer setSize = size -> viewSearchRequest.source().size(size); request.withContentOrSourceParamParserOrNull( diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java index c6d2fcc19563e..d68ad3f38b5e8 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java @@ -42,7 +42,7 @@ public void testValidateRequest() throws IOException { } public void testValidateRequestWithoutName() { - final SearchViewAction.Request request = new SearchViewAction.Request((String) null); + final SearchViewAction.Request request = new SearchViewAction.Request((String) null, new SearchRequest()); final ActionRequestValidationException e = request.validate(); MatcherAssert.assertThat(e.validationErrors().size(), equalTo(1)); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java index 076bcdf4e1eec..68e571b4203c1 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -10,6 +10,7 @@ import org.opensearch.ResourceNotFoundException; import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchRequest; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -176,7 +177,7 @@ public void listViewNames_noViews() { } public void searchView() { - final var request = spy(new SearchViewAction.Request(randomAlphaOfLength(8))); + final var request = spy(new SearchViewAction.Request(randomAlphaOfLength(8), new SearchRequest())); final var listener = mock(ActionListener.class); setGetViewOrThrowExceptionToReturnTypicalView(); From eae951179f767108d33c87efdc6b98c747a8a1e7 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 22:59:46 +0000 Subject: [PATCH 44/55] Remove unused codepath Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewTestBase.java | 2 +- .../action/admin/indices/view/SearchViewAction.java | 12 ------------ .../admin/indices/view/SearchViewRequestTests.java | 4 ++-- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java index 2d039f63f3dc2..f457a40975ac2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java @@ -57,7 +57,7 @@ protected List listViewNames() { } protected SearchResponse searchView(final String viewName) throws Exception { - final SearchViewAction.Request request = SearchViewAction.createRequestWith(viewName, new SearchRequest()); + final SearchViewAction.Request request = new SearchViewAction.Request(viewName, new SearchRequest()); final SearchResponse response = client().searchView(request).actionGet(); return response; } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java index 953be210b0825..1e20221242f06 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/SearchViewAction.java @@ -16,10 +16,8 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.inject.Inject; -import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.Strings; -import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.tasks.Task; @@ -42,16 +40,6 @@ private SearchViewAction() { super(NAME, SearchResponse::new); } - /** Given a search request, creates a ViewSearchRequest */ - public static Request createRequestWith(final String view, final SearchRequest searchRequest) throws IOException { - final BytesStreamOutput savedSearchRequest = new BytesStreamOutput(); - searchRequest.writeTo(savedSearchRequest); - savedSearchRequest.writeString(view); - - final BytesStreamInput input = new BytesStreamInput(savedSearchRequest.bytes().toBytesRef().bytes); - return new Request(input); - } - /** * Wraps the functionality of search requests and tailors for what is available * when searching through views diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java index d68ad3f38b5e8..d49c0c1a8f2bd 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/SearchViewRequestTests.java @@ -30,14 +30,14 @@ protected Writeable.Reader instanceReader() { @Override protected SearchViewAction.Request createTestInstance() { try { - return SearchViewAction.createRequestWith(randomAlphaOfLength(8), new SearchRequest()); + return new SearchViewAction.Request(randomAlphaOfLength(8), new SearchRequest()); } catch (final Exception e) { throw new RuntimeException(e); } } public void testValidateRequest() throws IOException { - final SearchViewAction.Request request = SearchViewAction.createRequestWith("my-view", new SearchRequest()); + final SearchViewAction.Request request = new SearchViewAction.Request("my-view", new SearchRequest()); MatcherAssert.assertThat(request.validate(), nullValue()); } From fcacb013ac5567039c5646bbdb1ea189087b3622 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Tue, 13 Feb 2024 19:59:13 -0600 Subject: [PATCH 45/55] Update exception serialization tests Signed-off-by: Peter Nied --- .../test/java/org/opensearch/ExceptionSerializationTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index 2f47bfb4df70a..d7026159d9ec0 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -40,6 +40,8 @@ import org.opensearch.action.OriginalIndices; import org.opensearch.action.RoutingMissingException; import org.opensearch.action.TimestampParsingException; +import org.opensearch.action.admin.indices.view.ViewAlreadyExistsException; +import org.opensearch.action.admin.indices.view.ViewNotFoundException; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.replication.ReplicationOperation; @@ -892,6 +894,8 @@ public void testIds() { ids.put(169, NodeWeighedAwayException.class); ids.put(170, SearchPipelineProcessingException.class); ids.put(171, CryptoRegistryException.class); + ids.put(172, ViewNotFoundException.class); + ids.put(173, ViewAlreadyExistsException.class); ids.put(10001, IndexCreateBlockException.class); Map, Integer> reverse = new HashMap<>(); From db8f0495f3cd1616e6adb1b7610ef29501e78d43 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 00:03:03 +0000 Subject: [PATCH 46/55] PR feedback Signed-off-by: Peter Nied --- CHANGELOG.md | 3 +- .../admin/indices/view/CreateViewAction.java | 39 +++++++++++++++- .../indices/view/CreateViewRequestTests.java | 44 +++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd1fe2f76bf2..8b7e008d8b724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625)) - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887)) - [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028)) +- Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957)) + ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 @@ -96,7 +98,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add community_id ingest processor ([#12121](https://github.com/opensearch-project/OpenSearch/pull/12121)) - Introduce query level setting `index.query.max_nested_depth` limiting nested queries ([#3268](https://github.com/opensearch-project/OpenSearch/issues/3268) - Add toString methods to MultiSearchRequest, MultiGetRequest and CreateIndexRequest ([#12163](https://github.com/opensearch-project/OpenSearch/pull/12163)) -- Views ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957)) ### Dependencies - Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288)) diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index 546698026a9de..b552b749f7278 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -20,6 +20,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.metadata.View; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.ValidationException; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.inject.Inject; import org.opensearch.core.action.ActionListener; @@ -37,11 +38,17 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; /** Action to create a view */ @ExperimentalApi public class CreateViewAction extends ActionType { + private static final int MAX_NAME_LENGTH = 64; + private static final int MAX_DESCRIPTION_LENGTH = 256; + private static final int MAX_TARGET_COUNT = 25; + private static final int MAX_TARGET_INDEX_PATTERN_LENGTH = 64; + public static final CreateViewAction INSTANCE = new CreateViewAction(); public static final String NAME = "cluster:admin/views/create"; @@ -102,11 +109,35 @@ public ActionRequestValidationException validate() { if (Strings.isNullOrEmpty(name)) { validationException = ValidateActions.addValidationError("name cannot be empty or null", validationException); } + if (name != null && name.length() > MAX_NAME_LENGTH) { + validationException = ValidateActions.addValidationError( + "name must be less than " + MAX_NAME_LENGTH + " characters in length", + validationException + ); + } + if (description != null && description.length() > MAX_DESCRIPTION_LENGTH) { + validationException = ValidateActions.addValidationError( + "description must be less than " + MAX_DESCRIPTION_LENGTH + " characters in length", + validationException + ); + } if (CollectionUtils.isEmpty(targets)) { validationException = ValidateActions.addValidationError("targets cannot be empty", validationException); } else { + System.out.println("targets.size()" + targets.size()); + if (targets.size() > MAX_TARGET_COUNT) { + validationException = ValidateActions.addValidationError( + "view cannot have more than " + MAX_TARGET_COUNT + " targets", + validationException + ); + } for (final Target target : targets) { - validationException = target.validate(); + final var validationMessages = Optional.ofNullable(target.validate()) + .map(ValidationException::validationErrors) + .orElse(List.of()); + for (final String validationMessage : validationMessages) { + validationException = ValidateActions.addValidationError(validationMessage, validationException); + } } } @@ -162,6 +193,12 @@ public ActionRequestValidationException validate() { if (Strings.isNullOrEmpty(indexPattern)) { validationException = ValidateActions.addValidationError("index pattern cannot be empty or null", validationException); } + if (indexPattern != null && indexPattern.length() > MAX_TARGET_INDEX_PATTERN_LENGTH) { + validationException = ValidateActions.addValidationError( + "target index pattern must be less than " + MAX_TARGET_INDEX_PATTERN_LENGTH + " characters in length", + validationException + ); + } return validationException; } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java index edb9f398debbe..e2211bb120366 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/CreateViewRequestTests.java @@ -51,4 +51,48 @@ public void testValidateRequestWithoutName() { MatcherAssert.assertThat(e.validationErrors(), contains("name cannot be empty or null", "targets cannot be empty")); } + public void testSizeThresholds() { + final String validName = randomAlphaOfLength(8); + final String validDescription = randomAlphaOfLength(20); + final int validTargetLength = randomIntBetween(1, 5); + final String validIndexPattern = randomAlphaOfLength(8); + + final CreateViewAction.Request requestNameTooBig = new CreateViewAction.Request( + randomAlphaOfLength(65), + validDescription, + randomList(1, validTargetLength, () -> new CreateViewAction.Request.Target(validIndexPattern)) + ); + MatcherAssert.assertThat( + requestNameTooBig.validate().validationErrors(), + contains("name must be less than 64 characters in length") + ); + + final CreateViewAction.Request requestDescriptionTooBig = new CreateViewAction.Request( + validName, + randomAlphaOfLength(257), + randomList(1, validTargetLength, () -> new CreateViewAction.Request.Target(validIndexPattern)) + ); + MatcherAssert.assertThat( + requestDescriptionTooBig.validate().validationErrors(), + contains("description must be less than 256 characters in length") + ); + + final CreateViewAction.Request requestTargetsSize = new CreateViewAction.Request( + validName, + validDescription, + randomList(26, 26, () -> new CreateViewAction.Request.Target(validIndexPattern)) + ); + MatcherAssert.assertThat(requestTargetsSize.validate().validationErrors(), contains("view cannot have more than 25 targets")); + + final CreateViewAction.Request requestTargetsIndexPatternSize = new CreateViewAction.Request( + validName, + validDescription, + randomList(1, 1, () -> new CreateViewAction.Request.Target(randomAlphaOfLength(65))) + ); + MatcherAssert.assertThat( + requestTargetsIndexPatternSize.validate().validationErrors(), + contains("target index pattern must be less than 64 characters in length") + ); + } + } From 49367369b95a8d268a5f5fe7cff2d53624c3f30e Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 00:09:06 +0000 Subject: [PATCH 47/55] Fix error in RemoteSegmentStoreDirectory when debug logging is enabled Signed-off-by: Peter Nied --- .../store/RemoteSegmentStoreDirectory.java | 8 +++--- .../opensearch/test/OpenSearchTestCase.java | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index dab99fd25b192..bfab9f8c18aa2 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -45,6 +45,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -745,9 +746,8 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException return; } - List metadataFilesEligibleToDelete = sortedMetadataFileList.subList( - lastNMetadataFilesToKeep, - sortedMetadataFileList.size() + List metadataFilesEligibleToDelete = new ArrayList<>( + sortedMetadataFileList.subList(lastNMetadataFilesToKeep, sortedMetadataFileList.size()) ); Set allLockFiles; try { @@ -764,7 +764,7 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException logger.debug( "metadataFilesEligibleToDelete={} metadataFilesToBeDeleted={}", metadataFilesEligibleToDelete, - metadataFilesEligibleToDelete + metadataFilesToBeDeleted ); Map activeSegmentFilesMetadataMap = new HashMap<>(); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 96bffcf2d3692..aac3fca9e1e16 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -145,6 +145,8 @@ import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; import java.math.BigInteger; import java.net.InetAddress; import java.net.UnknownHostException; @@ -170,6 +172,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.IntFunction; import java.util.function.Predicate; import java.util.function.Supplier; @@ -639,7 +642,32 @@ protected static void checkStaticState(boolean afterClass) throws Exception { try { // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the // StatusData instances to Strings as otherwise their toString output is useless + + final Function statusToString = (statusData) -> { + try (final StringWriter sw = new StringWriter(); final PrintWriter pw = new PrintWriter(sw)) { + + pw.print(statusData.getLevel()); + pw.print(":"); + pw.print(statusData.getMessage().getFormattedMessage()); + + if (statusData.getStackTraceElement() != null) { + final var messageSource = statusData.getStackTraceElement(); + pw.println("Source:"); + pw.println(messageSource.getFileName() + "@" + messageSource.getLineNumber()); + } + + if (statusData.getThrowable() != null) { + pw.println("Throwable:"); + statusData.getThrowable().printStackTrace(pw); + } + return sw.toString(); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + }; + assertThat( + statusData.stream().map(statusToString::apply).collect(Collectors.joining("\r\n")), statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()), empty() ); From 8c39071a1708fe8a1b5c2c5b5b6622341355384b Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 00:10:57 +0000 Subject: [PATCH 48/55] Add changelog entry Signed-off-by: Peter Nied --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82ef8d647c819..b095da6673d89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update supported version for must_exist parameter in update aliases API ([#11872](https://github.com/opensearch-project/OpenSearch/pull/11872)) - [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035)) - Fix Span operation names generated from RestActions ([#12005](https://github.com/opensearch-project/OpenSearch/pull/12005)) +- Fix error in RemoteSegmentStoreDirectory when debug logging is enabled ([#12328](https://github.com/opensearch-project/OpenSearch/pull/12328)) ### Security From 7259789af70eb997a104ccdfdb9d488930fe82e4 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 10:05:16 -0600 Subject: [PATCH 49/55] Use sorted set to store targets for views Signed-off-by: Peter Nied --- .../action/admin/indices/view/ViewIT.java | 14 +++++++++ .../admin/indices/view/ViewTestBase.java | 6 +++- .../admin/indices/view/CreateViewAction.java | 1 - .../admin/indices/view/ViewService.java | 11 +++++-- .../org/opensearch/cluster/metadata/View.java | 30 +++++++++++++------ .../admin/indices/view/ViewServiceTest.java | 3 +- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index e9de79109e641..e36115d0c65ad 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -41,6 +41,20 @@ public void testCreateView() throws Exception { MatcherAssert.assertThat(ex.getMessage(), is("View [" + viewName + "] already exists")); } + public void testCreateViewTargetsSet() throws Exception { + final String viewName = randomAlphaOfLength(8); + final String indexPattern = "a" + randomAlphaOfLength(8); + final String indexPattern2 = "b" + randomAlphaOfLength(8); + final List targetPatterns = List.of(indexPattern2, indexPattern, indexPattern); + + logger.info("Testing createView with targets that will be reordered and deduplicated"); + final View view = createView(viewName, targetPatterns).getView(); + MatcherAssert.assertThat(view.getName(), is(viewName)); + MatcherAssert.assertThat(view.getTargets().size(), is(2)); + MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern)); + MatcherAssert.assertThat(view.getTargets().last().getIndexPattern(), is(indexPattern2)); + } + public void testGetView() throws Exception { final String viewName = randomAlphaOfLength(8); createView(viewName, randomAlphaOfLength(8)); diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java index f457a40975ac2..7f45e08e1cf74 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java @@ -34,10 +34,14 @@ protected int createIndexWithDocs(final String indexName) throws Exception { } protected GetViewAction.Response createView(final String name, final String indexPattern) throws Exception { + return createView(name, List.of(indexPattern)); + } + + protected GetViewAction.Response createView(final String name, final List targets) throws Exception { final CreateViewAction.Request request = new CreateViewAction.Request( name, null, - List.of(new CreateViewAction.Request.Target(indexPattern)) + targets.stream().map(CreateViewAction.Request.Target::new).collect(Collectors.toList()) ); return client().admin().indices().createView(request).actionGet(); } diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java index b552b749f7278..9faf25ce10732 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/CreateViewAction.java @@ -124,7 +124,6 @@ public ActionRequestValidationException validate() { if (CollectionUtils.isEmpty(targets)) { validationException = ValidateActions.addValidationError("targets cannot be empty", validationException); } else { - System.out.println("targets.size()" + targets.size()); if (targets.size() > MAX_TARGET_COUNT) { validationException = ValidateActions.addValidationError( "view cannot have more than " + MAX_TARGET_COUNT + " targets", diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 0fd5b7c12eeec..294f88decba1f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.TreeSet; import java.util.function.LongSupplier; import java.util.stream.Collectors; @@ -51,7 +52,7 @@ public void createView(final CreateViewAction.Request request, final ActionListe .stream() .map(target -> new View.Target(target.getIndexPattern())) .collect(Collectors.toList()); - final View view = new View(request.getName(), request.getDescription(), currentTime, currentTime, targets); + final View view = new View(request.getName(), request.getDescription(), currentTime, currentTime, new TreeSet<>(targets)); createOrUpdateView(Operation.CreateView, view, listener); } @@ -64,7 +65,13 @@ public void updateView(final CreateViewAction.Request request, final ActionListe .stream() .map(target -> new View.Target(target.getIndexPattern())) .collect(Collectors.toList()); - final View updatedView = new View(request.getName(), request.getDescription(), originalView.getCreatedAt(), currentTime, targets); + final View updatedView = new View( + request.getName(), + request.getDescription(), + originalView.getCreatedAt(), + currentTime, + new TreeSet<>(targets) + ); createOrUpdateView(Operation.UpdateView, updatedView, listener); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/View.java b/server/src/main/java/org/opensearch/cluster/metadata/View.java index 09a2d09e7f7b5..1b1639bbca945 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/View.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/View.java @@ -23,6 +23,10 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.Collectors; /** View of data in OpenSearch indices */ @ExperimentalApi @@ -32,18 +36,18 @@ public class View extends AbstractDiffable implements ToXContentObject { private final String description; private final long createdAt; private final long modifiedAt; - private final List targets; + private final SortedSet targets; - public View(final String name, final String description, final Long createdAt, final Long modifiedAt, final List targets) { + public View(final String name, final String description, final Long createdAt, final Long modifiedAt, final Set targets) { this.name = Objects.requireNonNull(name, "Name must be provided"); this.description = description; this.createdAt = createdAt != null ? createdAt : -1; this.modifiedAt = modifiedAt != null ? modifiedAt : -1; - this.targets = Objects.requireNonNull(targets, "Targets are required on a view"); + this.targets = new TreeSet<>(Objects.requireNonNull(targets, "Targets are required on a view")); } public View(final StreamInput in) throws IOException { - this(in.readString(), in.readOptionalString(), in.readZLong(), in.readZLong(), in.readList(Target::new)); + this(in.readString(), in.readOptionalString(), in.readZLong(), in.readZLong(), new TreeSet<>(in.readList(Target::new))); } public String getName() { @@ -62,8 +66,8 @@ public long getModifiedAt() { return modifiedAt; } - public List getTargets() { - return targets; + public SortedSet getTargets() { + return new TreeSet<>(targets); } public static Diff readDiffFrom(final StreamInput in) throws IOException { @@ -89,7 +93,7 @@ public int hashCode() { /** The source of data used to project the view */ @ExperimentalApi - public static class Target implements Writeable, ToXContentObject { + public static class Target implements Writeable, ToXContentObject, Comparable { private final String indexPattern; @@ -144,6 +148,14 @@ public static Target fromXContent(final XContentParser parser) throws IOExceptio public void writeTo(final StreamOutput out) throws IOException { out.writeString(indexPattern); } + + @Override + public int compareTo(final Target o) { + if (this == o) return 0; + + final Target other = (Target) o; + return this.indexPattern.compareTo(other.indexPattern); + } } public static final ParseField NAME_FIELD = new ParseField("name"); @@ -155,7 +167,7 @@ public void writeTo(final StreamOutput out) throws IOException { @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "view", - args -> new View((String) args[0], (String) args[1], (Long) args[2], (Long) args[3], (List) args[4]) + args -> new View((String) args[0], (String) args[1], (Long) args[2], (Long) args[3], new TreeSet<>((List) args[4])) ); static { @@ -188,6 +200,6 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeOptionalString(description); out.writeZLong(createdAt); out.writeZLong(modifiedAt); - out.writeList(targets); + out.writeList(targets.stream().collect(Collectors.toList())); } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java index 68e571b4203c1..91813e1336cf2 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/ViewServiceTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongSupplier; @@ -50,7 +51,7 @@ public class ViewServiceTest { "description " + randomAlphaOfLength(20), -1L, -1L, - List.of(typicalTarget) + Set.of(typicalTarget) ); private ClusterService clusterService; From fa913f7d61ecdf623d480b62d20d69977ae50f6f Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 17:12:25 +0000 Subject: [PATCH 50/55] Test compile checks Signed-off-by: Peter Nied --- .../indices/view/GetViewResponseTests.java | 4 ++- .../cluster/metadata/ViewTests.java | 31 ++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java b/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java index 4dd075597a804..44dfbe5f1d781 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/view/GetViewResponseTests.java @@ -12,6 +12,8 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.test.AbstractWireSerializingTestCase; +import java.util.TreeSet; + public class GetViewResponseTests extends AbstractWireSerializingTestCase { @Override @@ -27,7 +29,7 @@ protected GetViewAction.Response createTestInstance() { randomAlphaOfLength(8), randomLong(), randomLong(), - randomList(5, () -> new View.Target(randomAlphaOfLength(8))) + new TreeSet<>(randomList(5, () -> new View.Target(randomAlphaOfLength(8)))) ) ); } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java index 145d3277bbf14..ad39e2b103087 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/ViewTests.java @@ -12,23 +12,24 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.test.AbstractSerializingTestCase; +import org.hamcrest.MatcherAssert; import java.io.IOException; -import java.util.List; +import java.util.Set; +import java.util.TreeSet; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; public class ViewTests extends AbstractSerializingTestCase { - private static List randomTargets() { - int numTargets = randomIntBetween(0, 128); - return randomList(numTargets, () -> new View.Target(randomAlphaOfLength(8))); + private static Set randomTargets() { + int numTargets = randomIntBetween(1, 25); + return new TreeSet<>(randomList(1, numTargets, () -> new View.Target(randomAlphaOfLength(8)))); } private static View randomInstance() { - final List targets = randomTargets(); + final Set targets = randomTargets(); final String viewName = randomAlphaOfLength(10); final String description = randomAlphaOfLength(100); return new View(viewName, description, Math.abs(randomLong()), Math.abs(randomLong()), targets); @@ -52,28 +53,28 @@ protected View createTestInstance() { public void testNullName() { final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View(null, null, null, null, null)); - assertThat(npe.getMessage(), equalTo("Name must be provided")); + MatcherAssert.assertThat(npe.getMessage(), equalTo("Name must be provided")); } public void testNullTargets() { final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View("name", null, null, null, null)); - assertThat(npe.getMessage(), equalTo("Targets are required on a view")); + MatcherAssert.assertThat(npe.getMessage(), equalTo("Targets are required on a view")); } public void testNullTargetIndexPattern() { final NullPointerException npe = assertThrows(NullPointerException.class, () -> new View.Target((String) null)); - assertThat(npe.getMessage(), equalTo("IndexPattern is required")); + MatcherAssert.assertThat(npe.getMessage(), equalTo("IndexPattern is required")); } public void testDefaultValues() { - final View view = new View("myName", null, null, null, List.of()); + final View view = new View("myName", null, null, null, Set.of()); - assertThat(view.getName(), equalTo("myName")); - assertThat(view.getDescription(), equalTo(null)); - assertThat(view.getCreatedAt(), equalTo(-1L)); - assertThat(view.getModifiedAt(), equalTo(-1L)); - assertThat(view.getTargets(), empty()); + MatcherAssert.assertThat(view.getName(), equalTo("myName")); + MatcherAssert.assertThat(view.getDescription(), equalTo(null)); + MatcherAssert.assertThat(view.getCreatedAt(), equalTo(-1L)); + MatcherAssert.assertThat(view.getModifiedAt(), equalTo(-1L)); + MatcherAssert.assertThat(view.getTargets(), empty()); } } From a9777be93e0834c104abede45d91705f68a99284 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 19:12:23 +0000 Subject: [PATCH 51/55] Fix compile issues related to switch to set Signed-off-by: Peter Nied --- .../java/org/opensearch/action/admin/indices/view/ViewIT.java | 4 ++-- .../opensearch/action/admin/indices/view/ViewTestBase.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java index e36115d0c65ad..85c70e098652c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java @@ -34,7 +34,7 @@ public void testCreateView() throws Exception { final View view = createView(viewName, indexPattern).getView(); MatcherAssert.assertThat(view.getName(), is(viewName)); MatcherAssert.assertThat(view.getTargets().size(), is(1)); - MatcherAssert.assertThat(view.getTargets().get(0).getIndexPattern(), is(indexPattern)); + MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern)); logger.info("Testing createView with existing view name"); final Exception ex = assertThrows(ViewAlreadyExistsException.class, () -> createView(viewName, randomAlphaOfLength(8))); @@ -96,7 +96,7 @@ public void testUpdateView() throws Exception { MatcherAssert.assertThat(updatedView, not(is(originalView))); MatcherAssert.assertThat(updatedView.getDescription(), is(newDescription)); MatcherAssert.assertThat(updatedView.getTargets(), hasSize(1)); - MatcherAssert.assertThat(updatedView.getTargets().get(0).getIndexPattern(), is(newIndexPattern)); + MatcherAssert.assertThat(updatedView.getTargets().first().getIndexPattern(), is(newIndexPattern)); logger.info("Testing updateView with non-existent view"); final String nonExistentView = "non-existent-" + randomAlphaOfLength(8); diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java index 7f45e08e1cf74..a44ba0cf7c717 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java @@ -14,6 +14,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import java.util.List; +import java.util.stream.Collectors; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; From 879638166dada1f1e1de1b7a62117d690be8d6e8 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 19:28:56 +0000 Subject: [PATCH 52/55] Add test to validate debug log statement doesn't have errors Signed-off-by: Peter Nied --- .../RemoteSegmentStoreDirectoryTests.java | 62 +++++++++++++++---- .../org/opensearch/test/MockLogAppender.java | 58 +++++++++++++++++ 2 files changed, 107 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 7944ee681f5fc..09bbad04408f5 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.store; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.CodecUtil; @@ -41,6 +42,8 @@ import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.MockLogAppender; +import org.opensearch.test.junit.annotations.TestLogging; import org.opensearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; @@ -58,6 +61,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.mockito.Mockito; @@ -971,21 +976,52 @@ public void testDeleteStaleCommitsWithinThreshold() throws Exception { verify(remoteMetadataDirectory, times(0)).openInput(any(String.class), eq(IOContext.DEFAULT)); } + @TestLogging(value = "_root:debug", reason = "Validate logging output") public void testDeleteStaleCommitsActualDelete() throws Exception { - Map> metadataFilenameContentMapping = populateMetadata(); - remoteSegmentStoreDirectory.init(); - - // popluateMetadata() adds stub to return 3 metadata files - // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted - remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); - - for (String metadata : metadataFilenameContentMapping.get(metadataFilename3).values()) { - String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; - verify(remoteDataDirectory).deleteFile(uploadedFilename); + try (final MockLogAppender appender = MockLogAppender.createForLoggers(LogManager.getRootLogger())) { + appender.addExpectation( + new MockLogAppender.PatternSeenWithLoggerPrefixExpectation( + "Metadata files to delete message", + "org.opensearch.index.store.RemoteSegmentStoreDirectory", + Level.DEBUG, + "metadataFilesEligibleToDelete=\\[" + metadataFilename3 + "\\] metadataFilesToBeDeleted=\\[" + metadataFilename3 + "\\]" + ) + ); + + final Map> metadataFilenameContentMapping = populateMetadata(); + final List filesToBeDeleted = metadataFilenameContentMapping.get(metadataFilename3) + .values() + .stream() + .map(metadata -> metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]) + .collect(Collectors.toList()); + + final String allFilesPattern = filesToBeDeleted.stream() + .map(Pattern::quote) // Make sure filenames are matched literally + .map(file -> "(?=.*?" + file + ")") // Allow for matches in any order + .collect(Collectors.joining("", "deletedSegmentFiles=\\[", ".*\\]")); + + appender.addExpectation( + new MockLogAppender.PatternSeenWithLoggerPrefixExpectation( + "Deleted file segments message", + "org.opensearch.index.store.RemoteSegmentStoreDirectory", + Level.DEBUG, + allFilesPattern + ) + ); + + remoteSegmentStoreDirectory.init(); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); + + for (final String file : filesToBeDeleted) { + verify(remoteDataDirectory).deleteFile(file); + } + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); + verify(remoteMetadataDirectory).deleteFile(metadataFilename3); + appender.assertAllExpectationsMatched(); } - ; - assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); - verify(remoteMetadataDirectory).deleteFile(metadataFilename3); } public void testDeleteStaleCommitsActualDeleteWithLocks() throws Exception { diff --git a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java index 59eda7a665d4c..d5d7c6d301032 100644 --- a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java +++ b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java @@ -40,14 +40,17 @@ import org.opensearch.common.logging.Loggers; import org.opensearch.common.regex.Regex; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.fail; /** * Test appender that can be used to verify that certain events were logged correctly @@ -259,6 +262,61 @@ public void assertMatched() { } + /** + * Used for cases when the logger is dynamically named such as to include an index name or shard id + * + * Best used in conjunction with the root logger: + * @TestLogging(value = "_root:debug", reason = "Validate logging output") + * + * */ + public static class PatternSeenWithLoggerPrefixExpectation implements LoggingExpectation { + private final String expectationName; + private final String loggerPrefix; + private final Level level; + private final String messageMatchingRegex; + + private final List loggerMatches = new ArrayList<>(); + private final AtomicBoolean eventSeen = new AtomicBoolean(false); + + public PatternSeenWithLoggerPrefixExpectation( + final String expectationName, + final String loggerPrefix, + final Level level, + final String messageMatchingRegex + ) { + this.expectationName = expectationName; + this.loggerPrefix = loggerPrefix; + this.level = level; + this.messageMatchingRegex = messageMatchingRegex; + } + + @Override + public void match(final LogEvent event) { + if (event.getLevel() == level && event.getLoggerName().startsWith(loggerPrefix)) { + final String formattedMessage = event.getMessage().getFormattedMessage(); + loggerMatches.add(formattedMessage); + if (formattedMessage.matches(messageMatchingRegex)) { + eventSeen.set(true); + } + } + } + + @Override + public void assertMatched() { + if (!eventSeen.get()) { + final StringBuilder failureMessage = new StringBuilder(); + failureMessage.append( + String.format("%s was not seen, found %d messages matching the logger.", expectationName, loggerMatches.size()) + ); + failureMessage.append("\r\nMessage matching regex: " + messageMatchingRegex); + if (!loggerMatches.isEmpty()) { + failureMessage.append("\r\nMessage details:\r\n" + String.join("\r\n", loggerMatches)); + } + fail(failureMessage.toString()); + } + } + } + private static String getLoggerName(String name) { if (name.startsWith("org.opensearch.")) { name = name.substring("org.opensearch.".length()); From 0dd44f76671f0d0bbf5580cd55231f75cd49c6af Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 19:28:56 +0000 Subject: [PATCH 53/55] Add test to validate debug log statement doesn't have errors Signed-off-by: Peter Nied --- .../RemoteSegmentStoreDirectoryTests.java | 62 +++++++++++++++---- .../org/opensearch/test/MockLogAppender.java | 58 +++++++++++++++++ 2 files changed, 107 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 7944ee681f5fc..09bbad04408f5 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.store; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.CodecUtil; @@ -41,6 +42,8 @@ import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.MockLogAppender; +import org.opensearch.test.junit.annotations.TestLogging; import org.opensearch.threadpool.ThreadPool; import org.junit.After; import org.junit.Before; @@ -58,6 +61,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.mockito.Mockito; @@ -971,21 +976,52 @@ public void testDeleteStaleCommitsWithinThreshold() throws Exception { verify(remoteMetadataDirectory, times(0)).openInput(any(String.class), eq(IOContext.DEFAULT)); } + @TestLogging(value = "_root:debug", reason = "Validate logging output") public void testDeleteStaleCommitsActualDelete() throws Exception { - Map> metadataFilenameContentMapping = populateMetadata(); - remoteSegmentStoreDirectory.init(); - - // popluateMetadata() adds stub to return 3 metadata files - // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted - remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); - - for (String metadata : metadataFilenameContentMapping.get(metadataFilename3).values()) { - String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; - verify(remoteDataDirectory).deleteFile(uploadedFilename); + try (final MockLogAppender appender = MockLogAppender.createForLoggers(LogManager.getRootLogger())) { + appender.addExpectation( + new MockLogAppender.PatternSeenWithLoggerPrefixExpectation( + "Metadata files to delete message", + "org.opensearch.index.store.RemoteSegmentStoreDirectory", + Level.DEBUG, + "metadataFilesEligibleToDelete=\\[" + metadataFilename3 + "\\] metadataFilesToBeDeleted=\\[" + metadataFilename3 + "\\]" + ) + ); + + final Map> metadataFilenameContentMapping = populateMetadata(); + final List filesToBeDeleted = metadataFilenameContentMapping.get(metadataFilename3) + .values() + .stream() + .map(metadata -> metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]) + .collect(Collectors.toList()); + + final String allFilesPattern = filesToBeDeleted.stream() + .map(Pattern::quote) // Make sure filenames are matched literally + .map(file -> "(?=.*?" + file + ")") // Allow for matches in any order + .collect(Collectors.joining("", "deletedSegmentFiles=\\[", ".*\\]")); + + appender.addExpectation( + new MockLogAppender.PatternSeenWithLoggerPrefixExpectation( + "Deleted file segments message", + "org.opensearch.index.store.RemoteSegmentStoreDirectory", + Level.DEBUG, + allFilesPattern + ) + ); + + remoteSegmentStoreDirectory.init(); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(2); + + for (final String file : filesToBeDeleted) { + verify(remoteDataDirectory).deleteFile(file); + } + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); + verify(remoteMetadataDirectory).deleteFile(metadataFilename3); + appender.assertAllExpectationsMatched(); } - ; - assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); - verify(remoteMetadataDirectory).deleteFile(metadataFilename3); } public void testDeleteStaleCommitsActualDeleteWithLocks() throws Exception { diff --git a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java index 59eda7a665d4c..d5d7c6d301032 100644 --- a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java +++ b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java @@ -40,14 +40,17 @@ import org.opensearch.common.logging.Loggers; import org.opensearch.common.regex.Regex; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.fail; /** * Test appender that can be used to verify that certain events were logged correctly @@ -259,6 +262,61 @@ public void assertMatched() { } + /** + * Used for cases when the logger is dynamically named such as to include an index name or shard id + * + * Best used in conjunction with the root logger: + * @TestLogging(value = "_root:debug", reason = "Validate logging output") + * + * */ + public static class PatternSeenWithLoggerPrefixExpectation implements LoggingExpectation { + private final String expectationName; + private final String loggerPrefix; + private final Level level; + private final String messageMatchingRegex; + + private final List loggerMatches = new ArrayList<>(); + private final AtomicBoolean eventSeen = new AtomicBoolean(false); + + public PatternSeenWithLoggerPrefixExpectation( + final String expectationName, + final String loggerPrefix, + final Level level, + final String messageMatchingRegex + ) { + this.expectationName = expectationName; + this.loggerPrefix = loggerPrefix; + this.level = level; + this.messageMatchingRegex = messageMatchingRegex; + } + + @Override + public void match(final LogEvent event) { + if (event.getLevel() == level && event.getLoggerName().startsWith(loggerPrefix)) { + final String formattedMessage = event.getMessage().getFormattedMessage(); + loggerMatches.add(formattedMessage); + if (formattedMessage.matches(messageMatchingRegex)) { + eventSeen.set(true); + } + } + } + + @Override + public void assertMatched() { + if (!eventSeen.get()) { + final StringBuilder failureMessage = new StringBuilder(); + failureMessage.append( + String.format("%s was not seen, found %d messages matching the logger.", expectationName, loggerMatches.size()) + ); + failureMessage.append("\r\nMessage matching regex: " + messageMatchingRegex); + if (!loggerMatches.isEmpty()) { + failureMessage.append("\r\nMessage details:\r\n" + String.join("\r\n", loggerMatches)); + } + fail(failureMessage.toString()); + } + } + } + private static String getLoggerName(String name) { if (name.startsWith("org.opensearch.")) { name = name.substring("org.opensearch.".length()); From d77f8122e2dbc5c9f3289bd7708fc48a51041602 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 20:45:26 +0000 Subject: [PATCH 54/55] Dial back the level of detail validated for the segments that are deleted Signed-off-by: Peter Nied --- .../store/RemoteSegmentStoreDirectoryTests.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 09bbad04408f5..e2ebb2e642bfe 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -61,7 +61,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.mockito.Mockito; @@ -995,20 +994,6 @@ public void testDeleteStaleCommitsActualDelete() throws Exception { .map(metadata -> metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]) .collect(Collectors.toList()); - final String allFilesPattern = filesToBeDeleted.stream() - .map(Pattern::quote) // Make sure filenames are matched literally - .map(file -> "(?=.*?" + file + ")") // Allow for matches in any order - .collect(Collectors.joining("", "deletedSegmentFiles=\\[", ".*\\]")); - - appender.addExpectation( - new MockLogAppender.PatternSeenWithLoggerPrefixExpectation( - "Deleted file segments message", - "org.opensearch.index.store.RemoteSegmentStoreDirectory", - Level.DEBUG, - allFilesPattern - ) - ); - remoteSegmentStoreDirectory.init(); // popluateMetadata() adds stub to return 3 metadata files From fc600acf327563dd25da57dbaaa320cc82dde4d0 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 15 Feb 2024 20:45:26 +0000 Subject: [PATCH 55/55] Dial back the level of detail validated for the segments that are deleted Signed-off-by: Peter Nied --- .../store/RemoteSegmentStoreDirectoryTests.java | 15 --------------- .../java/org/opensearch/test/MockLogAppender.java | 4 +--- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 09bbad04408f5..e2ebb2e642bfe 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -61,7 +61,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.mockito.Mockito; @@ -995,20 +994,6 @@ public void testDeleteStaleCommitsActualDelete() throws Exception { .map(metadata -> metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]) .collect(Collectors.toList()); - final String allFilesPattern = filesToBeDeleted.stream() - .map(Pattern::quote) // Make sure filenames are matched literally - .map(file -> "(?=.*?" + file + ")") // Allow for matches in any order - .collect(Collectors.joining("", "deletedSegmentFiles=\\[", ".*\\]")); - - appender.addExpectation( - new MockLogAppender.PatternSeenWithLoggerPrefixExpectation( - "Deleted file segments message", - "org.opensearch.index.store.RemoteSegmentStoreDirectory", - Level.DEBUG, - allFilesPattern - ) - ); - remoteSegmentStoreDirectory.init(); // popluateMetadata() adds stub to return 3 metadata files diff --git a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java index d5d7c6d301032..e52358030322c 100644 --- a/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java +++ b/test/framework/src/main/java/org/opensearch/test/MockLogAppender.java @@ -305,9 +305,7 @@ public void match(final LogEvent event) { public void assertMatched() { if (!eventSeen.get()) { final StringBuilder failureMessage = new StringBuilder(); - failureMessage.append( - String.format("%s was not seen, found %d messages matching the logger.", expectationName, loggerMatches.size()) - ); + failureMessage.append(expectationName + " was not seen, found " + loggerMatches.size() + " messages matching the logger."); failureMessage.append("\r\nMessage matching regex: " + messageMatchingRegex); if (!loggerMatches.isEmpty()) { failureMessage.append("\r\nMessage details:\r\n" + String.join("\r\n", loggerMatches));