Merge branch 'master' into deprecation-header-encoding

* master: Backport the size-based index rollver to v6.1.0 Add size-based condition to the index rollover API (elastic#27160) Remove the single argument Environment constructor (elastic#27235)
jasontedor · Nov 5, 2017 · fbdb73e · fbdb73e
2 parents b5009a1 + fd3fac9
commit fbdb73e
Show file tree

Hide file tree

Showing 40 changed files with 491 additions and 103 deletions.
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -19,8 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 
@@ -38,6 +40,9 @@ public abstract class Condition<T> implements NamedWriteable {
             new ParseField(MaxAgeCondition.NAME));
         PARSER.declareLong((conditions, value) ->
             conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME));
+        PARSER.declareString((conditions, s) ->
+                conditions.add(new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))),
+            new ParseField(MaxSizeCondition.NAME));
     }
 
     protected T value;
@@ -49,6 +54,14 @@ protected Condition(String name) {
 
     public abstract Result evaluate(Stats stats);
 
+    /**
+     * Checks if this condition is available in a specific version.
+     * This makes sure BWC when introducing a new condition which is not recognized by older versions.
+     */
+    boolean includedInVersion(Version version) {
+        return true;
+    }
+
     @Override
     public final String toString() {
         return "[" + name + ": " + value + "]";
@@ -60,10 +73,12 @@ public final String toString() {
     public static class Stats {
         public final long numDocs;
         public final long indexCreated;
+        public final ByteSizeValue indexSize;
 
-        public Stats(long numDocs, long indexCreated) {
+        public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
             this.numDocs = numDocs;
             this.indexCreated = indexCreated;
+            this.indexSize = indexSize;
         }
     }
 

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+
+import java.io.IOException;
+
+/**
+ * A size-based condition for an index size.
+ * Evaluates to <code>true</code> if the index size is at least {@link #value}.
+ */
+public class MaxSizeCondition extends Condition<ByteSizeValue> {
+    public static final String NAME = "max_size";
+
+    public MaxSizeCondition(ByteSizeValue value) {
+        super(NAME);
+        this.value = value;
+    }
+
+    public MaxSizeCondition(StreamInput in) throws IOException {
+        super(NAME);
+        this.value = new ByteSizeValue(in.readVLong(), ByteSizeUnit.BYTES);
+    }
+
+    @Override
+    public Result evaluate(Stats stats) {
+        return new Result(this, stats.indexSize.getBytes() >= value.getBytes());
+    }
+
+    @Override
+    boolean includedInVersion(Version version) {
+        return version.onOrAfter(Version.V_6_1_0);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(value.getBytes());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 
@@ -106,7 +107,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(dryRun);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
-            out.writeNamedWriteable(condition);
+            if (condition.includedInVersion(out.getVersion())) {
+                out.writeNamedWriteable(condition);
+            }
         }
         createIndexRequest.writeTo(out);
     }
@@ -155,6 +158,13 @@ public void addMaxIndexDocsCondition(long numDocs) {
         this.conditions.add(new MaxDocsCondition(numDocs));
     }
 
+    /**
+     * Adds a size-based condition to check if the index size is at least <code>size</code>.
+     */
+    public void addMaxIndexSizeCondition(ByteSizeValue size) {
+        this.conditions.add(new MaxSizeCondition(size));
+    }
+
     /**
      * Sets rollover index creation request to override index settings when
      * the rolled over index has to be created

diff --git a/...src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/...src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
 
@@ -52,6 +53,11 @@ public RolloverRequestBuilder addMaxIndexDocsCondition(long docs) {
         return this;
     }
 
+    public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){
+        this.request.addMaxIndexSizeCondition(size);
+        return this;
+    }
+
     public RolloverRequestBuilder dryRun(boolean dryRun) {
         this.request.dryRun(dryRun);
         return this;

diff --git a/...rc/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/...rc/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -43,6 +43,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -195,7 +196,8 @@ static String generateRolloverIndexName(String sourceIndexName, IndexNameExpress
     static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions,
                                                     final DocsStats docsStats, final IndexMetaData metaData) {
         final long numDocs = docsStats == null ? 0 : docsStats.getCount();
-        final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate());
+        final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes();
+        final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate(), new ByteSizeValue(indexSize));
         return conditions.stream()
             .map(condition -> condition.evaluate(stats))
             .collect(Collectors.toSet());

diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java
@@ -85,10 +85,6 @@ public class Environment {
     /** Path to the temporary file directory used by the JDK */
     private final Path tmpFile = PathUtils.get(System.getProperty("java.io.tmpdir"));
 
-    public Environment(Settings settings) {
-        this(settings, null);
-    }
-
     public Environment(final Settings settings, final Path configPath) {
         final Path homeFile;
         if (PATH_HOME_SETTING.exists(settings)) {

diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -22,6 +22,7 @@
 import org.elasticsearch.action.admin.indices.rollover.Condition;
 import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
 import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
+import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
 import org.elasticsearch.action.resync.TransportResyncReplicationAction;
 import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
 import org.elasticsearch.common.geo.ShapesAvailability;
@@ -79,6 +80,7 @@ public IndicesModule(List<MapperPlugin> mapperPlugins) {
     private void registerBuiltinWritables() {
         namedWritables.add(new Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new));
         namedWritables.add(new Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new));
+        namedWritables.add(new Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new));
     }
 
     public List<Entry> getNamedWriteables() {

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
@@ -28,6 +28,7 @@
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AbstractCharFilterFactory;
 import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
@@ -74,7 +75,7 @@ public void setUp() throws Exception {
                 .put("index.analysis.normalizer.my_normalizer.type", "custom")
                 .putList("index.analysis.normalizer.my_normalizer.filter", "lowercase").build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        environment = new Environment(settings);
+        environment = TestEnvironment.newEnvironment(settings);
         AnalysisPlugin plugin = new AnalysisPlugin() {
             class MockFactory extends AbstractTokenFilterFactory {
                 MockFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 
@@ -30,12 +32,12 @@ public void testMaxAge() throws Exception {
         final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1));
 
         long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis();
-        Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch));
+        Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxAgeCondition));
         assertThat(evaluate.matched, equalTo(true));
 
         long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis();
-        evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch));
+        evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxAgeCondition));
         assertThat(evaluate.matched, equalTo(false));
     }
@@ -44,13 +46,33 @@ public void testMaxDocs() throws Exception {
         final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
 
         long maxDocsMatch = randomIntBetween(100, 1000);
-        Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0));
+        Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxDocsCondition));
         assertThat(evaluate.matched, equalTo(true));
 
         long maxDocsNotMatch = randomIntBetween(0, 99);
-        evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch));
+        evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize()));
         assertThat(evaluate.condition, equalTo(maxDocsCondition));
         assertThat(evaluate.matched, equalTo(false));
     }
+
+    public void testMaxSize() throws Exception {
+        MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 20), ByteSizeUnit.MB));
+
+        Condition.Result result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
+            new ByteSizeValue(0, ByteSizeUnit.MB)));
+        assertThat(result.matched, equalTo(false));
+
+        result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
+            new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB)));
+        assertThat(result.matched, equalTo(false));
+
+        result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
+            new ByteSizeValue(randomIntBetween(20, 1000), ByteSizeUnit.MB)));
+        assertThat(result.matched, equalTo(true));
+    }
+
+    private ByteSizeValue randomByteSize() {
+        return new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES);
+    }
 }