Skip to content

Commit

Permalink
Merge branch 'master' into soft-delete-policy-off-by-one
Browse files Browse the repository at this point in the history 
* master:
  Remove _type term filters from cluster alert watches (elastic#38819)
  Adjust log and unmute testFailOverOnFollower (elastic#38762)
  Fix line separators in JSON logging tests (elastic#38771)
  Fix synchronization in LocalCheckpointTracker#contains (elastic#38755)
  muted test
  Remove TLSv1.2 pinning in ssl reload tests (elastic#38651)
  Don't fail init script if `/proc/.../max_map_count` absent (elastic#35933)
  Format Watcher.status.lastChecked and lastMetCondition (elastic#38626)
  SQL: Implement `::` cast operator (elastic#38774)
  Ignore failing test
  • Loading branch information
@jasontedor
jasontedor committed Feb 13, 2019
2 parents 7b564de + d09ed17 commit 6d23d8f
Show file tree
Hide file tree
Showing 33 changed files with 1,368 additions and 1,079 deletions.
2 changes: 2 additions & 0 deletions buildSrc/src/test/java/org/elasticsearch/gradle/BuildExamplePluginsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
import org.gradle.testkit.runner.GradleRunner;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;

Expand All @@ -38,6 +39,7 @@
import java.util.Objects;
import java.util.stream.Collectors;

@Ignore // https://github.com/elastic/elasticsearch/issues/38784
public class BuildExamplePluginsIT extends GradleIntegrationTestCase {

private static final List<File> EXAMPLE_PLUGINS = Collections.unmodifiableList(
Expand Down
2 changes: 1 addition & 1 deletion distribution/packages/src/deb/init.d/elasticsearch
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ case "$1" in
ulimit -l $MAX_LOCKED_MEMORY
fi

if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count -a "$MAX_MAP_COUNT" -gt $(cat /proc/sys/vm/max_map_count) ]; then
if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ] && [ "$MAX_MAP_COUNT" -gt $(cat /proc/sys/vm/max_map_count) ]; then
sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
fi

Expand Down
2 changes: 1 addition & 1 deletion distribution/packages/src/rpm/init.d/elasticsearch
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ start() {
if [ -n "$MAX_LOCKED_MEMORY" ]; then
ulimit -l $MAX_LOCKED_MEMORY
fi
if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count -a "$MAX_MAP_COUNT" -gt $(cat /proc/sys/vm/max_map_count) ]; then
if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ] && [ "$MAX_MAP_COUNT" -gt $(cat /proc/sys/vm/max_map_count) ]; then
sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
fi

Expand Down
14 changes: 13 additions & 1 deletion docs/reference/sql/functions/operators.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,21 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[multiply]
include-tagged::{sql-specs}/arithmetic.sql-spec[divide]
--------------------------------------------------

* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] or Reminder(`%`)
* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] or Remainder(`%`)

["source","sql",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{sql-specs}/arithmetic.sql-spec[mod]
--------------------------------------------------

[[sql-operators-cast]]
=== Cast Operators

* Cast (`::`)

`::` provides an alternative syntax to the <<sql-functions-type-conversion-cast>> function.

["source","sql",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{sql-specs}/docs.csv-spec[conversionStringToLongCastOperator]
--------------------------------------------------
37 changes: 19 additions & 18 deletions qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
* It has to be in a <code>org.elasticsearch.common.logging</code> package to use <code>PrefixLogger</code>
*/
public class JsonLoggerTests extends ESTestCase {
private static final String LINE_SEPARATOR = System.lineSeparator();

@BeforeClass
public static void initNodeName() {
Expand Down Expand Up @@ -109,15 +110,15 @@ public void testPrefixLoggerInJson() throws IOException {

public void testJsonInMessage() throws IOException {
final Logger testLogger = LogManager.getLogger("test");
String json = "{\n" +
" \"terms\" : {\n" +
" \"user\" : [\n" +
" \"u1\",\n" +
" \"u2\",\n" +
" \"u3\"\n" +
" ],\n" +
" \"boost\" : 1.0\n" +
" }\n" +
String json = "{" + LINE_SEPARATOR +
" \"terms\" : {" + LINE_SEPARATOR +
" \"user\" : [" + LINE_SEPARATOR +
" \"u1\"," + LINE_SEPARATOR +
" \"u2\"," + LINE_SEPARATOR +
" \"u3\"" + LINE_SEPARATOR +
" ]," + LINE_SEPARATOR +
" \"boost\" : 1.0" + LINE_SEPARATOR +
" }" + LINE_SEPARATOR +
"}";

testLogger.info(json);
Expand Down Expand Up @@ -151,15 +152,15 @@ public void testStacktrace() throws IOException {
public void testJsonInStacktraceMessageIsSplitted() throws IOException {
final Logger testLogger = LogManager.getLogger("test");

String json = "{\n" +
" \"terms\" : {\n" +
" \"user\" : [\n" +
" \"u1\",\n" +
" \"u2\",\n" +
" \"u3\"\n" +
" ],\n" +
" \"boost\" : 1.0\n" +
" }\n" +
String json = "{" + LINE_SEPARATOR +
" \"terms\" : {" + LINE_SEPARATOR +
" \"user\" : [" + LINE_SEPARATOR +
" \"u1\"," + LINE_SEPARATOR +
" \"u2\"," + LINE_SEPARATOR +
" \"u3\"" + LINE_SEPARATOR +
" ]," + LINE_SEPARATOR +
" \"boost\" : 1.0" + LINE_SEPARATOR +
" }" + LINE_SEPARATOR +
"}";
testLogger.error("error message " + json, new Exception(json));

Expand Down
6 changes: 3 additions & 3 deletions server/src/main/java/org/elasticsearch/index/seqno/LocalCheckpointTracker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,11 +157,11 @@ public boolean contains(final long seqNo) {
return true;
}
final long bitSetKey = getBitSetKey(seqNo);
final CountedBitSet bitSet;
final int bitSetOffset = seqNoToBitSetOffset(seqNo);
synchronized (this) {
bitSet = processedSeqNo.get(bitSetKey);
final CountedBitSet bitSet = processedSeqNo.get(bitSetKey);
return bitSet != null && bitSet.get(bitSetOffset);
}
return bitSet != null && bitSet.get(seqNoToBitSetOffset(seqNo));
}

/**
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -779,7 +779,8 @@ private Engine.IndexResult index(Engine engine, Engine.Index index) throws IOExc
try {
if (logger.isTraceEnabled()) {
// don't use index.source().utf8ToString() here source might not be valid UTF-8
logger.trace("index [{}][{}] (seq# [{}])", index.type(), index.id(), index.seqNo());
logger.trace("index [{}][{}] seq# [{}] allocation-id {}",
index.type(), index.id(), index.seqNo(), routingEntry().allocationId());
}
result = engine.index(index);
} catch (Exception e) {
Expand Down
23 changes: 12 additions & 11 deletions ...er/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@
import static org.hamcrest.Matchers.equalTo;

public class JsonThrowablePatternConverterTests extends ESTestCase {
JsonThrowablePatternConverter converter = JsonThrowablePatternConverter.newInstance(null, null);
private static final String LINE_SEPARATOR = System.lineSeparator();
private JsonThrowablePatternConverter converter = JsonThrowablePatternConverter.newInstance(null, null);

public void testNoStacktrace() throws IOException {
LogEvent event = Log4jLogEvent.newBuilder()
Expand All @@ -48,15 +49,15 @@ public void testNoStacktrace() throws IOException {

public void testStacktraceWithJson() throws IOException {

String json = "{\n" +
" \"terms\" : {\n" +
" \"user\" : [\n" +
" \"u1\",\n" +
" \"u2\",\n" +
" \"u3\"\n" +
" ],\n" +
" \"boost\" : 1.0\n" +
" }\n" +
String json = "{" + LINE_SEPARATOR +
" \"terms\" : {" + LINE_SEPARATOR +
" \"user\" : [" + LINE_SEPARATOR +
" \"u1\"," + LINE_SEPARATOR +
" \"u2\"," + LINE_SEPARATOR +
" \"u3\"" + LINE_SEPARATOR +
" ]," + LINE_SEPARATOR +
" \"boost\" : 1.0" + LINE_SEPARATOR +
" }" + LINE_SEPARATOR +
"}";
Exception thrown = new Exception(json);
LogEvent event = Log4jLogEvent.newBuilder()
Expand All @@ -73,7 +74,7 @@ public void testStacktraceWithJson() throws IOException {
.findFirst()
.orElseThrow(() -> new AssertionError("no logs parsed"));

int jsonLength = json.split("\n").length;
int jsonLength = json.split(LINE_SEPARATOR).length;
int stacktraceLength = thrown.getStackTrace().length;
assertThat("stacktrace should formatted in multiple lines. JsonLogLine= " + jsonLogLine+" result= "+result,
jsonLogLine.stacktrace().size(), equalTo(jsonLength + stacktraceLength));
Expand Down
5 changes: 3 additions & 2 deletions test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -997,8 +997,9 @@ public static List<DocIdSeqNoAndTerm> getDocIds(Engine engine, boolean refresh)
}
}
}
docs.sort(Comparator.comparing(DocIdSeqNoAndTerm::getId)
.thenComparingLong(DocIdSeqNoAndTerm::getSeqNo).thenComparingLong(DocIdSeqNoAndTerm::getPrimaryTerm));
docs.sort(Comparator.comparingLong(DocIdSeqNoAndTerm::getSeqNo)
.thenComparingLong(DocIdSeqNoAndTerm::getPrimaryTerm)
.thenComparing((DocIdSeqNoAndTerm::getId)));
return docs;
}
}
Expand Down
21 changes: 17 additions & 4 deletions x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.env.NodeEnvironment;
Expand Down Expand Up @@ -451,8 +452,18 @@ protected void assertIndexFullyReplicatedToFollower(String leaderIndex, String f
logger.info("--> asserting <<docId,seqNo>> between {} and {}", leaderIndex, followerIndex);
assertBusy(() -> {
Map<Integer, List<DocIdSeqNoAndTerm>> docsOnFollower = getDocIdAndSeqNos(clusterGroup.followerCluster, followerIndex);
logger.info("--> docs on the follower {}", docsOnFollower);
assertThat(docsOnFollower, equalTo(getDocIdAndSeqNos(clusterGroup.leaderCluster, leaderIndex)));
Map<Integer, List<DocIdSeqNoAndTerm>> docsOnLeader = getDocIdAndSeqNos(clusterGroup.leaderCluster, leaderIndex);
Map<Integer, Set<DocIdSeqNoAndTerm>> mismatchedDocs = new HashMap<>();
for (Map.Entry<Integer, List<DocIdSeqNoAndTerm>> fe : docsOnFollower.entrySet()) {
Set<DocIdSeqNoAndTerm> d1 = Sets.difference(
Sets.newHashSet(fe.getValue()), Sets.newHashSet(docsOnLeader.getOrDefault(fe.getKey(), Collections.emptyList())));
Set<DocIdSeqNoAndTerm> d2 = Sets.difference(
Sets.newHashSet(docsOnLeader.getOrDefault(fe.getKey(), Collections.emptyList())), Sets.newHashSet(fe.getValue()));
if (d1.isEmpty() == false || d2.isEmpty() == false) {
mismatchedDocs.put(fe.getKey(), Sets.union(d1, d2));
}
}
assertThat("mismatched documents [" + mismatchedDocs + "]", docsOnFollower, equalTo(docsOnLeader));
}, 120, TimeUnit.SECONDS);

logger.info("--> asserting seq_no_stats between {} and {}", leaderIndex, followerIndex);
Expand Down Expand Up @@ -481,13 +492,15 @@ private Map<Integer, List<DocIdSeqNoAndTerm>> getDocIdAndSeqNos(InternalTestClus
Randomness.shuffle(shardRoutings);
final Map<Integer, List<DocIdSeqNoAndTerm>> docs = new HashMap<>();
for (ShardRouting shardRouting : shardRoutings) {
if (shardRouting == null || shardRouting.assignedToNode() == false || docs.containsKey(shardRouting.shardId().id())) {
if (shardRouting == null || shardRouting.assignedToNode() == false) {
continue;
}
IndexShard indexShard = cluster.getInstance(IndicesService.class, state.nodes().get(shardRouting.currentNodeId()).getName())
.indexServiceSafe(shardRouting.index()).getShard(shardRouting.id());
try {
docs.put(shardRouting.shardId().id(), IndexShardTestCase.getDocIdAndSeqNos(indexShard).stream()
final List<DocIdSeqNoAndTerm> docsOnShard = IndexShardTestCase.getDocIdAndSeqNos(indexShard);
logger.info("--> shard {} docs {} seq_no_stats {}", shardRouting, docsOnShard, indexShard.seqNoStats());
docs.put(shardRouting.shardId().id(), docsOnShard.stream()
// normalize primary term as the follower use its own term
.map(d -> new DocIdSeqNoAndTerm(d.getId(), d.getSeqNo(), 1L))
.collect(Collectors.toList()));
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowStatsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ public void testFollowStatsApiResourceNotFound() throws Exception {
assertAcked(client().execute(PauseFollowAction.INSTANCE, new PauseFollowAction.Request("follower1")).actionGet());
}

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/38779")
public void testFollowStatsApiIncludeShardFollowStatsWithRemovedFollowerIndex() throws Exception {
final String leaderIndexSettings = getIndexSettings(1, 0,
singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true"));
Expand Down
34 changes: 18 additions & 16 deletions x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,22 +45,24 @@
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;

@TestLogging("org.elasticsearch.xpack.ccr:TRACE,org.elasticsearch.index.shard:DEBUG")
@TestLogging("org.elasticsearch.xpack.ccr:TRACE,org.elasticsearch.xpack.ccr.action.ShardChangesAction:DEBUG,"
+ "org.elasticsearch.index.shard:TRACE")
public class FollowerFailOverIT extends CcrIntegTestCase {

@Override
protected boolean reuseClusters() {
return false;
}

@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/38633")
public void testFailOverOnFollower() throws Exception {
final String leaderIndex = "leader_test_failover";
final String followerIndex = "follower_test_failover";
int numberOfReplicas = between(1, 2);
getFollowerCluster().startMasterOnlyNode();
getFollowerCluster().ensureAtLeastNumDataNodes(numberOfReplicas + between(1, 2));
String leaderIndexSettings = getIndexSettings(1, numberOfReplicas,
singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true"));
assertAcked(leaderClient().admin().indices().prepareCreate("leader-index").setSource(leaderIndexSettings, XContentType.JSON));
assertAcked(leaderClient().admin().indices().prepareCreate(leaderIndex).setSource(leaderIndexSettings, XContentType.JSON));
AtomicBoolean stopped = new AtomicBoolean();
Thread[] threads = new Thread[between(1, 8)];
AtomicInteger docID = new AtomicInteger();
Expand All @@ -77,20 +79,20 @@ public void testFailOverOnFollower() throws Exception {
}
if (frequently()) {
String id = Integer.toString(frequently() ? docID.incrementAndGet() : between(0, 10)); // sometimes update
IndexResponse indexResponse = leaderClient().prepareIndex("leader-index", "doc", id)
IndexResponse indexResponse = leaderClient().prepareIndex(leaderIndex, "doc", id)
.setSource("{\"f\":" + id + "}", XContentType.JSON).get();
logger.info("--> index id={} seq_no={}", indexResponse.getId(), indexResponse.getSeqNo());
logger.info("--> index {} id={} seq_no={}", leaderIndex, indexResponse.getId(), indexResponse.getSeqNo());
} else {
String id = Integer.toString(between(0, docID.get()));
DeleteResponse deleteResponse = leaderClient().prepareDelete("leader-index", "doc", id).get();
logger.info("--> delete id={} seq_no={}", deleteResponse.getId(), deleteResponse.getSeqNo());
DeleteResponse deleteResponse = leaderClient().prepareDelete(leaderIndex, "doc", id).get();
logger.info("--> delete {} id={} seq_no={}", leaderIndex, deleteResponse.getId(), deleteResponse.getSeqNo());
}
}
});
threads[i].start();
}
availableDocs.release(between(100, 200));
PutFollowAction.Request follow = putFollow("leader-index", "follower-index");
PutFollowAction.Request follow = putFollow(leaderIndex, followerIndex);
follow.getParameters().setMaxReadRequestOperationCount(randomIntBetween(32, 2048));
follow.getParameters().setMaxReadRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB));
follow.getParameters().setMaxOutstandingReadRequests(randomIntBetween(1, 10));
Expand All @@ -99,27 +101,27 @@ public void testFailOverOnFollower() throws Exception {
follow.getParameters().setMaxOutstandingWriteRequests(randomIntBetween(1, 10));
logger.info("--> follow request {}", Strings.toString(follow));
followerClient().execute(PutFollowAction.INSTANCE, follow).get();
disableDelayedAllocation("follower-index");
ensureFollowerGreen("follower-index");
awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex("follower-index"), 0), between(30, 80));
disableDelayedAllocation(followerIndex);
ensureFollowerGreen(followerIndex);
awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex(followerIndex), 0), between(30, 80));
final ClusterState clusterState = getFollowerCluster().clusterService().state();
for (ShardRouting shardRouting : clusterState.routingTable().allShards("follower-index")) {
for (ShardRouting shardRouting : clusterState.routingTable().allShards(followerIndex)) {
if (shardRouting.primary()) {
DiscoveryNode assignedNode = clusterState.nodes().get(shardRouting.currentNodeId());
getFollowerCluster().restartNode(assignedNode.getName(), new InternalTestCluster.RestartCallback());
break;
}
}
availableDocs.release(between(50, 200));
ensureFollowerGreen("follower-index");
ensureFollowerGreen(followerIndex);
availableDocs.release(between(50, 200));
awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex("follower-index"), 0), between(100, 150));
awaitGlobalCheckpointAtLeast(followerClient(), new ShardId(resolveFollowerIndex(followerIndex), 0), between(100, 150));
stopped.set(true);
for (Thread thread : threads) {
thread.join();
}
assertIndexFullyReplicatedToFollower("leader-index", "follower-index");
pauseFollow("follower-index");
assertIndexFullyReplicatedToFollower(leaderIndex, followerIndex);
pauseFollow(followerIndex);
}

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33337")
Expand Down
8 changes: 6 additions & 2 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/watch/WatchStatus.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ public ZonedDateTime lastChecked() {
return lastChecked;
}

public ZonedDateTime lastMetCondition() {
return lastMetCondition;
}

public ActionStatus actionStatus(String actionId) {
return actions.get(actionId);
}
Expand Down Expand Up @@ -252,10 +256,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field(Field.STATE.getPreferredName(), state, params);
}
if (lastChecked != null) {
builder.timeField(Field.LAST_CHECKED.getPreferredName(), lastChecked);
writeDate(Field.LAST_CHECKED.getPreferredName(), builder, lastChecked);
}
if (lastMetCondition != null) {
builder.timeField(Field.LAST_MET_CONDITION.getPreferredName(), lastMetCondition);
writeDate(Field.LAST_MET_CONDITION.getPreferredName(), builder, lastMetCondition);
}
if (actions != null) {
builder.startObject(Field.ACTIONS.getPreferredName());
Expand Down
Loading

0 comments on commit 6d23d8f

Please sign in to comment.