From d2a1aff5129406feff471eafd7c30e4968806eae Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 24 Jun 2020 16:01:50 +0200 Subject: [PATCH 01/14] backport index name resolver --- .../metadata/IndexNameExpressionResolver.java | 70 +++++++------------ 1 file changed, 27 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index f111c7a576f23..8840d981f5942 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -27,21 +27,21 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.joda.JodaDateFormatter; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.DateMathParser; -import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.InvalidIndexNameException; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -49,7 +49,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -64,12 +63,11 @@ public class IndexNameExpressionResolver { private final DateMathExpressionResolver dateMathExpressionResolver; public IndexNameExpressionResolver(Settings settings) { - dateMathExpressionResolver = new DateMathExpressionResolver(settings); expressionResolvers = Arrays.asList( - dateMathExpressionResolver, - new WildcardExpressionResolver()); + dateMathExpressionResolver = new DateMathExpressionResolver(settings), + new WildcardExpressionResolver() + ); } - /** * Same as {@link #concreteIndexNames(ClusterState, IndicesOptions, String...)}, but the index expressions and options @@ -848,6 +846,7 @@ private static List resolveEmptyOrTrivialWildcard(IndicesOptions options static final class DateMathExpressionResolver implements ExpressionResolver { + private static final DateFormatter DEFAULT_DATE_FORMATTER = DateFormatters.forPattern("uuuu.MM.dd"); private static final String EXPRESSION_LEFT_BOUND = "<"; private static final String EXPRESSION_RIGHT_BOUND = ">"; private static final char LEFT_BOUND = '{'; @@ -855,15 +854,15 @@ static final class DateMathExpressionResolver implements ExpressionResolver { private static final char ESCAPE_CHAR = '\\'; private static final char TIME_ZONE_BOUND = '|'; - private final DateTimeZone defaultTimeZone; + private final ZoneId defaultTimeZone; private final String defaultDateFormatterPattern; - private final DateTimeFormatter defaultDateFormatter; + private final DateFormatter defaultDateFormatter; DateMathExpressionResolver(Settings settings) { String defaultTimeZoneId = settings.get("date_math_expression_resolver.default_time_zone", "UTC"); - this.defaultTimeZone = DateTimeZone.forID(defaultTimeZoneId); - defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "YYYY.MM.dd"); - this.defaultDateFormatter = DateTimeFormat.forPattern(defaultDateFormatterPattern); + this.defaultTimeZone = ZoneId.of(defaultTimeZoneId); + defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "uuuu.MM.dd"); + this.defaultDateFormatter = DateFormatters.forPattern("uuuu.MM.dd"); } @Override @@ -930,11 +929,10 @@ String resolveExpression(String expression, final Context context) { int dateTimeFormatLeftBoundIndex = inPlaceHolderString.indexOf(LEFT_BOUND); String mathExpression; String dateFormatterPattern; - DateTimeFormatter dateFormatter; - final DateTimeZone timeZone; + DateFormatter dateFormatter; + final ZoneId timeZone; if (dateTimeFormatLeftBoundIndex < 0) { mathExpression = inPlaceHolderString; - dateFormatterPattern = defaultDateFormatterPattern; dateFormatter = defaultDateFormatter; timeZone = defaultTimeZone; } else { @@ -947,24 +945,24 @@ String resolveExpression(String expression, final Context context) { inPlaceHolderString); } mathExpression = inPlaceHolderString.substring(0, dateTimeFormatLeftBoundIndex); - String patternAndTZid = + String dateFormatterPatternAndTimeZoneId = inPlaceHolderString.substring(dateTimeFormatLeftBoundIndex + 1, inPlaceHolderString.length() - 1); - int formatPatternTimeZoneSeparatorIndex = patternAndTZid.indexOf(TIME_ZONE_BOUND); + int formatPatternTimeZoneSeparatorIndex = dateFormatterPatternAndTimeZoneId.indexOf(TIME_ZONE_BOUND); if (formatPatternTimeZoneSeparatorIndex != -1) { - dateFormatterPattern = patternAndTZid.substring(0, formatPatternTimeZoneSeparatorIndex); - timeZone = DateTimeZone.forID(patternAndTZid.substring(formatPatternTimeZoneSeparatorIndex + 1)); + dateFormatterPattern = dateFormatterPatternAndTimeZoneId.substring(0, formatPatternTimeZoneSeparatorIndex); + timeZone = ZoneId.of(dateFormatterPatternAndTimeZoneId.substring(formatPatternTimeZoneSeparatorIndex + 1)); } else { - dateFormatterPattern = patternAndTZid; + dateFormatterPattern = dateFormatterPatternAndTimeZoneId; timeZone = defaultTimeZone; } - dateFormatter = DateTimeFormat.forPattern(dateFormatterPattern); + dateFormatter = DateFormatters.forPattern(dateFormatterPattern); } - DateTimeFormatter parser = dateFormatter.withLocale(Locale.ROOT).withZone(timeZone); - JodaDateFormatter formatter = new JodaDateFormatter(dateFormatterPattern, parser, parser); + DateFormatter formatter = dateFormatter.withZone(timeZone); DateMathParser dateMathParser = formatter.toDateMathParser(); - long millis = dateMathParser.parse(mathExpression, context::getStartTime, false, - DateUtils.dateTimeZoneToZoneId(timeZone)); - String time = formatter.formatMillis(millis); + long millis = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone); + + + String time = formatter.format(Instant.ofEpochMilli(millis)); beforePlaceHolderSb.append(time); inPlaceHolderSb = new StringBuilder(); inPlaceHolder = false; @@ -1007,18 +1005,4 @@ String resolveExpression(String expression, final Context context) { return beforePlaceHolderSb.toString(); } } - - /** - * Returns true iff the given expression resolves to the given index name otherwise false - */ - public final boolean matchesIndex(String indexName, String expression, ClusterState state) { - final String[] concreteIndices = concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), expression); - for (String index : concreteIndices) { - if (Regex.simpleMatch(index, indexName)) { - return true; - } - } - return indexName.equals(expression); - } - } From 1eb41c831d67baf450c6f54038eee4429340f9ba Mon Sep 17 00:00:00 2001 From: pgomulka Date: Wed, 24 Jun 2020 17:03:43 +0200 Subject: [PATCH 02/14] fix geting local date from week based date --- .../metadata/IndexNameExpressionResolver.java | 2 +- .../common/time/DateFormatters.java | 26 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 8840d981f5942..ea4254acc80c1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -955,7 +955,7 @@ String resolveExpression(String expression, final Context context) { dateFormatterPattern = dateFormatterPatternAndTimeZoneId; timeZone = defaultTimeZone; } - dateFormatter = DateFormatters.forPattern(dateFormatterPattern); + dateFormatter = DateFormatter.forPattern(dateFormatterPattern); } DateFormatter formatter = dateFormatter.withZone(timeZone); DateMathParser dateMathParser = formatter.toDateMathParser(); diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index 402ca20402de6..b002a2d8ded03 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -1676,24 +1676,28 @@ public static ZonedDateTime from(TemporalAccessor accessor) { } else if (accessor.isSupported(MONTH_OF_YEAR)) { // missing year, falling back to the epoch and then filling return getLocaldate(accessor).atStartOfDay(zoneId); + } else if (accessor.isSupported(WeekFields.SUNDAY_START.weekBasedYear()) || accessor.isSupported(WeekFields.ISO.weekBasedYear())) { + return localDateFromWeekBasedDate(WeekFields.SUNDAY_START, accessor).atStartOfDay(zoneId); } else if (accessor.isSupported(WeekFields.ISO.weekBasedYear())) { - if (accessor.isSupported(WeekFields.ISO.weekOfWeekBasedYear())) { - return Year.of(accessor.get(WeekFields.ISO.weekBasedYear())) - .atDay(1) - .with(WeekFields.ISO.weekOfWeekBasedYear(), accessor.getLong(WeekFields.ISO.weekOfWeekBasedYear())) - .atStartOfDay(zoneId); - } else { - return Year.of(accessor.get(WeekFields.ISO.weekBasedYear())) - .atDay(1) - .with(TemporalAdjusters.firstInMonth(DayOfWeek.MONDAY)) - .atStartOfDay(zoneId); - } + return localDateFromWeekBasedDate(WeekFields.ISO, accessor).atStartOfDay(zoneId); } // we should not reach this piece of code, everything being parsed we should be able to // convert to a zoned date time! If not, we have to extend the above methods throw new IllegalArgumentException("temporal accessor [" + accessor + "] cannot be converted to zoned date time"); } + private static LocalDate localDateFromWeekBasedDate(WeekFields WEEK_FIELDS, TemporalAccessor accessor) { + if (accessor.isSupported(WEEK_FIELDS.weekOfWeekBasedYear())) { + return LocalDate.ofEpochDay(0) + .with(WEEK_FIELDS.weekBasedYear(), accessor.get(WEEK_FIELDS.weekBasedYear())) + .with(WEEK_FIELDS.weekOfWeekBasedYear(), accessor.get(WEEK_FIELDS.weekOfWeekBasedYear())) + .with(ChronoField.DAY_OF_WEEK, WeekFields.ISO.getFirstDayOfWeek().getValue()); + } else { + return LocalDate.ofEpochDay(0) + .with(WEEK_FIELDS.weekBasedYear(), accessor.get(WEEK_FIELDS.weekBasedYear())) + .with(ChronoField.DAY_OF_WEEK, WeekFields.ISO.getFirstDayOfWeek().getValue()); + } + } private static LocalDate getLocaldate(TemporalAccessor accessor) { if (accessor.isSupported(MONTH_OF_YEAR)) { From d12a2097b0085c43ba1d48d2094ab7ef5666e7ea Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 25 Jun 2020 08:36:48 +0200 Subject: [PATCH 03/14] checkstyle --- .../cluster/metadata/IndexNameExpressionResolver.java | 7 ++++--- .../java/org/elasticsearch/common/time/DateFormatters.java | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index ea4254acc80c1..f28ffa02246ef 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -41,7 +41,6 @@ import java.time.Instant; import java.time.ZoneId; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -949,8 +948,10 @@ String resolveExpression(String expression, final Context context) { inPlaceHolderString.substring(dateTimeFormatLeftBoundIndex + 1, inPlaceHolderString.length() - 1); int formatPatternTimeZoneSeparatorIndex = dateFormatterPatternAndTimeZoneId.indexOf(TIME_ZONE_BOUND); if (formatPatternTimeZoneSeparatorIndex != -1) { - dateFormatterPattern = dateFormatterPatternAndTimeZoneId.substring(0, formatPatternTimeZoneSeparatorIndex); - timeZone = ZoneId.of(dateFormatterPatternAndTimeZoneId.substring(formatPatternTimeZoneSeparatorIndex + 1)); + dateFormatterPattern + = dateFormatterPatternAndTimeZoneId.substring(0, formatPatternTimeZoneSeparatorIndex); + timeZone = ZoneId.of( + dateFormatterPatternAndTimeZoneId.substring(formatPatternTimeZoneSeparatorIndex + 1)); } else { dateFormatterPattern = dateFormatterPatternAndTimeZoneId; timeZone = defaultTimeZone; diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index b002a2d8ded03..6fdf6ca879f87 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; -import java.time.DayOfWeek; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; @@ -37,7 +36,6 @@ import java.time.temporal.ChronoField; import java.time.temporal.IsoFields; import java.time.temporal.TemporalAccessor; -import java.time.temporal.TemporalAdjusters; import java.time.temporal.TemporalQueries; import java.time.temporal.WeekFields; import java.util.ArrayList; From 662d3ac59e44a585e0754606efc48de00063a6e7 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 25 Jun 2020 09:18:35 +0200 Subject: [PATCH 04/14] Revert "GlobalBuildInfo support packed-refs with work-tree (#50791)" This reverts commit 7bebc65cb2b398c87f4df3d61de50d98ec712916. --- .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 97db7f878df16..afb451f27b803 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -1138,10 +1138,10 @@ class BuildPlugin implements Plugin { Path refFile = gitDir.resolve(refName) if (Files.exists(refFile)) { revision = readFirstLine(refFile) - } else if (Files.exists(gitDir.resolve("packed-refs"))) { + } else if (Files.exists(dotGit.resolve("packed-refs"))) { // Check packed references for commit ID Pattern p = Pattern.compile("^([a-f1-9]{40}) " + refName + "\$") - Stream lines = Files.lines(gitDir.resolve("packed-refs")); + Stream lines = Files.lines(dotGit.resolve("packed-refs")); try { revision = lines.map( { s -> p.matcher(s) }) .filter( { m -> m.matches() }) From eccb89d4e385ac9150fe6f647cf2ada4d5ed9614 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 25 Jun 2020 09:42:20 +0200 Subject: [PATCH 05/14] fix millis parsing --- .../cluster/metadata/IndexNameExpressionResolver.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index f28ffa02246ef..72eeb24aba3f2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -39,7 +39,6 @@ import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.indices.InvalidIndexNameException; -import java.time.Instant; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -963,7 +962,7 @@ String resolveExpression(String expression, final Context context) { long millis = dateMathParser.parse(mathExpression, context::getStartTime, false, timeZone); - String time = formatter.format(Instant.ofEpochMilli(millis)); + String time = formatter.formatMillis(millis); beforePlaceHolderSb.append(time); inPlaceHolderSb = new StringBuilder(); inPlaceHolder = false; From 9febfc88a5f33620e85c549145749f5568c53e68 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 25 Jun 2020 10:44:09 +0200 Subject: [PATCH 06/14] Revert "Revert "GlobalBuildInfo support packed-refs with work-tree (#50791)"" This reverts commit 662d3ac5 --- .../main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index afb451f27b803..97db7f878df16 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -1138,10 +1138,10 @@ class BuildPlugin implements Plugin { Path refFile = gitDir.resolve(refName) if (Files.exists(refFile)) { revision = readFirstLine(refFile) - } else if (Files.exists(dotGit.resolve("packed-refs"))) { + } else if (Files.exists(gitDir.resolve("packed-refs"))) { // Check packed references for commit ID Pattern p = Pattern.compile("^([a-f1-9]{40}) " + refName + "\$") - Stream lines = Files.lines(dotGit.resolve("packed-refs")); + Stream lines = Files.lines(gitDir.resolve("packed-refs")); try { revision = lines.map( { s -> p.matcher(s) }) .filter( { m -> m.matches() }) From 73b9e08db9a3e9d6e4e757b0639aea320d9e65a1 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 25 Jun 2020 11:36:45 +0200 Subject: [PATCH 07/14] draft test --- .../test/ingest/20_date_index_processor.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml new file mode 100644 index 0000000000000..a7f5232a707fd --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml @@ -0,0 +1,31 @@ + +--- +"Test Put Versioned Pipeline": + - do: + ingest.put_pipeline: + id: "joda_index_name_pipeline" + body: > + { + "processors": [ + { + "date_index_name": { + "field": "my_timestamp", + "date_rounding": "d", + "index_name_prefix": "prefix-", + "index_name_format": "xxxx-w" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test_1 + type: _doc + pipeline: joda_index_name_pipeline + body: { my_timestamp: "2020-08-10T01:01:01.000Z" } + + - is_true: _id + - match: { _index: "prefix-2020-33" } + - match: { result: "created" } From c1208847f3fb18abd33ca9f0ea5162954ca349ac Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 25 Jun 2020 11:55:57 +0200 Subject: [PATCH 08/14] joda and java yml test --- .../ingest/100_date_index_name_processor.yml | 63 +++++++++++++++++++ .../101_date_index_name_joda_processor.yml | 37 +++++++++++ .../test/ingest/20_date_index_processor.yml | 31 --------- 3 files changed, 100 insertions(+), 31 deletions(-) create mode 100644 modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml index ccf83cc96bf7b..99982d51a0b73 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yml @@ -34,3 +34,66 @@ teardown: date: "2016-04-22T16:32:14.968Z" } - match: { _index: "events-2016-04-22"} + +--- +"Test date index name processor with joda pattern": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "processors": [ + { + "date_index_name" : { + "field": "date", + "date_rounding": "d", + "index_name_prefix": "prefix-", + "index_name_format": "xxxx-w" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: _doc + id: 1 + pipeline: "1" + body: { + date: "2020-08-10T01:01:01.000Z" + } + - match: { _index: "prefix-2020-33"} + + +--- +"Test date index name processor with java pattern": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "processors": [ + { + "date_index_name" : { + "field": "date", + "date_rounding": "d", + "index_name_prefix": "prefix-", + "index_name_format": "8YYYY-w" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: _doc + id: 1 + pipeline: "1" + body: { + date: "2020-08-10T01:01:01.000Z" + } + - match: { _index: "prefix-2020-33"} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml new file mode 100644 index 0000000000000..8705a9d451ed0 --- /dev/null +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml @@ -0,0 +1,37 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "1" + ignore: 404 + +--- +"Test date index name processor with defaults": + - do: + ingest.put_pipeline: + id: "1" + body: > + { + "processors": [ + { + "date_index_name" : { + "field": "date", + "date_rounding": "d", + "index_name_prefix": "prefix-", + "index_name_format": "xxxx-w" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: _doc + id: 1 + pipeline: "1" + body: { + date: "2020-08-10T01:01:01.000Z" + } + - match: { _index: "prefix-2020-323"} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml deleted file mode 100644 index a7f5232a707fd..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_index_processor.yml +++ /dev/null @@ -1,31 +0,0 @@ - ---- -"Test Put Versioned Pipeline": - - do: - ingest.put_pipeline: - id: "joda_index_name_pipeline" - body: > - { - "processors": [ - { - "date_index_name": { - "field": "my_timestamp", - "date_rounding": "d", - "index_name_prefix": "prefix-", - "index_name_format": "xxxx-w" - } - } - ] - } - - match: { acknowledged: true } - - - do: - index: - index: test_1 - type: _doc - pipeline: joda_index_name_pipeline - body: { my_timestamp: "2020-08-10T01:01:01.000Z" } - - - is_true: _id - - match: { _index: "prefix-2020-33" } - - match: { result: "created" } From 4eed78b4df0d812bbe1c8353f8b3895c57ec379d Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 25 Jun 2020 14:02:44 +0200 Subject: [PATCH 09/14] remove unused test --- .../101_date_index_name_joda_processor.yml | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml deleted file mode 100644 index 8705a9d451ed0..0000000000000 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/101_date_index_name_joda_processor.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -teardown: - - do: - ingest.delete_pipeline: - id: "1" - ignore: 404 - ---- -"Test date index name processor with defaults": - - do: - ingest.put_pipeline: - id: "1" - body: > - { - "processors": [ - { - "date_index_name" : { - "field": "date", - "date_rounding": "d", - "index_name_prefix": "prefix-", - "index_name_format": "xxxx-w" - } - } - ] - } - - match: { acknowledged: true } - - - do: - index: - index: test - type: _doc - id: 1 - pipeline: "1" - body: { - date: "2020-08-10T01:01:01.000Z" - } - - match: { _index: "prefix-2020-323"} From e7f1ba28d244bf12e23f26addc603400354b9212 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 26 Jun 2020 16:01:39 +0200 Subject: [PATCH 10/14] add index and search test case --- .../test/index/81_week_based_java_date.yml | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/81_week_based_java_date.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/81_week_based_java_date.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/81_week_based_java_date.yml new file mode 100644 index 0000000000000..ff971b3f57cef --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/81_week_based_java_date.yml @@ -0,0 +1,52 @@ +--- +"Index java 8 date without timezone": + + - skip: + version: " - 6.7.99" + reason: fixed in 6.8.11 + + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + mappings: + doc: + properties: + date_field: + type: date + format: "8YYYY-ww" + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index", "_type": "doc", "_id": "1"}}' + - '{"date_field": "2020-32"}' + - '{"index": {"_index": "test_index", "_type": "doc", "_id": "2"}}' + - '{"date_field": "2020-33"}' + + - match: { errors: false } + + - do: + get: + index: test_index + type: doc + id: 2 + + - match: { _index: test_index } + - match: { _type: doc } + - match: { _id: "2"} + - match: { _version: 1} + - match: { _source: { "date_field": "2020-33" }} + + - do: + search: + index: test_index + body: { "query": { "range": { "date_field": { "gte": "2020-33" } } } } + + - match: {hits.total: 1 } + - match: {hits.hits.0._index: test_index } + - match: {hits.hits.0._type: doc } + - match: {hits.hits.0._source.date_field: "2020-33" } + From e773628aebbecdc56432b89d130d9f2905472749 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 2 Jul 2020 08:49:48 +0200 Subject: [PATCH 11/14] code review fixes --- .../cluster/metadata/IndexNameExpressionResolver.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 72eeb24aba3f2..4f755c3060d1d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -61,8 +61,8 @@ public class IndexNameExpressionResolver { private final DateMathExpressionResolver dateMathExpressionResolver; public IndexNameExpressionResolver(Settings settings) { + dateMathExpressionResolver = new DateMathExpressionResolver(settings); expressionResolvers = Arrays.asList( - dateMathExpressionResolver = new DateMathExpressionResolver(settings), new WildcardExpressionResolver() ); } @@ -859,8 +859,8 @@ static final class DateMathExpressionResolver implements ExpressionResolver { DateMathExpressionResolver(Settings settings) { String defaultTimeZoneId = settings.get("date_math_expression_resolver.default_time_zone", "UTC"); this.defaultTimeZone = ZoneId.of(defaultTimeZoneId); - defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "uuuu.MM.dd"); - this.defaultDateFormatter = DateFormatters.forPattern("uuuu.MM.dd"); + defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "8uuuu.MM.dd"); + this.defaultDateFormatter = DateFormatters.forPattern(defaultDateFormatterPattern); } @Override From ed9e8c6ee24ba84f017bb522ec507450a418fcfa Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 2 Jul 2020 09:15:18 +0200 Subject: [PATCH 12/14] fix formatter creation --- .../cluster/metadata/IndexNameExpressionResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 4f755c3060d1d..95cdd5d920fca 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -860,7 +860,7 @@ static final class DateMathExpressionResolver implements ExpressionResolver { String defaultTimeZoneId = settings.get("date_math_expression_resolver.default_time_zone", "UTC"); this.defaultTimeZone = ZoneId.of(defaultTimeZoneId); defaultDateFormatterPattern = settings.get("date_math_expression_resolver.default_date_format", "8uuuu.MM.dd"); - this.defaultDateFormatter = DateFormatters.forPattern(defaultDateFormatterPattern); + this.defaultDateFormatter = DateFormatter.forPattern(defaultDateFormatterPattern); } @Override From de1067a1d943da49b4b21982d0e766666b6ab044 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 2 Jul 2020 10:09:46 +0200 Subject: [PATCH 13/14] add back dmath expresion resolver to a list --- .../cluster/metadata/IndexNameExpressionResolver.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 95cdd5d920fca..a73f64cb8b82a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -63,6 +63,7 @@ public class IndexNameExpressionResolver { public IndexNameExpressionResolver(Settings settings) { dateMathExpressionResolver = new DateMathExpressionResolver(settings); expressionResolvers = Arrays.asList( + dateMathExpressionResolver, new WildcardExpressionResolver() ); } From 0a89df06b3f2446ff892e03fe51215849ab9bdcb Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 2 Jul 2020 10:55:55 +0200 Subject: [PATCH 14/14] import --- .../main/java/org/elasticsearch/common/time/DateFormatters.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java index a23622eedf4f8..6e342d9e8cf38 100644 --- a/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java +++ b/server/src/main/java/org/elasticsearch/common/time/DateFormatters.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; -import java.time.DayOfWeek; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime;