From 34e2b596248bfc392925ede659fa2a651d362e18 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Mon, 10 Feb 2020 13:55:44 +1000 Subject: [PATCH] Add histogram datatype (#4367) This commit adds the histogram datatype to the client. The histogram datatype is available in Elasticsearch 7.6.0+ with at least a basic license level Closes #4358 (cherry picked from commit 6b3f816611a01ee5a48a0390c45ed11cad69a335) --- .../Mapping/DynamicTemplate/SingleMapping.cs | 4 ++ src/Nest/Mapping/Types/FieldType.cs | 5 +- src/Nest/Mapping/Types/Properties.cs | 6 +++ src/Nest/Mapping/Types/PropertyFormatter.cs | 4 ++ .../Histogram/HistogramAttribute.cs | 18 ++++++++ .../Histogram/HistogramProperty.cs | 46 +++++++++++++++++++ src/Nest/Mapping/Visitor/IMappingVisitor.cs | 4 ++ src/Nest/Mapping/Visitor/IPropertyVisitor.cs | 2 + src/Nest/Mapping/Visitor/MappingWalker.cs | 10 +++- .../Mapping/Visitor/NoopPropertyVisitor.cs | 5 ++ .../GetMapping/GetMappingApiTest.cs | 2 + .../Histogram/HistogramAttributeTests.cs | 32 +++++++++++++ .../Histogram/HistogramPropertyTests.cs | 44 ++++++++++++++++++ 13 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 src/Nest/Mapping/Types/Specialized/Histogram/HistogramAttribute.cs create mode 100644 src/Nest/Mapping/Types/Specialized/Histogram/HistogramProperty.cs create mode 100644 tests/Tests/Mapping/Types/Specialized/Histogram/HistogramAttributeTests.cs create mode 100644 tests/Tests/Mapping/Types/Specialized/Histogram/HistogramPropertyTests.cs diff --git a/src/Nest/Mapping/DynamicTemplate/SingleMapping.cs b/src/Nest/Mapping/DynamicTemplate/SingleMapping.cs index eff7ed2c95f..ce2e94cf2f9 100644 --- a/src/Nest/Mapping/DynamicTemplate/SingleMapping.cs +++ b/src/Nest/Mapping/DynamicTemplate/SingleMapping.cs @@ -67,6 +67,10 @@ public IProperty Join(Func, IJoinProperty> selector) = selector?.Invoke(new JoinPropertyDescriptor()); /// + public IProperty Histogram(Func, IHistogramProperty> selector) => + selector?.Invoke(new HistogramPropertyDescriptor()); + + /// public IProperty FieldAlias(Func, IFieldAliasProperty> selector) => selector?.Invoke(new FieldAliasPropertyDescriptor()); diff --git a/src/Nest/Mapping/Types/FieldType.cs b/src/Nest/Mapping/Types/FieldType.cs index 472bc858a28..e249fd21068 100644 --- a/src/Nest/Mapping/Types/FieldType.cs +++ b/src/Nest/Mapping/Types/FieldType.cs @@ -138,6 +138,9 @@ public enum FieldType Flattened, [EnumMember(Value = "shape")] - Shape + Shape, + + [EnumMember(Value = "histogram")] + Histogram } } diff --git a/src/Nest/Mapping/Types/Properties.cs b/src/Nest/Mapping/Types/Properties.cs index f2dd2e8a870..d3d397cce9e 100644 --- a/src/Nest/Mapping/Types/Properties.cs +++ b/src/Nest/Mapping/Types/Properties.cs @@ -121,6 +121,9 @@ TReturnType Nested(Func, INestedProp /// TReturnType Join(Func, IJoinProperty> selector); + /// + TReturnType Histogram(Func, IHistogramProperty> selector); + /// TReturnType FieldAlias(Func, IFieldAliasProperty> selector); @@ -212,6 +215,9 @@ public PropertiesDescriptor Object(Func public PropertiesDescriptor Flattened(Func, IFlattenedProperty> selector) => SetProperty(selector); + /// + public PropertiesDescriptor Histogram(Func, IHistogramProperty> selector) => SetProperty(selector); + public PropertiesDescriptor Custom(IProperty customType) => SetProperty(customType); private PropertiesDescriptor SetProperty(Func selector) diff --git a/src/Nest/Mapping/Types/PropertyFormatter.cs b/src/Nest/Mapping/Types/PropertyFormatter.cs index 65090d61eef..f2f9cd6fe59 100644 --- a/src/Nest/Mapping/Types/PropertyFormatter.cs +++ b/src/Nest/Mapping/Types/PropertyFormatter.cs @@ -94,6 +94,7 @@ public IProperty Deserialize(ref JsonReader reader, IJsonFormatterResolver forma case FieldType.RankFeature: return Deserialize(ref segmentReader, formatterResolver); case FieldType.RankFeatures: return Deserialize(ref segmentReader, formatterResolver); case FieldType.Flattened: return Deserialize(ref segmentReader, formatterResolver); + case FieldType.Histogram: return Deserialize(ref segmentReader, formatterResolver); case FieldType.None: // no "type" field in the property mapping, or FieldType enum could not be parsed from typeString return Deserialize(ref segmentReader, formatterResolver); @@ -199,6 +200,9 @@ public void Serialize(ref JsonWriter writer, IProperty value, IJsonFormatterReso case IFlattenedProperty flattenedProperty: Serialize(ref writer, flattenedProperty, formatterResolver); break; + case IHistogramProperty histogramProperty: + Serialize(ref writer, histogramProperty, formatterResolver); + break; case IGenericProperty genericProperty: Serialize(ref writer, genericProperty, formatterResolver); break; diff --git a/src/Nest/Mapping/Types/Specialized/Histogram/HistogramAttribute.cs b/src/Nest/Mapping/Types/Specialized/Histogram/HistogramAttribute.cs new file mode 100644 index 00000000000..1f33109a839 --- /dev/null +++ b/src/Nest/Mapping/Types/Specialized/Histogram/HistogramAttribute.cs @@ -0,0 +1,18 @@ +namespace Nest +{ + public class HistogramAttribute : ElasticsearchPropertyAttributeBase, IHistogramProperty + { + public HistogramAttribute() : base(FieldType.Histogram) { } + + /// + public bool IgnoreMalformed + { + get => Self.IgnoreMalformed.GetValueOrDefault(); + set => Self.IgnoreMalformed = value; + } + + bool? IHistogramProperty.IgnoreMalformed { get; set; } + + private IHistogramProperty Self => this; + } +} diff --git a/src/Nest/Mapping/Types/Specialized/Histogram/HistogramProperty.cs b/src/Nest/Mapping/Types/Specialized/Histogram/HistogramProperty.cs new file mode 100644 index 00000000000..d1432af62a9 --- /dev/null +++ b/src/Nest/Mapping/Types/Specialized/Histogram/HistogramProperty.cs @@ -0,0 +1,46 @@ +using System.Diagnostics; +using System.Runtime.Serialization; +using Elasticsearch.Net.Utf8Json; + +namespace Nest +{ + /// + /// A field to store pre-aggregated numerical data representing a histogram. + /// + /// Available in Elasticsearch 7.6.0+ with at least basic license level + /// + [InterfaceDataContract] + public interface IHistogramProperty : IProperty + { + /// + /// Whether to ignore malformed input values. + /// + [DataMember(Name = "ignore_malformed")] + bool? IgnoreMalformed { get; set; } + } + + /// + [DebuggerDisplay("{DebugDisplay}")] + public class HistogramProperty : PropertyBase, IHistogramProperty + { + public HistogramProperty() : base(FieldType.Histogram) { } + + /// + public bool? IgnoreMalformed { get; set; } + } + + /// + [DebuggerDisplay("{DebugDisplay}")] + public class HistogramPropertyDescriptor + : PropertyDescriptorBase, IHistogramProperty, T>, IHistogramProperty + where T : class + { + bool? IHistogramProperty.IgnoreMalformed { get; set; } + + public HistogramPropertyDescriptor() : base(FieldType.Histogram) { } + + /// + public HistogramPropertyDescriptor IgnoreMalformed(bool? ignoreMalformed = true) => + Assign(ignoreMalformed, (a, v) => a.IgnoreMalformed = v); + } +} diff --git a/src/Nest/Mapping/Visitor/IMappingVisitor.cs b/src/Nest/Mapping/Visitor/IMappingVisitor.cs index ac91624bf54..32075ab3222 100644 --- a/src/Nest/Mapping/Visitor/IMappingVisitor.cs +++ b/src/Nest/Mapping/Visitor/IMappingVisitor.cs @@ -61,6 +61,8 @@ public interface IMappingVisitor void Visit(ISearchAsYouTypeProperty property); void Visit(IFlattenedProperty property); + + void Visit(IHistogramProperty property); } public class NoopMappingVisitor : IMappingVisitor @@ -124,5 +126,7 @@ public virtual void Visit(IRankFeaturesProperty property) { } public virtual void Visit(ISearchAsYouTypeProperty property) { } public virtual void Visit(IFlattenedProperty property) { } + + public virtual void Visit(IHistogramProperty property) { } } } diff --git a/src/Nest/Mapping/Visitor/IPropertyVisitor.cs b/src/Nest/Mapping/Visitor/IPropertyVisitor.cs index 5bea0948928..f8749c0ca68 100644 --- a/src/Nest/Mapping/Visitor/IPropertyVisitor.cs +++ b/src/Nest/Mapping/Visitor/IPropertyVisitor.cs @@ -60,6 +60,8 @@ public interface IPropertyVisitor void Visit(IFlattenedProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute); + void Visit(IHistogramProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute); + IProperty Visit(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute); bool SkipProperty(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute); diff --git a/src/Nest/Mapping/Visitor/MappingWalker.cs b/src/Nest/Mapping/Visitor/MappingWalker.cs index f82736272e8..65803aea6b5 100644 --- a/src/Nest/Mapping/Visitor/MappingWalker.cs +++ b/src/Nest/Mapping/Visitor/MappingWalker.cs @@ -182,8 +182,6 @@ public void Accept(IProperties properties) Accept(t.Fields); }); break; - case FieldType.None: - continue; case FieldType.Percolator: Visit(field, t => { _visitor.Visit(t); }); break; @@ -253,6 +251,14 @@ public void Accept(IProperties properties) _visitor.Visit(t); }); break; + case FieldType.Histogram: + Visit(field, t => + { + _visitor.Visit(t); + }); + break; + case FieldType.None: + continue; } } } diff --git a/src/Nest/Mapping/Visitor/NoopPropertyVisitor.cs b/src/Nest/Mapping/Visitor/NoopPropertyVisitor.cs index e87b7e8f268..250e5b49786 100644 --- a/src/Nest/Mapping/Visitor/NoopPropertyVisitor.cs +++ b/src/Nest/Mapping/Visitor/NoopPropertyVisitor.cs @@ -60,6 +60,8 @@ public virtual void Visit(IKeywordProperty type, PropertyInfo propertyInfo, Elas public virtual void Visit(IFlattenedProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) { } + public virtual void Visit(IHistogramProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) { } + public virtual IProperty Visit(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) => null; public void Visit(IProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) @@ -144,6 +146,9 @@ public void Visit(IProperty type, PropertyInfo propertyInfo, ElasticsearchProper case IRankFeaturesProperty rankFeatures: Visit(rankFeatures, propertyInfo, attribute); break; + case IHistogramProperty histogram: + Visit(histogram, propertyInfo, attribute); + break; } } } diff --git a/tests/Tests/Indices/MappingManagement/GetMapping/GetMappingApiTest.cs b/tests/Tests/Indices/MappingManagement/GetMapping/GetMappingApiTest.cs index 6ccd12289a1..01a826cfe5b 100644 --- a/tests/Tests/Indices/MappingManagement/GetMapping/GetMappingApiTest.cs +++ b/tests/Tests/Indices/MappingManagement/GetMapping/GetMappingApiTest.cs @@ -211,6 +211,8 @@ internal class TestVisitor : IMappingVisitor public void Visit(IFlattenedProperty property) => Increment("flattened"); + public void Visit(IHistogramProperty property) => Increment("histogram"); + private void Increment(string key) { if (!Counts.ContainsKey(key)) Counts.Add(key, 0); diff --git a/tests/Tests/Mapping/Types/Specialized/Histogram/HistogramAttributeTests.cs b/tests/Tests/Mapping/Types/Specialized/Histogram/HistogramAttributeTests.cs new file mode 100644 index 00000000000..a96c9f123c8 --- /dev/null +++ b/tests/Tests/Mapping/Types/Specialized/Histogram/HistogramAttributeTests.cs @@ -0,0 +1,32 @@ +using Nest; + +namespace Tests.Mapping.Types.Specialized.Histogram +{ + public class HistogramTest + { + [Histogram(IgnoreMalformed = true)] + public string Full { get; set; } + + [Histogram] + public string Minimal { get; set; } + } + + public class HistogramAttributeTests : AttributeTestsBase + { + protected override object ExpectJson => new + { + properties = new + { + full = new + { + type = "histogram", + ignore_malformed = true + }, + minimal = new + { + type = "histogram" + } + } + }; + } +} diff --git a/tests/Tests/Mapping/Types/Specialized/Histogram/HistogramPropertyTests.cs b/tests/Tests/Mapping/Types/Specialized/Histogram/HistogramPropertyTests.cs new file mode 100644 index 00000000000..903530e717e --- /dev/null +++ b/tests/Tests/Mapping/Types/Specialized/Histogram/HistogramPropertyTests.cs @@ -0,0 +1,44 @@ +using System; +using Elastic.Xunit.XunitPlumbing; +using Nest; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Mapping.Types.Specialized.Histogram +{ + [SkipVersion("<7.6.0", "Introduced in 7.6.0")] + public class HistogramPropertyTests : PropertyTestsBase + { + public HistogramPropertyTests(WritableCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override object ExpectJson => new + { + properties = new + { + name = new + { + type = "histogram", + ignore_malformed = true, + } + } + }; + + protected override Func, IPromise> FluentProperties => f => f + .Histogram(s => s + .Name(p => p.Name) + .IgnoreMalformed() + ); + + + protected override IProperties InitializerProperties => new Properties + { + { + "name", new HistogramProperty + { + IgnoreMalformed = true, + } + } + }; + } +}