From da33bbae74b22e33f86e2649c9da817d7db835b6 Mon Sep 17 00:00:00 2001 From: Qinyouzeng <102203523+Qinyouzeng@users.noreply.github.com> Date: Fri, 30 Jun 2023 09:03:47 +0800 Subject: [PATCH] fix: tsc elasticsearch query bugs (#637) * fix: fix less or great by datetime query, update keyword query operate by and * chore: remove 8.0 * chore: update * chore: remove log * fix: allow update change log and trace index name * chore: restore --- ...istributed.StackExchangeRedis.Tests.csproj | 2 +- .../Constants/ElasticConstant.cs | 6 +-- .../Extenistions/IElasticClientExtenstion.cs | 27 ++++++++----- .../Extenistions/ServiceExtenistion.cs | 38 ++++++++++++------- .../Service/LogServiceTests.cs | 5 +++ .../StaticConfig.cs | 1 + .../Masa.Utils.Extensions.DotNet.Tests.csproj | 2 +- 7 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/Contrib/Caching/Distributed/Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests.csproj b/src/Contrib/Caching/Distributed/Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests.csproj index 3f757532e..76e911169 100644 --- a/src/Contrib/Caching/Distributed/Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests.csproj +++ b/src/Contrib/Caching/Distributed/Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests.csproj @@ -1,7 +1,7 @@ - net6.0;net7.0;net8.0; + net6.0;net7.0; enable false enable diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs index 082049c1e..4d8bf89bf 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs @@ -26,15 +26,15 @@ public static class ElasticConstant internal static void InitLog(string indexName, bool isIndependent = false) { - if (Log != null || string.IsNullOrEmpty(indexName)) + if (string.IsNullOrEmpty(indexName)) return; Log = new LogTraceSetting(indexName, isIndependent, TIMESTAMP); } internal static void InitTrace(string indexName, bool isIndependent = false) { - if (Trace != null || string.IsNullOrEmpty(indexName)) - return; + if (string.IsNullOrEmpty(indexName)) + return; Trace = new LogTraceSetting(indexName, isIndependent, TIMESTAMP); } } diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs index 9d5619586..87bf59e9c 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs @@ -28,7 +28,7 @@ await client.SearchAsync(s => searchDescriptorFunc(s.Index(indexName)). } catch (Exception ex) { - throw new UserFriendlyException($"SearchAsync execute error {ex.Message}"); + throw new UserFriendlyException($"SearchAsync execute error", ex); } } @@ -37,21 +37,19 @@ await client.SearchAsync(s => searchDescriptorFunc(s.Index(indexName)). public static async Task> GetMappingAsync(this ICaller caller, string indexName, CancellationToken token = default) { var path = $"/{indexName}/_mapping"; - var result = await caller.GetAsync(path, false, token); + var result = await caller.GetAsync(path, token); var json = (JsonElement)result!; - JsonElement mapping = default; - bool findMapping = false; foreach (var item in json.EnumerateObject()) { - if (!findMapping && item.Value.TryGetProperty("mappings", out mapping)) + JsonElement mapping; + if (item.Value.TryGetProperty("mappings", out mapping)) { - findMapping = true; - break; + var mappings = GetRepProperties(mapping, default!)!; + if (mappings != null && mappings.Any()) + return mappings; } } - if (findMapping) - return GetRepProperties(mapping, default!)!; throw new UserFriendlyException($"can't find mapping for index: {indexName}"); } @@ -197,7 +195,7 @@ private static SearchDescriptor AddCondition(this Sear } if (!string.IsNullOrEmpty(query.Keyword)) { - list.Add(queryContainer => queryContainer.QueryString(queryString => queryString.Query(query.Keyword))); + list.Add(queryContainer => queryContainer.QueryString(queryString => queryString.Query(query.Keyword).DefaultOperator(Operator.And))); } query.AppendConditions(); @@ -257,6 +255,7 @@ private static Func, QueryContainer> CompareCo { CreateFieldKeyword(query.Name, mapping, out var fieldName, out var keyword); var value = query.Value; + bool isDate = value is DateTime; switch (query.Type) { @@ -265,12 +264,20 @@ private static Func, QueryContainer> CompareCo case ConditionTypes.NotEqual: return (container) => !container.Match(f => f.Field(keyword).Query(value?.ToString())); case ConditionTypes.Great: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).GreaterThan((DateTime)value)); return (container) => container.Range(f => f.Field(keyword).GreaterThan(Convert.ToDouble(value))); case ConditionTypes.Less: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).LessThan((DateTime)value)); return (container) => container.Range(f => f.Field(keyword).LessThan(Convert.ToDouble(value))); case ConditionTypes.GreatEqual: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).GreaterThanOrEquals((DateTime)value)); return (container) => container.Range(f => f.Field(keyword).GreaterThanOrEquals(Convert.ToDouble(value))); case ConditionTypes.LessEqual: + if (isDate) + return (container) => container.DateRange(f => f.Field(keyword).LessThanOrEquals((DateTime)value)); return (container) => container.Range(f => f.Field(keyword).LessThanOrEquals(Convert.ToDouble(value))); case ConditionTypes.In: return (container) => container.Terms(f => f.Field(keyword).Terms((IEnumerable)value)); diff --git a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs index 3a34b21de..e95ad673e 100644 --- a/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs +++ b/src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs @@ -10,7 +10,8 @@ public static class ServiceExtenistion public static IServiceCollection AddElasticClientLog(this IServiceCollection services, string[] nodes, string indexName) { ElasticConstant.InitLog(indexName, true); - AddElasticsearch(services, nodes, ElasticConstant.LOG_CALLER_CLIENT_NAME).AddSingleton(); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, nodes, ElasticConstant.LOG_CALLER_CLIENT_NAME).AddScoped(); ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.LOG_CALLER_CLIENT_NAME, indexName); return services; } @@ -19,8 +20,9 @@ public static IServiceCollection AddElasticClientLog(this IServiceCollection ser Action elasearchConnectionAction, Action callerAction, string indexName) { ElasticConstant.InitLog(indexName, true); - AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.LOG_CALLER_CLIENT_NAME) - .AddSingleton(); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.LOG_CALLER_CLIENT_NAME) + .AddScoped(); ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.LOG_CALLER_CLIENT_NAME, indexName); return services; } @@ -28,7 +30,8 @@ public static IServiceCollection AddElasticClientLog(this IServiceCollection ser public static IServiceCollection AddElasticClientTrace(this IServiceCollection services, string[] nodes, string indexName) { ElasticConstant.InitTrace(indexName, true); - AddElasticsearch(services, nodes, ElasticConstant.TRACE_CALLER_CLIENT_NAME).AddSingleton(); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, nodes, ElasticConstant.TRACE_CALLER_CLIENT_NAME).AddScoped(); ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.TRACE_CALLER_CLIENT_NAME, indexName); return services; } @@ -37,8 +40,9 @@ public static IServiceCollection AddElasticClientTrace(this IServiceCollection s Action elasearchConnectionAction, Action callerAction, string indexName) { ElasticConstant.InitTrace(indexName, true); - AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.TRACE_CALLER_CLIENT_NAME) - .AddSingleton(); + if (services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.TRACE_CALLER_CLIENT_NAME) + .AddScoped(); ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.TRACE_CALLER_CLIENT_NAME, indexName); return services; } @@ -48,9 +52,10 @@ public static IServiceCollection AddElasticClientLogAndTrace(this IServiceCollec { ElasticConstant.InitLog(logIndexName); ElasticConstant.InitTrace(traceIndexName); - AddElasticsearch(services, nodes, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME) - .AddSingleton() - .AddSingleton(); + if (services.BuildServiceProvider().GetService() == null || services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, nodes, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME) + .AddScoped() + .AddScoped(); ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, logIndexName); ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, traceIndexName); return services; @@ -62,9 +67,10 @@ public static IServiceCollection AddElasticClientLogAndTrace(this IServiceCollec { ElasticConstant.InitLog(logIndexName); ElasticConstant.InitTrace(traceIndexName); - AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME) - .AddSingleton() - .AddSingleton(); + if (services.BuildServiceProvider().GetService() == null || services.BuildServiceProvider().GetService() == null) + AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME) + .AddScoped() + .AddScoped(); ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, logIndexName); ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, traceIndexName); return services; @@ -91,9 +97,13 @@ private static IServiceCollection AddElasticsearch(IServiceCollection services, Action callerAction, string name) { ArgumentNullException.ThrowIfNull(callerAction); + var factory = services.BuildServiceProvider().GetService(); + var callerFactory = services.BuildServiceProvider().GetService(); - return services.AddElasticsearch(name, elasticsearchConnectionAction) - .AddCaller(name, option => option.UseHttpClient(callerAction).UseAuthentication()); + if (factory == null || factory.Create(name) == null || callerFactory == null || callerFactory.Create(name) == null) + services.AddElasticsearch(name, elasticsearchConnectionAction) + .AddCaller(name, option => option.UseHttpClient(callerAction).UseAuthentication()); + return services; } internal static IElasticClient CreateElasticClient(this IElasticClientFactory elasticsearchFactory, bool isLog) diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/Service/LogServiceTests.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/Service/LogServiceTests.cs index e4d991eb4..9f90aa5e5 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/Service/LogServiceTests.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/Service/LogServiceTests.cs @@ -17,6 +17,10 @@ public static void InitializeLog(TestContext testContext) httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_NAME, HttpMethod.Delete)); httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_NAME, HttpMethod.Put, mapping)); httpClient.Send(StaticConfig.CreateMessage($"{StaticConfig.LOG_INDEX_NAME}/_doc", HttpMethod.Post, dataJson)); + + httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_RENAME, HttpMethod.Delete)); + httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_RENAME, HttpMethod.Put, mapping)); + httpClient.Send(StaticConfig.CreateMessage($"{StaticConfig.LOG_INDEX_RENAME}/_doc", HttpMethod.Post, dataJson)); Task.Delay(1000).Wait(); } @@ -25,6 +29,7 @@ public void Initialize() { ServiceCollection services = new(); services.AddElasticClientLog(new string[] { StaticConfig.HOST }, StaticConfig.LOG_INDEX_NAME); + services.AddElasticClientLog(new string[] { StaticConfig.HOST }, StaticConfig.LOG_INDEX_RENAME); var serviceProvider = services.BuildServiceProvider(); _logService = serviceProvider.GetRequiredService(); } diff --git a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/StaticConfig.cs b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/StaticConfig.cs index ac23b2ad6..18b3d1adf 100644 --- a/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/StaticConfig.cs +++ b/src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/StaticConfig.cs @@ -6,6 +6,7 @@ namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests; public static class StaticConfig { internal const string LOG_INDEX_NAME = "test_logs"; + internal const string LOG_INDEX_RENAME = "test_logs_new"; internal const string HOST = "http://localhost:9200"; internal const string TRACE_INDEX_NAME = "test_traces"; private static IConfiguration Configuration; diff --git a/src/Utils/Extensions/Tests/Masa.Utils.Extensions.DotNet.Tests/Masa.Utils.Extensions.DotNet.Tests.csproj b/src/Utils/Extensions/Tests/Masa.Utils.Extensions.DotNet.Tests/Masa.Utils.Extensions.DotNet.Tests.csproj index d24c99172..05c595b64 100644 --- a/src/Utils/Extensions/Tests/Masa.Utils.Extensions.DotNet.Tests/Masa.Utils.Extensions.DotNet.Tests.csproj +++ b/src/Utils/Extensions/Tests/Masa.Utils.Extensions.DotNet.Tests/Masa.Utils.Extensions.DotNet.Tests.csproj @@ -1,7 +1,7 @@ - net6.0;net7.0;net8.0; + net6.0;net7.0; enable enable false