Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: tsc elasticsearch query bugs #637

Merged
merged 6 commits into from
Jun 30, 2023
Merged

fix: tsc elasticsearch query bugs #637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e36624b
fix: fix less or great by datetime query, update keyword query operat…
Qinyouzeng Jun 28, 2023
0e32582
chore: remove 8.0
Qinyouzeng Jun 28, 2023
5f66f14
chore: update
Qinyouzeng Jun 28, 2023
0c2edbc
chore: remove log
Qinyouzeng Jun 28, 2023
aa0ee9d
fix: allow update change log and trace index name
Qinyouzeng Jun 29, 2023
faec1f4
chore: restore
Qinyouzeng Jun 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...StackExchangeRedis.Tests/Masa.Contrib.Caching.Distributed.StackExchangeRedis.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFrameworks>net6.0;net7.0;net8.0;</TargetFrameworks>
<TargetFrameworks>net6.0;net7.0;</TargetFrameworks>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<ImplicitUsings>enable</ImplicitUsings>
Expand Down
6 changes: 3 additions & 3 deletions src/Contrib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Constants/ElasticConstant.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ public static class ElasticConstant

internal static void InitLog(string indexName, bool isIndependent = false)
{
if (Log != null || string.IsNullOrEmpty(indexName))
if (string.IsNullOrEmpty(indexName))
return;
Log = new LogTraceSetting(indexName, isIndependent, TIMESTAMP);
}

internal static void InitTrace(string indexName, bool isIndependent = false)
{
if (Trace != null || string.IsNullOrEmpty(indexName))
return;
if (string.IsNullOrEmpty(indexName))
return;
Trace = new LogTraceSetting(indexName, isIndependent, TIMESTAMP);
}
}
27 changes: 17 additions & 10 deletions ...ackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/IElasticClientExtenstion.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ await client.SearchAsync<TResult>(s => searchDescriptorFunc(s.Index(indexName)).
}
catch (Exception ex)
{
throw new UserFriendlyException($"SearchAsync execute error {ex.Message}");
throw new UserFriendlyException($"SearchAsync execute error", ex);
}
}

Expand All @@ -37,21 +37,19 @@ await client.SearchAsync<TResult>(s => searchDescriptorFunc(s.Index(indexName)).
public static async Task<IEnumerable<MappingResponseDto>> GetMappingAsync(this ICaller caller, string indexName, CancellationToken token = default)
{
var path = $"/{indexName}/_mapping";
var result = await caller.GetAsync<object>(path, false, token);
var result = await caller.GetAsync<object>(path, token);
var json = (JsonElement)result!;
JsonElement mapping = default;
bool findMapping = false;
foreach (var item in json.EnumerateObject())
{
if (!findMapping && item.Value.TryGetProperty("mappings", out mapping))
JsonElement mapping;
if (item.Value.TryGetProperty("mappings", out mapping))
{
findMapping = true;
break;
var mappings = GetRepProperties(mapping, default!)!;
if (mappings != null && mappings.Any())
return mappings;
}
}

if (findMapping)
return GetRepProperties(mapping, default!)!;
throw new UserFriendlyException($"can't find mapping for index: {indexName}");
}

Expand Down Expand Up @@ -197,7 +195,7 @@ private static SearchDescriptor<TResult> AddCondition<TQuery, TResult>(this Sear
}
if (!string.IsNullOrEmpty(query.Keyword))
{
list.Add(queryContainer => queryContainer.QueryString(queryString => queryString.Query(query.Keyword)));
list.Add(queryContainer => queryContainer.QueryString(queryString => queryString.Query(query.Keyword).DefaultOperator(Operator.And)));
}

query.AppendConditions();
Expand Down Expand Up @@ -257,6 +255,7 @@ private static Func<QueryContainerDescriptor<TResult>, QueryContainer> CompareCo
{
CreateFieldKeyword(query.Name, mapping, out var fieldName, out var keyword);
var value = query.Value;
bool isDate = value is DateTime;

switch (query.Type)
{
Expand All @@ -265,12 +264,20 @@ private static Func<QueryContainerDescriptor<TResult>, QueryContainer> CompareCo
case ConditionTypes.NotEqual:
return (container) => !container.Match(f => f.Field(keyword).Query(value?.ToString()));
case ConditionTypes.Great:
if (isDate)
return (container) => container.DateRange(f => f.Field(keyword).GreaterThan((DateTime)value));
return (container) => container.Range(f => f.Field(keyword).GreaterThan(Convert.ToDouble(value)));
case ConditionTypes.Less:
if (isDate)
return (container) => container.DateRange(f => f.Field(keyword).LessThan((DateTime)value));
return (container) => container.Range(f => f.Field(keyword).LessThan(Convert.ToDouble(value)));
case ConditionTypes.GreatEqual:
if (isDate)
return (container) => container.DateRange(f => f.Field(keyword).GreaterThanOrEquals((DateTime)value));
return (container) => container.Range(f => f.Field(keyword).GreaterThanOrEquals(Convert.ToDouble(value)));
case ConditionTypes.LessEqual:
if (isDate)
return (container) => container.DateRange(f => f.Field(keyword).LessThanOrEquals((DateTime)value));
return (container) => container.Range(f => f.Field(keyword).LessThanOrEquals(Convert.ToDouble(value)));
case ConditionTypes.In:
return (container) => container.Terms(f => f.Field(keyword).Terms((IEnumerable<object>)value));
Expand Down
38 changes: 24 additions & 14 deletions ...rib/StackSdks/Masa.Contrib.StackSdks.Tsc.Elasticsearch/Extenistions/ServiceExtenistion.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ public static class ServiceExtenistion
public static IServiceCollection AddElasticClientLog(this IServiceCollection services, string[] nodes, string indexName)
{
ElasticConstant.InitLog(indexName, true);
AddElasticsearch(services, nodes, ElasticConstant.LOG_CALLER_CLIENT_NAME).AddSingleton<ILogService, LogService>();
if (services.BuildServiceProvider().GetService<ILogService>() == null)
AddElasticsearch(services, nodes, ElasticConstant.LOG_CALLER_CLIENT_NAME).AddScoped<ILogService, LogService>();
ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.LOG_CALLER_CLIENT_NAME, indexName);
return services;
}
Expand All @@ -19,16 +20,18 @@ public static IServiceCollection AddElasticClientLog(this IServiceCollection ser
Action<ElasticsearchOptions> elasearchConnectionAction, Action<MasaHttpClient> callerAction, string indexName)
{
ElasticConstant.InitLog(indexName, true);
AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.LOG_CALLER_CLIENT_NAME)
.AddSingleton<ILogService, LogService>();
if (services.BuildServiceProvider().GetService<ILogService>() == null)
AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.LOG_CALLER_CLIENT_NAME)
.AddScoped<ILogService, LogService>();
ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.LOG_CALLER_CLIENT_NAME, indexName);
return services;
}

public static IServiceCollection AddElasticClientTrace(this IServiceCollection services, string[] nodes, string indexName)
{
ElasticConstant.InitTrace(indexName, true);
AddElasticsearch(services, nodes, ElasticConstant.TRACE_CALLER_CLIENT_NAME).AddSingleton<ITraceService, TraceService>();
if (services.BuildServiceProvider().GetService<ITraceService>() == null)
AddElasticsearch(services, nodes, ElasticConstant.TRACE_CALLER_CLIENT_NAME).AddScoped<ITraceService, TraceService>();
ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.TRACE_CALLER_CLIENT_NAME, indexName);
return services;
}
Expand All @@ -37,8 +40,9 @@ public static IServiceCollection AddElasticClientTrace(this IServiceCollection s
Action<ElasticsearchOptions> elasearchConnectionAction, Action<MasaHttpClient> callerAction, string indexName)
{
ElasticConstant.InitTrace(indexName, true);
AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.TRACE_CALLER_CLIENT_NAME)
.AddSingleton<ITraceService, TraceService>();
if (services.BuildServiceProvider().GetService<ITraceService>() == null)
AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.TRACE_CALLER_CLIENT_NAME)
.AddScoped<ITraceService, TraceService>();
ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.TRACE_CALLER_CLIENT_NAME, indexName);
return services;
}
Expand All @@ -48,9 +52,10 @@ public static IServiceCollection AddElasticClientLogAndTrace(this IServiceCollec
{
ElasticConstant.InitLog(logIndexName);
ElasticConstant.InitTrace(traceIndexName);
AddElasticsearch(services, nodes, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME)
.AddSingleton<ILogService, LogService>()
.AddSingleton<ITraceService, TraceService>();
if (services.BuildServiceProvider().GetService<ILogService>() == null || services.BuildServiceProvider().GetService<ITraceService>() == null)
AddElasticsearch(services, nodes, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME)
.AddScoped<ILogService, LogService>()
.AddScoped<ITraceService, TraceService>();
ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, logIndexName);
ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, traceIndexName);
return services;
Expand All @@ -62,9 +67,10 @@ public static IServiceCollection AddElasticClientLogAndTrace(this IServiceCollec
{
ElasticConstant.InitLog(logIndexName);
ElasticConstant.InitTrace(traceIndexName);
AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME)
.AddSingleton<ILogService, LogService>()
.AddSingleton<ITraceService, TraceService>();
if (services.BuildServiceProvider().GetService<ILogService>() == null || services.BuildServiceProvider().GetService<ITraceService>() == null)
AddElasticsearch(services, elasearchConnectionAction, callerAction, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME)
.AddScoped<ILogService, LogService>()
.AddScoped<ITraceService, TraceService>();
ElasticConstant.Log.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, logIndexName);
ElasticConstant.Trace.Mappings = GetLazyMapping(services, ElasticConstant.DEFAULT_CALLER_CLIENT_NAME, traceIndexName);
return services;
Expand All @@ -91,9 +97,13 @@ private static IServiceCollection AddElasticsearch(IServiceCollection services,
Action<MasaHttpClient> callerAction, string name)
{
ArgumentNullException.ThrowIfNull(callerAction);
var factory = services.BuildServiceProvider().GetService<IElasticClientFactory>();
var callerFactory = services.BuildServiceProvider().GetService<ICallerFactory>();

return services.AddElasticsearch(name, elasticsearchConnectionAction)
.AddCaller(name, option => option.UseHttpClient(callerAction).UseAuthentication());
if (factory == null || factory.Create(name) == null || callerFactory == null || callerFactory.Create(name) == null)
services.AddElasticsearch(name, elasticsearchConnectionAction)
.AddCaller(name, option => option.UseHttpClient(callerAction).UseAuthentication());
return services;
}

internal static IElasticClient CreateElasticClient(this IElasticClientFactory elasticsearchFactory, bool isLog)
Expand Down
5 changes: 5 additions & 0 deletions ...StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/Service/LogServiceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ public static void InitializeLog(TestContext testContext)
httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_NAME, HttpMethod.Delete));
httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_NAME, HttpMethod.Put, mapping));
httpClient.Send(StaticConfig.CreateMessage($"{StaticConfig.LOG_INDEX_NAME}/_doc", HttpMethod.Post, dataJson));

httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_RENAME, HttpMethod.Delete));
httpClient.Send(StaticConfig.CreateMessage(StaticConfig.LOG_INDEX_RENAME, HttpMethod.Put, mapping));
httpClient.Send(StaticConfig.CreateMessage($"{StaticConfig.LOG_INDEX_RENAME}/_doc", HttpMethod.Post, dataJson));
Task.Delay(1000).Wait();
}

Expand All @@ -25,6 +29,7 @@ public void Initialize()
{
ServiceCollection services = new();
services.AddElasticClientLog(new string[] { StaticConfig.HOST }, StaticConfig.LOG_INDEX_NAME);
services.AddElasticClientLog(new string[] { StaticConfig.HOST }, StaticConfig.LOG_INDEX_RENAME);
var serviceProvider = services.BuildServiceProvider();
_logService = serviceProvider.GetRequiredService<ILogService>();
}
Expand Down
1 change: 1 addition & 0 deletions src/Contrib/StackSdks/Tests/Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests/StaticConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ namespace Masa.Contrib.StackSdks.Tsc.Elasticsearch.Tests;
public static class StaticConfig
{
internal const string LOG_INDEX_NAME = "test_logs";
internal const string LOG_INDEX_RENAME = "test_logs_new";
internal const string HOST = "http://localhost:9200";
internal const string TRACE_INDEX_NAME = "test_traces";
private static IConfiguration Configuration;
Expand Down
2 changes: 1 addition & 1 deletion ...nsions/Tests/Masa.Utils.Extensions.DotNet.Tests/Masa.Utils.Extensions.DotNet.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFrameworks>net6.0;net7.0;net8.0;</TargetFrameworks>
<TargetFrameworks>net6.0;net7.0;</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
Expand Down
Loading