Skip to content

Commit

Permalink
FIX-2387 Exclude unshared mnemonics when comparing log data (#2393)
Browse files Browse the repository at this point in the history
  • Loading branch information
@janmarius
janmarius authored Apr 29, 2024
1 parent 4ad4b1b commit 3ea8818
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 69 deletions.
63 changes: 15 additions & 48 deletions Src/WitsmlExplorer.Api/Workers/CompareLogDataWorker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public class CompareLogDataWorker : BaseWorker<CompareLogDataJob>, IWorker
public JobType JobType => JobType.CompareLogData;
private List<ICompareLogDataItem> _compareLogDataReportItems;
private Dictionary<string, int> _mnemonicsMismatchCount;
private Dictionary<string, string> _unsharedMnemonics;
private string _sourceServerName;
private string _targetServerName;
private const int MaxMismatchesLimit = 500;
Expand Down Expand Up @@ -61,6 +62,9 @@ public CompareLogDataWorker(ILogger<CompareLogDataJob> logger, IWitsmlClientProv
// Set up mnemonics mismatch count dictionary
_mnemonicsMismatchCount = new Dictionary<string, int>();

// Set up unshared mnemonics dictionary
_unsharedMnemonics = new Dictionary<string, string>();

// Get logs
WitsmlLog sourceLog = await LogWorkerTools.GetLog(GetSourceWitsmlClientOrThrow(), job.SourceLog, ReturnElements.HeaderOnly);
WitsmlLog targetLog = await LogWorkerTools.GetLog(GetTargetWitsmlClientOrThrow(), job.TargetLog, ReturnElements.HeaderOnly);
Expand Down Expand Up @@ -102,12 +106,12 @@ public CompareLogDataWorker(ILogger<CompareLogDataJob> logger, IWitsmlClientProv
}
else if (sourceLogMnemonics.Contains(mnemonic))
{
await AddUnsharedMnemonicData(ServerType.Source, GetSourceWitsmlClientOrThrow(), sourceLog, mnemonic);
_unsharedMnemonics[mnemonic] = sourceLog.Name;

}
else if (targetLogMnemonics.Contains(mnemonic))
{
await AddUnsharedMnemonicData(ServerType.Target, GetTargetWitsmlClientOrThrow(), targetLog, mnemonic);
_unsharedMnemonics[mnemonic] = targetLog.Name;
}
else
{
Expand Down Expand Up @@ -154,49 +158,6 @@ private async Task AddSharedMnemonicData(WitsmlLog sourceLog, WitsmlLog targetLo
}
}

private async Task AddUnsharedMnemonicData(ServerType serverType, IWitsmlClient witsmlClient, WitsmlLog log, string mnemonic)
{
WitsmlLogData mnemonicData = await LogWorkerTools.GetLogDataForCurve(witsmlClient, log, mnemonic, Logger);

foreach (string dataRow in mnemonicData.Data.Select(row => row.Data))
{
if (_mnemonicsMismatchCount[mnemonic] >= MaxMismatchesLimit) break;

var data = dataRow.Split(CommonConstants.DataSeparator);
var index = data.First();
var value = data.Last();

if (_isEqualNumOfDecimals && serverType == ServerType.Source)
{
AddReportItem(mnemonic, index, value, null);
}
else if (_isEqualNumOfDecimals && serverType == ServerType.Target)
{
AddReportItem(mnemonic, index, null, value);
}
else if (!_isEqualNumOfDecimals && serverType == ServerType.Source && (_sourceDepthLogDecimals < _targetDepthLogDecimals))
{
AddUnequalServerDecimalsReportItem(mnemonic, index, null, value, null);
}
else if (!_isEqualNumOfDecimals && serverType == ServerType.Source && (_sourceDepthLogDecimals > _targetDepthLogDecimals))
{
AddUnequalServerDecimalsReportItem(mnemonic, null, index, null, value);
}
else if (!_isEqualNumOfDecimals && serverType == ServerType.Target && (_targetDepthLogDecimals < _sourceDepthLogDecimals))
{
AddUnequalServerDecimalsReportItem(mnemonic, index, null, value, null);
}
else if (!_isEqualNumOfDecimals && serverType == ServerType.Target && (_targetDepthLogDecimals > _sourceDepthLogDecimals))
{
AddUnequalServerDecimalsReportItem(mnemonic, null, index, null, value);
}
else
{
throw new ArgumentException($"serverType={serverType} not supported.");
}
}
}

private void CompareLogData(Dictionary<string, string> sourceData, Dictionary<string, string> targetData, string mnemonic)
{
List<string> indexes = sourceData.Keys.Union(targetData.Keys).ToList();
Expand Down Expand Up @@ -287,23 +248,29 @@ private bool IsMnemonicDataEqual(string sourceValue, string targetValue)
private BaseReport GenerateReport(WitsmlLog sourceLog, WitsmlLog targetLog)
{
var sortedMnemonicsMismatchCount = _mnemonicsMismatchCount.OrderByDescending(x => x.Value);
string mnemonicsMismatchCountResult = "\nNumber of mismatches for each mnemonic:";
string mnemonicsMismatchCountResult = "\nNumber of mismatches for each shared mnemonic:";
foreach (KeyValuePair<string, int> keyValues in sortedMnemonicsMismatchCount)
{
mnemonicsMismatchCountResult += keyValues.Value >= 500 ? $"\n{keyValues.Key}: {keyValues.Value:n0} or more" : $"\n{keyValues.Key}: {keyValues.Value:n0}";
}

string unsharedMnemonicsResult = _unsharedMnemonics.Count > 0 ? "\nUnshared mnemonics:" : "";
foreach (KeyValuePair<string, string> mnemonic in _unsharedMnemonics)
{
unsharedMnemonicsResult += $"\n{mnemonic.Key}: Only exists in {mnemonic.Value}";
}

string indexRange = _compareAllIndexes ? "" : GetSharedIntervalReportFormat(sourceLog, targetLog);

return new BaseReport
{
Title = $"Log data comparison",
Summary = _compareLogDataReportItems.Count > 0
? $"Found {_compareLogDataReportItems.Count:n0} mismatches in the {(_isDepthLog ? "depth" : "time")} logs '{sourceLog.Name}' and '{targetLog.Name}':{indexRange}" + mnemonicsMismatchCountResult
? $"Found {_compareLogDataReportItems.Count:n0} mismatches in the shared mnemonics in the {(_isDepthLog ? "depth" : "time")} logs '{sourceLog.Name}' and '{targetLog.Name}':{indexRange}" + unsharedMnemonicsResult + mnemonicsMismatchCountResult
: $"No mismatches were found in the data indexes of the {(_isDepthLog ? "depth" : "time")} logs '{sourceLog.Name}' and '{targetLog.Name}'{indexRange}.",
ReportItems = _compareLogDataReportItems,
WarningMessage = _compareLogDataReportItems.Count >= MaxMismatchesLimit ? $"When finding {MaxMismatchesLimit:n0} mismatches while searching through data indexes for any mnemonic, we stop comparing the log data for that particular mnemonic. This is because {MaxMismatchesLimit:n0} is the maximum limit for mismatches during the search for each mnemonic. It indicates that there might be an issue with the compare log setup. However, you can still access the report for the comparison performed below." : null,
JobDetails = $"SourceServer::{_sourceServerName}|TargetServer::{_targetServerName}|SourceLog::{sourceLog.Name}|TargetLog::{targetLog.Name}"
JobDetails = $"SourceServer::{_sourceServerName}|TargetServer::{_targetServerName}|SourceLog::{sourceLog.Name}|TargetLog::{targetLog.Name}|Number of mismatches for shared mnemonics::{_compareLogDataReportItems.Count:n0}|Number of unshared mnemonics::{_unsharedMnemonics.Count:n0}"
};
}

Expand Down
10 changes: 9 additions & 1 deletion Src/WitsmlExplorer.Frontend/components/Modals/ReportModal.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import React, { useEffect, useState } from "react";
import JobService from "services/jobService";
import NotificationService from "services/notificationService";
import styled from "styled-components";
import { Colors } from "styles/Colors";

export interface ReportModal {
report?: BaseReport;
Expand Down Expand Up @@ -104,7 +105,8 @@ export const ReportModal = (props: ReportModal): React.ReactElement => {
{report.jobDetails.split("|").map((jobDetail) => {
const keyValuePair = jobDetail.split("::");
return (
<TextField
<StyledTextField
colors={colors}
key={keyValuePair[0].trim()}
readOnly
id={keyValuePair[0].trim()}
Expand Down Expand Up @@ -231,6 +233,12 @@ export const useGetReportOnJobFinished = (jobId: string): BaseReport => {
return report;
};

const StyledTextField = styled(TextField)<{ colors: Colors }>`
label {
color: ${(props) => props.colors.interactive.primaryResting};
}
`;

const ContentLayout = styled.div`
display: flex;
flex-direction: column;
Expand Down
28 changes: 8 additions & 20 deletions Tests/WitsmlExplorer.Api.Tests/Workers/CompareLogDataWorker.Tests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ public async Task CompareLogDataWorker_MismatchInSharedIndexTimeLogs_ReturnsMism
}

[Fact]
public async Task CompareLogData_DifferentNumberOfMnemonicsDepthLogs_ReturnsMismatchedReportItems()
public async Task CompareLogData_DifferentNumberOfMnemonicsDepthLogs_ReturnsOnlySharedMnemonicsMismatchedReportItems()
{
SetupWorker(0, 0);
string indexType = WitsmlLog.WITSML_INDEX_TYPE_MD;
Expand All @@ -222,32 +222,26 @@ public async Task CompareLogData_DifferentNumberOfMnemonicsDepthLogs_ReturnsMism
StartIndex = "0",
EndIndex = "1",
LogCurveInfo = new() { ("IndexCurve", CommonConstants.Unit.Meter), ("Curve1", "Unit1"), ("Curve2", "Unit2") },
Data = new() { "0,0,0", "1,0,0" }
Data = new() { "0,1,0", "1,0,0" }
};

var job = SetupTest(sourceLog, targetLog);
var (workerResult, refreshAction) = await _worker.Execute(job);
List<CompareLogDataItem> resultReportItems = job.JobInfo.Report.ReportItems.Select(x => (CompareLogDataItem)x).ToList();

int expectedNumberOfMismatches = 2;
CompareLogDataItem expectedMismatchItem1 = CreateCompareLogDataItem("0", "Curve2", null, "0");
CompareLogDataItem expectedMismatchItem2 = CreateCompareLogDataItem("1", "Curve2", null, "0");
int expectedNumberOfMismatches = 1;
CompareLogDataItem expectedMismatchItem1 = CreateCompareLogDataItem("0", "Curve1", "0", "1");

Assert.Equal(expectedNumberOfMismatches, resultReportItems.Count);

Assert.Equal(expectedMismatchItem1.Index, resultReportItems[0].Index);
Assert.Equal(expectedMismatchItem1.Mnemonic, resultReportItems[0].Mnemonic);
Assert.Equal(expectedMismatchItem1.SourceValue, resultReportItems[0].SourceValue);
Assert.Equal(expectedMismatchItem1.TargetValue, resultReportItems[0].TargetValue);

Assert.Equal(expectedMismatchItem2.Index, resultReportItems[1].Index);
Assert.Equal(expectedMismatchItem2.Mnemonic, resultReportItems[1].Mnemonic);
Assert.Equal(expectedMismatchItem2.SourceValue, resultReportItems[1].SourceValue);
Assert.Equal(expectedMismatchItem2.TargetValue, resultReportItems[1].TargetValue);
}

[Fact]
public async Task CompareLogData_DifferentNumberOfMnemonicsTimeLogs_ReturnsMismatchedReportItems()
public async Task CompareLogData_DifferentNumberOfMnemonicsTimeLogs_ReturnsOnlySharedMnemonicsMismatchedReportItems()
{
SetupWorker(0, 0);
string indexType = WitsmlLog.WITSML_INDEX_TYPE_DATE_TIME;
Expand All @@ -266,28 +260,22 @@ public async Task CompareLogData_DifferentNumberOfMnemonicsTimeLogs_ReturnsMisma
StartIndex = "2023-09-28T08:10:00Z",
EndIndex = "2023-09-28T08:11:00Z",
LogCurveInfo = new() { ("IndexCurve", "DateTime"), ("Curve1", "Unit1"), ("Curve2", "Unit2") },
Data = new() { "2023-09-28T08:10:00Z,0,0", "2023-09-28T08:11:00Z,0,0" }
Data = new() { "2023-09-28T08:10:00Z,1,0", "2023-09-28T08:11:00Z,0,0" }
};

var job = SetupTest(sourceLog, targetLog);
var (workerResult, refreshAction) = await _worker.Execute(job);
List<CompareLogDataItem> resultReportItems = job.JobInfo.Report.ReportItems.Select(x => (CompareLogDataItem)x).ToList();

int expectedNumberOfMismatches = 2;
CompareLogDataItem expectedMismatchItem1 = CreateCompareLogDataItem("2023-09-28T08:10:00Z", "Curve2", null, "0");
CompareLogDataItem expectedMismatchItem2 = CreateCompareLogDataItem("2023-09-28T08:11:00Z", "Curve2", null, "0");
int expectedNumberOfMismatches = 1;
CompareLogDataItem expectedMismatchItem1 = CreateCompareLogDataItem("2023-09-28T08:10:00Z", "Curve1", "0", "1");

Assert.Equal(expectedNumberOfMismatches, resultReportItems.Count);

Assert.Equal(expectedMismatchItem1.Index, resultReportItems[0].Index);
Assert.Equal(expectedMismatchItem1.Mnemonic, resultReportItems[0].Mnemonic);
Assert.Equal(expectedMismatchItem1.SourceValue, resultReportItems[0].SourceValue);
Assert.Equal(expectedMismatchItem1.TargetValue, resultReportItems[0].TargetValue);

Assert.Equal(expectedMismatchItem2.Index, resultReportItems[1].Index);
Assert.Equal(expectedMismatchItem2.Mnemonic, resultReportItems[1].Mnemonic);
Assert.Equal(expectedMismatchItem2.SourceValue, resultReportItems[1].SourceValue);
Assert.Equal(expectedMismatchItem2.TargetValue, resultReportItems[1].TargetValue);
}

[Fact]
Expand Down

0 comments on commit 3ea8818

Please sign in to comment.