From e6915d8228b53214d3fc8f0c11467c2985cd9eac Mon Sep 17 00:00:00 2001 From: Linary Date: Mon, 26 Apr 2021 14:29:24 +0800 Subject: [PATCH] Fix rate inaccurate when print progress (#205) Change-Id: I7806987d76affb416f6f7daabd45a3999027c47c --- .../hugegraph/loader/HugeGraphLoader.java | 4 ++-- .../hugegraph/loader/metrics/LoadSummary.java | 18 +++++++++++------- .../hugegraph/loader/metrics/RangesTimer.java | 8 ++++---- .../baidu/hugegraph/loader/util/Printer.java | 12 +++++++++--- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/baidu/hugegraph/loader/HugeGraphLoader.java b/src/main/java/com/baidu/hugegraph/loader/HugeGraphLoader.java index fc466e1a6..f231e6e94 100644 --- a/src/main/java/com/baidu/hugegraph/loader/HugeGraphLoader.java +++ b/src/main/java/com/baidu/hugegraph/loader/HugeGraphLoader.java @@ -178,8 +178,8 @@ private void loadInputs() { // Waiting for async worker threads finish this.manager.waitFinished(); } finally { - summary.stopFlowRangeTimer(ElemType.VERTEX); - summary.stopFlowRangeTimer(ElemType.EDGE); + summary.calculateTotalTime(ElemType.VERTEX); + summary.calculateTotalTime(ElemType.EDGE); summary.stopTotalTimer(); } Printer.printFinalProgress(this.context); diff --git a/src/main/java/com/baidu/hugegraph/loader/metrics/LoadSummary.java b/src/main/java/com/baidu/hugegraph/loader/metrics/LoadSummary.java index 3be623a25..4d0759525 100644 --- a/src/main/java/com/baidu/hugegraph/loader/metrics/LoadSummary.java +++ b/src/main/java/com/baidu/hugegraph/loader/metrics/LoadSummary.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import org.apache.commons.lang3.time.StopWatch; @@ -34,8 +35,8 @@ public final class LoadSummary { private final LongAdder vertexLoaded; private final LongAdder edgeLoaded; private final StopWatch totalTimer; - private final LongAdder vertexTime; - private final LongAdder edgeTime; + private final AtomicLong vertexTime; + private final AtomicLong edgeTime; private final RangesTimer vertexRangesTimer; private final RangesTimer edgeRangesTimer; // Every input struct has a metric @@ -45,8 +46,8 @@ public LoadSummary() { this.vertexLoaded = new LongAdder(); this.edgeLoaded = new LongAdder(); this.totalTimer = new StopWatch(); - this.vertexTime = new LongAdder(); - this.edgeTime = new LongAdder(); + this.vertexTime = new AtomicLong(); + this.edgeTime = new AtomicLong(); this.vertexRangesTimer = new RangesTimer(10000); this.edgeRangesTimer = new RangesTimer(10000); this.inputMetricsMap = InsertionOrderUtil.newMap(); @@ -117,11 +118,11 @@ public void addTimeRange(ElemType type, long start, long end) { timer.addTimeRange(start, end); } - public void stopFlowRangeTimer(ElemType type) { + public void calculateTotalTime(ElemType type) { RangesTimer timer = type.isVertex() ? this.vertexRangesTimer : this.edgeRangesTimer; - LongAdder elemTime = type.isVertex() ? this.vertexTime : this.edgeTime; - elemTime.add(timer.totalTime()); + AtomicLong elemTime = type.isVertex() ? this.vertexTime : this.edgeTime; + elemTime.set(timer.totalTime()); } public long totalTime() { @@ -149,6 +150,9 @@ public void stopTotalTimer() { } public long loadRate(ElemType type) { + // Ensure vetex time and edge time has been set + this.calculateTotalTime(type); + boolean isVertex = type.isVertex(); long totalTime = isVertex ? this.vertexTime() : this.edgeTime(); if (totalTime == 0) { diff --git a/src/main/java/com/baidu/hugegraph/loader/metrics/RangesTimer.java b/src/main/java/com/baidu/hugegraph/loader/metrics/RangesTimer.java index 863080fcc..468ce46a7 100644 --- a/src/main/java/com/baidu/hugegraph/loader/metrics/RangesTimer.java +++ b/src/main/java/com/baidu/hugegraph/loader/metrics/RangesTimer.java @@ -58,8 +58,8 @@ public RangesTimer(int capacity) { public synchronized long totalTime() { if (!this.ranges.isEmpty()) { - long time = this.caculate(); - this.totalTime += time; + long incrTime = this.caculate(); + this.totalTime += incrTime; this.ranges.clear(); } return this.totalTime; @@ -67,8 +67,8 @@ public synchronized long totalTime() { public synchronized void addTimeRange(long start, long end) { if (this.ranges.size() >= this.capacity) { - long time = this.caculate(); - this.totalTime += time; + long incrTime = this.caculate(); + this.totalTime += incrTime; this.ranges.clear(); } this.ranges.add(new TimeRange(start, end)); diff --git a/src/main/java/com/baidu/hugegraph/loader/util/Printer.java b/src/main/java/com/baidu/hugegraph/loader/util/Printer.java index 936b9fee9..539bf7bcb 100644 --- a/src/main/java/com/baidu/hugegraph/loader/util/Printer.java +++ b/src/main/java/com/baidu/hugegraph/loader/util/Printer.java @@ -114,12 +114,18 @@ private static void printCountReport(LoadReport report) { } private static void printMeterReport(LoadSummary summary) { + long totalTime = summary.totalTime(); + long vertexTime = summary.vertexTime(); + long edgeTime = summary.edgeTime(); + long readTime = totalTime - vertexTime - edgeTime; + printAndLog("meter metrics"); - printAndLog("total time", TimeUtil.readableTime(summary.totalTime())); - printAndLog("vertex time", TimeUtil.readableTime(summary.vertexTime())); + printAndLog("total time", TimeUtil.readableTime(totalTime)); + printAndLog("read time", TimeUtil.readableTime(readTime)); + printAndLog("vertex load time", TimeUtil.readableTime(vertexTime)); printAndLog("vertex load rate(vertices/s)", summary.loadRate(ElemType.VERTEX)); - printAndLog("edge time", TimeUtil.readableTime(summary.edgeTime())); + printAndLog("edge load time", TimeUtil.readableTime(edgeTime)); printAndLog("edge load rate(edges/s)", summary.loadRate(ElemType.EDGE)); }