From 2d6184576be4b027e89618de93cbef3b88fca4df Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 4 Sep 2024 15:19:39 +0530 Subject: [PATCH 1/3] Log a small number of segments to refresh per datasource in the Coordinator --- .../segment/metadata/AbstractSegmentMetadataCache.java | 8 ++++++++ .../segment/metadata/CoordinatorSegmentMetadataCache.java | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index a2d5999a0e19..133d4d64231f 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -761,6 +761,9 @@ public Set refreshSegmentsForDataSource(final String dataSource, fina final Map segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString); final Set retVal = new HashSet<>(); + + logSegmentsToRefresh(dataSource, segments); + final Sequence sequence = runSegmentMetadataQuery( Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY) ); @@ -805,6 +808,11 @@ public Set refreshSegmentsForDataSource(final String dataSource, fina return retVal; } + void logSegmentsToRefresh(String dataSource, Set ids) + { + // no-op + } + /** * Action to be executed on the result of Segment metadata query. * Returns if the segment metadata was updated. diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index 49254c1da5cf..5b865d8d865f 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -23,6 +23,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.inject.Inject; import org.apache.druid.client.CoordinatorServerView; @@ -529,6 +530,12 @@ public void refresh(final Set segmentsToRefresh, final Set da } } + @Override + void logSegmentsToRefresh(String dataSource, Set ids) + { + log.info("Refreshing segments [%s] for datasource [%s]", Iterables.limit(ids, 10), dataSource); + } + private void filterRealtimeSegments(Set segmentIds) { synchronized (lock) { From f8e62c2035cbcb8a6581f1c9dc8225f872a92970 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 4 Sep 2024 16:55:19 +0530 Subject: [PATCH 2/3] review comments --- .../druid/segment/metadata/AbstractSegmentMetadataCache.java | 3 +++ .../segment/metadata/CoordinatorSegmentMetadataCache.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java index 133d4d64231f..d918ec5e3f29 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -808,6 +808,9 @@ public Set refreshSegmentsForDataSource(final String dataSource, fina return retVal; } + /** + * Log the segment details for a datasource to be refreshed for debugging purpose. + */ void logSegmentsToRefresh(String dataSource, Set ids) { // no-op diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index 5b865d8d865f..cfb70de2d8da 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -533,7 +533,7 @@ public void refresh(final Set segmentsToRefresh, final Set da @Override void logSegmentsToRefresh(String dataSource, Set ids) { - log.info("Refreshing segments [%s] for datasource [%s]", Iterables.limit(ids, 10), dataSource); + log.info("Refreshing segments [%s] for datasource [%s]", Iterables.limit(ids, 5), dataSource); } private void filterRealtimeSegments(Set segmentIds) From a564b8cece20195012ea47ad2d758a80e085015a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 4 Sep 2024 18:14:56 +0530 Subject: [PATCH 3/3] Update log message --- .../druid/segment/metadata/CoordinatorSegmentMetadataCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index cfb70de2d8da..ba695b1ac7f2 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -533,7 +533,7 @@ public void refresh(final Set segmentsToRefresh, final Set da @Override void logSegmentsToRefresh(String dataSource, Set ids) { - log.info("Refreshing segments [%s] for datasource [%s]", Iterables.limit(ids, 5), dataSource); + log.info("Logging a sample of 5 segments [%s] to be refreshed for datasource [%s]", Iterables.limit(ids, 5), dataSource); } private void filterRealtimeSegments(Set segmentIds)