diff --git a/docs/content/development/extensions-core/namespaced-lookup.md b/docs/content/development/extensions-core/namespaced-lookup.md index 43677cf9401e..f98295d82a44 100644 --- a/docs/content/development/extensions-core/namespaced-lookup.md +++ b/docs/content/development/extensions-core/namespaced-lookup.md @@ -61,7 +61,7 @@ Cached namespace lookups can be specified as part of the [cluster wide config fo "tsColumn": "timeColumn" }, "firstCacheTimeout": 120000, - "oneToOne":true + "injective":true } ``` @@ -70,7 +70,7 @@ The parameters are as follows |--------|-----------|--------|-------| |`extractionNamespace`|Specifies how to populate the local cache. See below|Yes|-| |`firstCacheTimeout`|How long to wait (in ms) for the first run of the cache to populate. 0 indicates to not wait|No|`60000` (1 minute)| -|`oneToOne`|If the underlying map is injective (keys and values are unique) then optimizations can occur internally by setting this to `true`|No|`false`| +|`injective`|If the underlying map is injective (keys and values are unique) then optimizations can occur internally by setting this to `true`|No|`false`| Proper functionality of Namespaced lookups requires the following extension to be loaded on the broker, peon, and historical nodes: `druid-namespace-lookup` @@ -87,6 +87,10 @@ setting namespaces (broker, peon, historical) The cache is populated in different ways depending on the settings below. In general, most namespaces employ a `pollPeriod` at the end of which time they poll the remote resource of interest for updates. +`onHeap` uses `ConcurrentMap`s in the java heap, and thus affects garbage collection and heap sizing. +`offHeap` uses a 10MB on-heap buffer and MapDB using memory-mapped files in the java temporary directory. +So if total `cachedNamespace` lookup size is in excess of 10MB, the extra will be kept in memory as page cache, and paged in and out by general OS tunings. + # Supported Lookups For additional lookups, please see our [extensions list](../extensions.html). diff --git a/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/query/lookup/KafkaLookupExtractorFactory.java b/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/query/lookup/KafkaLookupExtractorFactory.java index f40d234b9a64..341ef046148f 100644 --- a/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/query/lookup/KafkaLookupExtractorFactory.java +++ b/extensions-core/kafka-extraction-namespace/src/main/java/io/druid/query/lookup/KafkaLookupExtractorFactory.java @@ -103,7 +103,7 @@ public KafkaLookupExtractorFactory( @JsonProperty("kafkaTopic") final String kafkaTopic, @JsonProperty("kafkaProperties") final Map kafkaProperties, @JsonProperty("connectTimeout") @Min(0) long connectTimeout, - @JsonProperty("isOneToOne") boolean isOneToOne + @JsonProperty("isInjective") boolean isOneToOne ) { this.kafkaTopic = Preconditions.checkNotNull(kafkaTopic, "kafkaTopic required"); diff --git a/extensions-core/namespace-lookup/src/main/java/io/druid/query/lookup/NamespaceLookupExtractorFactory.java b/extensions-core/namespace-lookup/src/main/java/io/druid/query/lookup/NamespaceLookupExtractorFactory.java index b9bab1be0422..3c8a0e9eac89 100644 --- a/extensions-core/namespace-lookup/src/main/java/io/druid/query/lookup/NamespaceLookupExtractorFactory.java +++ b/extensions-core/namespace-lookup/src/main/java/io/druid/query/lookup/NamespaceLookupExtractorFactory.java @@ -72,7 +72,7 @@ public class NamespaceLookupExtractorFactory implements LookupExtractorFactory private final LookupIntrospectHandler lookupIntrospectHandler; private final ExtractionNamespace extractionNamespace; private final long firstCacheTimeout; - private final boolean oneToOne; + private final boolean injective; private final String extractorID; @@ -80,7 +80,7 @@ public class NamespaceLookupExtractorFactory implements LookupExtractorFactory public NamespaceLookupExtractorFactory( @JsonProperty("extractionNamespace") ExtractionNamespace extractionNamespace, @JsonProperty("firstCacheTimeout") Long firstCacheTimeout, - @JsonProperty("oneToOne") boolean oneToOne, + @JsonProperty("injective") boolean injective, @JacksonInject final NamespaceExtractionCacheManager manager ) { @@ -90,7 +90,7 @@ public NamespaceLookupExtractorFactory( ); this.firstCacheTimeout = firstCacheTimeout == null ? DEFAULT_SCHEDULE_TIMEOUT : firstCacheTimeout; Preconditions.checkArgument(this.firstCacheTimeout >= 0); - this.oneToOne = oneToOne; + this.injective = injective; this.manager = manager; this.extractorID = buildID(); this.lookupIntrospectHandler = new LookupIntrospectHandler() @@ -207,7 +207,7 @@ public boolean replaces(@Nullable LookupExtractorFactory other) { if (other != null && other instanceof NamespaceLookupExtractorFactory) { NamespaceLookupExtractorFactory that = (NamespaceLookupExtractorFactory) other; - if (isOneToOne() != ((NamespaceLookupExtractorFactory) other).isOneToOne()) { + if (isInjective() != ((NamespaceLookupExtractorFactory) other).isInjective()) { return true; } if (getFirstCacheTimeout() != ((NamespaceLookupExtractorFactory) other).getFirstCacheTimeout()) { @@ -237,9 +237,9 @@ public long getFirstCacheTimeout() } @JsonProperty - public boolean isOneToOne() + public boolean isInjective() { - return oneToOne; + return injective; } private String buildID() @@ -275,7 +275,7 @@ public LookupExtractor get() } while (!preVersion.equals(postVersion)); final byte[] v = StringUtils.toUtf8(postVersion); final byte[] id = StringUtils.toUtf8(extractorID); - return new MapLookupExtractor(map, isOneToOne()) + return new MapLookupExtractor(map, isInjective()) { @Override public byte[] getCacheKey()