Some adjustments to config examples. (#6973)

* Some adjustments to config examples. - Add ExitOnOutOfMemoryError to jvm.config examples. It was added a pretty long time ago (8u92) and is helpful since it prevents zombie processes from hanging around. (OOMEs tend to bork things) - Disable Broker caching and enable it on Historicals in example configs. This config tends to scale better since it enables the Historicals to merge results rather than sending everything by-segment to the Broker. Also switch to "caffeine" cache from "local". - Increase concurrency a bit for Broker example config. - Enable SQL in the example config, a baby step towards making SQL more of a thing. (It's still off by default in the code.) - Reduce memory use a bit for the quickstart configs. - Add example Router configs, in case someone wants to use that. One reason might be to get the fancy new console (#6923). * Add example Router configs. * Fix up router example properties. * Add router to quickstart supervise conf.
apache · Feb 1, 2019 · 4e42632 · 4e42632
1 parent 7f1b19b
commit 4e42632
Show file tree

Hide file tree

Showing 24 changed files with 250 additions and 33 deletions.
diff --git a/examples/conf/druid/_common/common.runtime.properties b/examples/conf/druid/_common/common.runtime.properties
@@ -120,3 +120,8 @@ druid.emitter.logging.logLevel=info
 # ommiting this will lead to index double as float at the storage layer
 
 druid.indexing.doubleStorage=double
+
+#
+# SQL
+#
+druid.sql.enable=true
diff --git a/examples/conf/druid/broker/jvm.config b/examples/conf/druid/broker/jvm.config
@@ -2,6 +2,7 @@
 -Xms24g
 -Xmx24g
 -XX:MaxDirectMemorySize=4096m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/conf/druid/broker/runtime.properties b/examples/conf/druid/broker/runtime.properties
@@ -20,16 +20,17 @@
 druid.service=druid/broker
 druid.plaintextPort=8082
 
-# HTTP server threads
-druid.broker.http.numConnections=5
-druid.server.http.numThreads=25
+# HTTP server settings
+druid.server.http.numThreads=60
+
+# HTTP client settings
+druid.broker.http.numConnections=10
 
 # Processing threads and buffers
 druid.processing.buffer.sizeBytes=536870912
-druid.processing.numThreads=7
+druid.processing.numMergeBuffers=2
+druid.processing.numThreads=1
 
-# Query cache
-druid.broker.cache.useCache=true
-druid.broker.cache.populateCache=true
-druid.cache.type=local
-druid.cache.sizeInBytes=2000000000
+# Query cache disabled -- push down caching and merging instead
+druid.broker.cache.useCache=false
+druid.broker.cache.populateCache=false
diff --git a/examples/conf/druid/coordinator/jvm.config b/examples/conf/druid/coordinator/jvm.config
@@ -1,6 +1,7 @@
 -server
 -Xms3g
 -Xmx3g
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/conf/druid/historical/jvm.config b/examples/conf/druid/historical/jvm.config
@@ -2,6 +2,7 @@
 -Xms8g
 -Xmx8g
 -XX:MaxDirectMemorySize=4096m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/conf/druid/historical/runtime.properties b/examples/conf/druid/historical/runtime.properties
@@ -30,3 +30,9 @@ druid.processing.numThreads=7
 # Segment storage
 druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":130000000000}]
 druid.server.maxSize=130000000000
+
+# Query cache
+druid.historical.cache.useCache=true
+druid.historical.cache.populateCache=true
+druid.cache.type=caffeine
+druid.cache.sizeInBytes=2000000000
diff --git a/examples/conf/druid/middleManager/jvm.config b/examples/conf/druid/middleManager/jvm.config
@@ -1,6 +1,7 @@
 -server
 -Xms64m
 -Xmx64m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/conf/druid/middleManager/runtime.properties b/examples/conf/druid/middleManager/runtime.properties
@@ -24,7 +24,7 @@ druid.plaintextPort=8091
 druid.worker.capacity=3
 
 # Task launch parameters
-druid.indexer.runner.javaOpts=-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+druid.indexer.runner.javaOpts=-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
 druid.indexer.task.baseTaskDir=var/druid/task
 
 # HTTP server threads

diff --git a/examples/conf/druid/overlord/jvm.config b/examples/conf/druid/overlord/jvm.config
@@ -1,6 +1,7 @@
 -server
 -Xms3g
 -Xmx3g
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/conf/druid/router/jvm.config b/examples/conf/druid/router/jvm.config
@@ -0,0 +1,10 @@
+-server
+-Xms512m
+-Xmx512m
+-XX:+UseG1GC
+-XX:MaxDirectMemorySize=512m
+-XX:+ExitOnOutOfMemoryError
+-Duser.timezone=UTC
+-Dfile.encoding=UTF-8
+-Djava.io.tmpdir=var/tmp
+-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
diff --git a/examples/conf/druid/router/runtime.properties b/examples/conf/druid/router/runtime.properties
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+druid.service=druid/router
+druid.plaintextPort=8888
+
+# HTTP proxy
+druid.router.http.numConnections=50
+druid.router.http.readTimeout=PT5M
+druid.router.http.numMaxThreads=100
+druid.server.http.numThreads=100
+
+# Service discovery
+druid.router.defaultBrokerServiceName=druid/broker
+druid.router.coordinatorServiceName=druid/coordinator
+
+# Management proxy to coordinator / overlord: required for unified web console.
+druid.router.managementProxy.enabled=true
diff --git a/examples/quickstart/tutorial/conf/druid/broker/jvm.config b/examples/quickstart/tutorial/conf/druid/broker/jvm.config
@@ -1,7 +1,8 @@
 -server
--Xms1g
--Xmx1g
--XX:MaxDirectMemorySize=1792m
+-Xms256m
+-Xmx256m
+-XX:MaxDirectMemorySize=768m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/quickstart/tutorial/conf/druid/broker/runtime.properties b/examples/quickstart/tutorial/conf/druid/broker/runtime.properties
@@ -1,16 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 druid.service=druid/broker
 druid.plaintextPort=8082
 
-# HTTP server threads
-druid.broker.http.numConnections=5
-druid.server.http.numThreads=9
+# HTTP server settings
+druid.server.http.numThreads=12
+
+# HTTP client settings
+druid.broker.http.numConnections=10
+druid.broker.http.maxQueuedBytes=5000000
 
 # Processing threads and buffers
-druid.processing.buffer.sizeBytes=256000000
-druid.processing.numThreads=2
+druid.processing.buffer.sizeBytes=100000000
+druid.processing.numMergeBuffers=2
+druid.processing.numThreads=1
+druid.processing.tmpDir=var/druid/processing
 
-# Query cache (we use a small local cache)
-druid.broker.cache.useCache=true
-druid.broker.cache.populateCache=true
-druid.cache.type=local
-druid.cache.sizeInBytes=10000000
+# Query cache disabled -- push down caching and merging instead
+druid.broker.cache.useCache=false
+druid.broker.cache.populateCache=false
diff --git a/examples/quickstart/tutorial/conf/druid/coordinator/jvm.config b/examples/quickstart/tutorial/conf/druid/coordinator/jvm.config
@@ -1,6 +1,7 @@
 -server
--Xms256m
--Xmx256m
+-Xms128m
+-Xmx128m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/quickstart/tutorial/conf/druid/coordinator/runtime.properties b/examples/quickstart/tutorial/conf/druid/coordinator/runtime.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 druid.service=druid/coordinator
 druid.plaintextPort=8081
 

diff --git a/examples/quickstart/tutorial/conf/druid/historical/jvm.config b/examples/quickstart/tutorial/conf/druid/historical/jvm.config
@@ -1,7 +1,8 @@
 -server
--Xms1g
--Xmx1g
--XX:MaxDirectMemorySize=1280m
+-Xms256m
+-Xmx256m
+-XX:MaxDirectMemorySize=768m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/quickstart/tutorial/conf/druid/historical/runtime.properties b/examples/quickstart/tutorial/conf/druid/historical/runtime.properties
@@ -1,13 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 druid.service=druid/historical
 druid.plaintextPort=8083
 
 # HTTP server threads
-druid.server.http.numThreads=9
+druid.server.http.numThreads=12
 
 # Processing threads and buffers
-druid.processing.buffer.sizeBytes=256000000
+druid.processing.buffer.sizeBytes=100000000
+druid.processing.numMergeBuffers=1
 druid.processing.numThreads=2
+druid.processing.tmpDir=var/druid/processing
 
 # Segment storage
 druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}]
 druid.server.maxSize=300000000000
+
+# Query cache
+druid.historical.cache.useCache=true
+druid.historical.cache.populateCache=true
+druid.cache.type=caffeine
+druid.cache.sizeInBytes=10000000
diff --git a/examples/quickstart/tutorial/conf/druid/middleManager/jvm.config b/examples/quickstart/tutorial/conf/druid/middleManager/jvm.config
@@ -1,6 +1,7 @@
 -server
 -Xms64m
 -Xmx64m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/quickstart/tutorial/conf/druid/middleManager/runtime.properties b/examples/quickstart/tutorial/conf/druid/middleManager/runtime.properties
@@ -1,11 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 druid.service=druid/middleManager
 druid.plaintextPort=8091
 
 # Number of tasks per middleManager
 druid.worker.capacity=3
 
 # Task launch parameters
-druid.indexer.runner.javaOpts=-server -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+druid.indexer.runner.javaOpts=-server -Xms512m -Xmx512m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
 druid.indexer.task.baseTaskDir=var/druid/task
 
 # HTTP server threads

diff --git a/examples/quickstart/tutorial/conf/druid/overlord/jvm.config b/examples/quickstart/tutorial/conf/druid/overlord/jvm.config
@@ -1,6 +1,7 @@
 -server
--Xms256m
--Xmx256m
+-Xms128m
+-Xmx128m
+-XX:+ExitOnOutOfMemoryError
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
 -Djava.io.tmpdir=var/tmp

diff --git a/examples/quickstart/tutorial/conf/druid/overlord/runtime.properties b/examples/quickstart/tutorial/conf/druid/overlord/runtime.properties
@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 druid.service=druid/overlord
 druid.plaintextPort=8090
 

diff --git a/examples/quickstart/tutorial/conf/druid/router/jvm.config b/examples/quickstart/tutorial/conf/druid/router/jvm.config
@@ -0,0 +1,10 @@
+-server
+-Xms128m
+-Xmx128m
+-XX:+UseG1GC
+-XX:MaxDirectMemorySize=128m
+-XX:+ExitOnOutOfMemoryError
+-Duser.timezone=UTC
+-Dfile.encoding=UTF-8
+-Djava.io.tmpdir=var/tmp
+-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager