Adding a Unified web console. (apache#6923)

* Adding new web console.

* fixed css

* fix form height

* fix typo

* do import custom react-table css

* added repo field so npm does not complain

* ask travis for node 10

* move indexing-service/src/main/resources/indexer_static into web-console

* fix resource names and paths

* add licenses

* fix exclude file

* add licenses to misc files and tidy up

* remove rebase marker

* fix link

* updated env variable name

* tidy up licenses and surface errors

* cleanup

* remove unused code, fix missing await

* TeamCity does not like the name aux

* add more links to tasks view

* rm pages

* update gitignore

* update readme to be accurate

* make clean script

* removed old console dependancy

* update Jetty routes

* add a comment for welcome files for coordinator

* do not show inital notifaction for now

* renamed overlord console back to console.html

* fix coordinator console

* rename coordinator-console.html to index.html

.travis.yml

@@ -22,6 +22,10 @@ sudo: false
 dist: trusty
 group: deprecated-2017Q4
 
+language: node_js
+node_js:
+  - "10"
+
 jdk:
   - oraclejdk8
 

pom.xml

@@ -123,6 +123,7 @@
         <module>aws-common</module>
         <module>extendedset</module>
         <module>hll</module>
+        <module>web-console</module>
         <!-- Core extensions -->
         <module>extensions-core/avro-extensions</module>
         <module>extensions-core/datasketches</module>
@@ -1279,6 +1280,15 @@
                         <target>${maven.compiler.target}</target>
                     </configuration>
                 </plugin>
+                <plugin>
+                    <groupId>com.github.eirslett</groupId>
+                    <artifactId>frontend-maven-plugin</artifactId>
+                    <version>1.6</version>
+                    <configuration>
+                        <nodeVersion>v10.13.0</nodeVersion>
+                        <npmVersion>6.4.1</npmVersion>
+                    </configuration>
+                </plugin>
             </plugins>
         </pluginManagement>
     </build>

server/pom.xml

@@ -43,9 +43,9 @@
             <version>${project.parent.version}</version>
         </dependency>
         <dependency>
-            <groupId>io.druid</groupId>
+            <groupId>org.apache.druid</groupId>
             <artifactId>druid-console</artifactId>
-            <version>0.0.4</version>
+            <version>${project.parent.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.druid</groupId>

services/src/main/java/org/apache/druid/cli/CliOverlord.java

@@ -114,6 +114,7 @@
 import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
 import org.eclipse.jetty.util.resource.ResourceCollection;
 
 import java.util.List;
@@ -130,6 +131,7 @@ public class CliOverlord extends ServerRunnable
 
   protected static List<String> UNSECURED_PATHS = ImmutableList.of(
       "/",
+      "/favicon.png",
       "/console.html",
       "/old-console/*",
       "/images/*",
@@ -353,17 +355,14 @@ public void initialize(Server server, Injector injector)
       final ServletContextHandler root = new ServletContextHandler(ServletContextHandler.SESSIONS);
       root.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
       root.setInitParameter("org.eclipse.jetty.servlet.Default.redirectWelcome", "true");
-      root.setWelcomeFiles(new String[]{"index.html", "console.html"});
+      root.setWelcomeFiles(new String[]{"console.html"});
 
       ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
 
       root.addServlet(holderPwd, "/");
       root.setBaseResource(
           new ResourceCollection(
-              new String[]{
-                  TaskMaster.class.getClassLoader().getResource("static").toExternalForm(),
-                  TaskMaster.class.getClassLoader().getResource("indexer_static").toExternalForm()
-              }
+              Resource.newClassPathResource("org/apache/druid/console")
           )
       );
 

services/src/main/java/org/apache/druid/cli/CliRouter.java

@@ -98,9 +98,7 @@ public void configure(Binder binder)
 
             binder.bind(TieredBrokerHostSelector.class).in(ManageLifecycle.class);
             binder.bind(QueryHostFinder.class).in(LazySingleton.class);
-            binder.bind(new TypeLiteral<List<TieredBrokerSelectorStrategy>>()
-            {
-            })
+            binder.bind(new TypeLiteral<List<TieredBrokerSelectorStrategy>>() {})
                   .toProvider(TieredBrokerSelectorStrategiesProvider.class)
                   .in(LazySingleton.class);
 

services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java

@@ -26,7 +26,6 @@
 import com.google.inject.Key;
 import com.google.inject.servlet.GuiceFilter;
 import org.apache.druid.guice.annotations.Json;
-import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.server.coordinator.DruidCoordinatorConfig;
 import org.apache.druid.server.http.OverlordProxyServlet;
 import org.apache.druid.server.http.RedirectFilter;
@@ -45,7 +44,6 @@
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.resource.Resource;
-import org.eclipse.jetty.util.resource.ResourceCollection;
 
 import java.util.List;
 import java.util.Properties;
@@ -55,32 +53,26 @@
 class CoordinatorJettyServerInitializer implements JettyServerInitializer
 {
   private static List<String> UNSECURED_PATHS = Lists.newArrayList(
-      "/favicon.ico",
-      "/css/*",
-      "/druid.js",
-      "/druid.css",
+      "/favicon.png",
       "/pages/*",
-      "/fonts/*",
+      "/coordinator-console/*",
+      "/index.html",
       "/old-console/*",
       "/coordinator/false",
       "/overlord/false",
       "/status/health",
       "/druid/coordinator/v1/isLeader"
   );
 
-  private static Logger log = new Logger(CoordinatorJettyServerInitializer.class);
-
   private final DruidCoordinatorConfig config;
   private final boolean beOverlord;
-  private final AuthConfig authConfig;
   private final ServerConfig serverConfig;
 
   @Inject
-  CoordinatorJettyServerInitializer(DruidCoordinatorConfig config, Properties properties, AuthConfig authConfig, ServerConfig serverConfig)
+  CoordinatorJettyServerInitializer(DruidCoordinatorConfig config, Properties properties, ServerConfig serverConfig)
   {
     this.config = config;
     this.beOverlord = CliCoordinator.isOverlord(properties);
-    this.authConfig = authConfig;
     this.serverConfig = serverConfig;
   }
 
@@ -89,25 +81,15 @@ public void initialize(Server server, Injector injector)
   {
     final ServletContextHandler root = new ServletContextHandler(ServletContextHandler.SESSIONS);
     root.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
+    root.setInitParameter("org.eclipse.jetty.servlet.Default.redirectWelcome", "true");
+    // index.html is the welcome file for old-console
+    root.setWelcomeFiles(new String[]{"index.html"});
 
     ServletHolder holderPwd = new ServletHolder("default", DefaultServlet.class);
 
     root.addServlet(holderPwd, "/");
     if (config.getConsoleStatic() == null) {
-      ResourceCollection staticResources;
-      if (beOverlord) {
-        staticResources = new ResourceCollection(
-            Resource.newClassPathResource("io/druid/console"),
-            Resource.newClassPathResource("static"),
-            Resource.newClassPathResource("indexer_static")
-        );
-      } else {
-        staticResources = new ResourceCollection(
-            Resource.newClassPathResource("io/druid/console"),
-            Resource.newClassPathResource("static")
-        );
-      }
-      root.setBaseResource(staticResources);
+      root.setBaseResource(Resource.newClassPathResource("org/apache/druid/console"));
     } else {
       // used for console development
       root.setResourceBase(config.getConsoleStatic());

services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java

@@ -20,6 +20,7 @@
 package org.apache.druid.cli;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
@@ -46,6 +47,7 @@
 import org.eclipse.jetty.servlet.DefaultServlet;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
 
 import javax.servlet.Servlet;
 import java.util.List;
@@ -60,6 +62,18 @@ public class RouterJettyServerInitializer implements JettyServerInitializer
       DruidAvaticaHandler.AVATICA_PATH
   );
 
+  protected static List<String> UNSECURED_PATHS_FOR_UI = ImmutableList.of(
+      "/",
+      "/coordinator-console/*",
+      "/public/*",
+      "/old-console/*",
+      "/pages/*",
+      "/unified-console.html",
+      "/favicon.png",
+      "/console.html",
+      "/index.html"
+  );
+
   private final DruidHttpClientConfig routerHttpClientConfig;
   private final DruidHttpClientConfig globalHttpClientConfig;
   private final ManagementProxyConfig managementProxyConfig;
@@ -92,6 +106,10 @@ public RouterJettyServerInitializer(
   public void initialize(Server server, Injector injector)
   {
     final ServletContextHandler root = new ServletContextHandler(ServletContextHandler.SESSIONS);
+    root.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
+    root.setInitParameter("org.eclipse.jetty.servlet.Default.redirectWelcome", "true");
+    // index.html is the welcome file for old-console
+    root.setWelcomeFiles(new String[]{"unified-console.html", "index.html"});
 
     root.addServlet(new ServletHolder(new DefaultServlet()), "/*");
 
@@ -107,13 +125,20 @@ public void initialize(Server server, Injector injector)
       root.addServlet(managementForwardingServletHolder, "/proxy/*");
     }
 
+    if (managementProxyConfig.isEnabled()) {
+      root.setBaseResource(Resource.newClassPathResource("org/apache/druid/console"));
+    }
+
     final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
     final AuthenticatorMapper authenticatorMapper = injector.getInstance(AuthenticatorMapper.class);
 
     AuthenticationUtils.addSecuritySanityCheckFilter(root, jsonMapper);
 
     // perform no-op authorization for these resources
     AuthenticationUtils.addNoopAuthorizationFilters(root, UNSECURED_PATHS);
+    if (managementProxyConfig.isEnabled()) {
+      AuthenticationUtils.addNoopAuthorizationFilters(root, UNSECURED_PATHS_FOR_UI);
+    }
     AuthenticationUtils.addNoopAuthorizationFilters(root, authConfig.getUnsecuredPaths());
 
     final List<Authenticator> authenticators = authenticatorMapper.getAuthenticatorChain();

web-console/.gitignore

@@ -0,0 +1,13 @@
+node/
+node_modules/
+resources/
+public/
+lib/*.css
+
+coordinator-console/
+pages/
+index.html
+
+.tscache
+tscommand-*.tmp.txt
+

web-console/README.md

@@ -0,0 +1,65 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+# Apache Druid web console
+
+This is the unified Druid web console that servers as a data management layer for Druid.
+
+## How to watch and run for development
+
+1. You need to be withing the `web-console` directory
+2. Install the modules with `npm install`
+3. Run `npm start` will start in development mode and will proxy druid requests to `localhost:8888`
+
+**Note:** you can provide an environment variable to proxy to a different Druid host like so: `druid_host=1.2.3.4:8888 npm start` 
+
+
+## Description of the directory structure
+
+A lot of the directory structure was created to preserve the existing console structure as much as possible.
+
+As part of this repo:
+
+- `console.html` - Entry file for the overlord console
+- `lib/` - A place where some overrides to the react-table stylus files live, this is outside of the normal SCSS build system.
+- `old-console/` - Files for the overlord console
+- `public/` - The compiled destination of the file powering this console
+- `script/` - Some helper bash scripts for running this console
+- `src/` - This directory (together with `lib`) constitutes all the source code for this console 
+
+Generated/copied dynamically
+
+- `index.html` - Entry file for the coordinator console
+- `pages/` - The files for the older coordinator console
+- `coordinator-console/` - Files for the coordinator console
+
+
+## List of non SQL data reading APIs used
+
+```
+GET /status
+GET /druid/indexer/v1/supervisor?full
+GET /druid/indexer/v1/workers
+GET /druid/coordinator/v1/loadqueue?simple
+GET /druid/coordinator/v1/config
+GET /druid/coordinator/v1/metadata/datasources?includeDisabled
+GET /druid/coordinator/v1/rules
+GET /druid/coordinator/v1/config/compaction
+GET /druid/coordinator/v1/tiers
+```