Merge pull request #2734 from b-slim/LookupIntrospection2

[QTL][Lookup] adding introspection endpoint

docs/content/querying/lookups.md

@@ -301,3 +301,9 @@ It is possible to save the configuration across restarts such that a node will n
 |Property|Description|Default|
 |--------|-----------|-------|
 |`druid.lookup.snapshotWorkingDir`| Working path used to store snapshot of current lookup configuration, leaving this property null will disable snapshot/bootstrap utility|null|
+
+## Introspect a Lookup
+
+Lookup implementations can provide some introspection capabilities by implementing `LookupIntrospectHandler`. User will send request to `/druid/lookups/v1/introspect/{lookupId}` to enable introspection on a given lookup.
+
+For instance you can list all the keys/values of a map based lookup by issuing a `GET` request to `/druid/lookups/v1/introspect/{lookupId}/keys"` or `/druid/lookups/v1/introspect/{lookupId}/values"` 

processing/src/main/java/io/druid/query/lookup/LookupExtractorFactory.java

@@ -19,10 +19,8 @@
 
 package io.druid.query.lookup;
 
-import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.Supplier;
-import io.druid.query.extraction.MapLookupExtractorFactory;
 
 import javax.annotation.Nullable;
 
@@ -32,9 +30,6 @@
  * If a LookupExtractorFactory wishes to support idempotent updates, it needs to implement the  `replaces` method
  */
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
-@JsonSubTypes(value = {
-    @JsonSubTypes.Type(name = "map", value = MapLookupExtractorFactory.class)
-})
 public interface LookupExtractorFactory extends Supplier<LookupExtractor>
 {
   /**
@@ -55,12 +50,18 @@ public interface LookupExtractorFactory extends Supplier<LookupExtractor>
    * @return true if successfully closed the {@link LookupExtractor}
    */
   public boolean close();
-
   /**
    * Determine if this LookupExtractorFactory should replace some other LookupExtractorFactory.
    * This is used to implement no-down-time
    * @param other Some other LookupExtractorFactory which might need replaced
    * @return `true` if the other should be replaced by this one. `false` if this one should not replace the other factory
    */
   boolean replaces(@Nullable LookupExtractorFactory other);
+
+  /**
+   * @return Returns the actual introspection request handler, can return {@code null} if it is not supported.
+   * This will be called once per HTTP request to introspect the actual lookup.
+   */
+  @Nullable
+  public LookupIntrospectHandler getIntrospectHandler();
 }

processing/src/main/java/io/druid/query/lookup/LookupIntrospectHandler.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.lookup;
+
+
+/**
+ * This interface is empty because it only exists to signal intent. The actual http endpoints are provided
+ * through JAX-RS annotations on the {@link LookupIntrospectHandler} objects.
+ * Note that, if you decide to implement {@link LookupExtractorFactory#getIntrospectHandler()} as request scoped, therefore {@link LookupIntrospectHandler} should have as light of a footprint as possible.
+ */
+public interface LookupIntrospectHandler
+{
+}

processing/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java

@@ -246,6 +246,13 @@ public boolean replaces(@Nullable LookupExtractorFactory other)
         return false;
       }
 
+      @Nullable
+      @Override
+      public LookupIntrospectHandler getIntrospectHandler()
+      {
+        return null;
+      }
+
       @Override
       public LookupExtractor get()
       {

server/pom.xml

@@ -207,6 +207,24 @@
             <artifactId>caliper</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.sun.jersey.jersey-test-framework</groupId>
+            <artifactId>jersey-test-framework-core</artifactId>
+            <version>1.19</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.sun.jersey.jersey-test-framework</groupId>
+            <artifactId>jersey-test-framework-grizzly2</artifactId>
+            <version>1.19</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>pl.pragmatists</groupId>
+            <artifactId>JUnitParams</artifactId>
+            <version>1.0.4</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

server/src/main/java/io/druid/query/lookup/LookupIntrospectionResource.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.lookup;
+
+import com.google.inject.Inject;
+import com.metamx.common.logger.Logger;
+
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
+@Path("/druid/v1/lookups/introspect")
+public class LookupIntrospectionResource
+{
+  private static final Logger LOGGER = new Logger(LookupIntrospectionResource.class);
+
+  private final LookupReferencesManager lookupReferencesManager;
+
+  @Inject
+  public LookupIntrospectionResource(@Context LookupReferencesManager lookupReferencesManager)
+  {
+    this.lookupReferencesManager = lookupReferencesManager;
+  }
+
+  @Path("/{lookupId}")
+  public Object introspectLookup(@PathParam("lookupId") final String lookupId)
+  {
+    final LookupExtractorFactory lookupExtractorFactory = lookupReferencesManager.get(lookupId);
+    if (lookupExtractorFactory == null) {
+      LOGGER.error("trying to introspect non existing lookup [%s]", lookupId);
+      return Response.status(Response.Status.NOT_FOUND).build();
+    }
+    LookupIntrospectHandler introspectHandler = lookupExtractorFactory.getIntrospectHandler();
+    if (introspectHandler != null) {
+      return introspectHandler;
+    } else {
+      LOGGER.warn(
+          "Trying to introspect lookup [%s] of type [%s] but implementation doesn't provide resource",
+          lookupId,
+          lookupExtractorFactory.get().getClass()
+      );
+      return Response.status(Response.Status.NOT_FOUND).build();
+    }
+  }
+}

server/src/main/java/io/druid/query/lookup/LookupModule.java

@@ -25,6 +25,8 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.net.HostAndPort;
 import com.google.inject.Binder;
@@ -50,7 +52,6 @@
 import org.apache.curator.utils.ZKPaths;
 
 import javax.ws.rs.Path;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -68,7 +69,9 @@ public static String getTierListenerPath(String tier)
   @Override
   public List<? extends Module> getJacksonModules()
   {
-    return Collections.emptyList();
+    return ImmutableList.<Module>of(
+        new SimpleModule("DruidLookupModule").registerSubtypes(MapLookupExtractorFactory.class)
+    );
   }
 
   @Override
@@ -78,6 +81,7 @@ public void configure(Binder binder)
     LifecycleModule.register(binder, LookupReferencesManager.class);
     JsonConfigProvider.bind(binder, PROPERTY_BASE, LookupListeningAnnouncerConfig.class);
     Jerseys.addResource(binder, LookupListeningResource.class);
+    Jerseys.addResource(binder, LookupIntrospectionResource.class);
     LifecycleModule.register(binder, LookupResourceListenerAnnouncer.class);
   }
 }

processing/src/main/java/io/druid/query/extraction/MapLookupExtractorFactory.java → server/src/main/java/io/druid/query/lookup/MapLookupExtractorFactory.java

@@ -1,40 +1,47 @@
 /*
  * Licensed to Metamarkets Group Inc. (Metamarkets) under one
- * or more contributor license agreements.  See the NOTICE file
+ * or more contributor license agreements. See the NOTICE file
  * distributed with this work for additional information
- * regarding copyright ownership.  Metamarkets licenses this file
+ * regarding copyright ownership. Metamarkets licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
+ * with the License. You may obtain a copy of the License at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
+ * KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations
  * under the License.
  */
 
-package io.druid.query.extraction;
+package io.druid.query.lookup;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.base.Preconditions;
-import io.druid.query.lookup.LookupExtractor;
-import io.druid.query.lookup.LookupExtractorFactory;
+import io.druid.query.extraction.MapLookupExtractor;
 
 import javax.annotation.Nullable;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
 import java.util.Map;
 
+@JsonTypeName("map")
 public class MapLookupExtractorFactory implements LookupExtractorFactory
 {
   @JsonProperty
   private final Map<String, String> map;
   @JsonProperty
   private final boolean isOneToOne;
   private final MapLookupExtractor lookupExtractor;
+  private final LookupIntrospectHandler lookupIntrospectHandler;
 
   @JsonCreator
   public MapLookupExtractorFactory(
@@ -45,6 +52,7 @@ public MapLookupExtractorFactory(
     this.map = Preconditions.checkNotNull(map, "map cannot be null");
     this.isOneToOne = isOneToOne;
     this.lookupExtractor = new MapLookupExtractor(map, isOneToOne);
+    this.lookupIntrospectHandler = new MapLookupIntrospectionHandler(this.map);
   }
 
   @Override
@@ -63,6 +71,7 @@ public boolean close()
    * For MapLookups, the replaces consideration is very easy, it simply considers if the other is the same as this one
    *
    * @param other Some other LookupExtractorFactory which might need replaced
+   *
    * @return true - should replace,   false - should not replace
    */
   @Override
@@ -71,6 +80,13 @@ public boolean replaces(@Nullable LookupExtractorFactory other)
     return !equals(other);
   }
 
+  @Nullable
+  @Override
+  public LookupIntrospectHandler getIntrospectHandler()
+  {
+    return lookupIntrospectHandler;
+  }
+
   @Override
   public LookupExtractor get()
   {
@@ -103,4 +119,34 @@ public int hashCode()
     result = 31 * result + (isOneToOne ? 1 : 0);
     return result;
   }
+
+  public static class MapLookupIntrospectionHandler implements LookupIntrospectHandler
+  {
+    final private Map<String, String> map;
+    public MapLookupIntrospectionHandler(Map<String, String> map)
+    {
+      this.map = map;
+    }
+
+    @GET
+    @Path("/keys")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getKeys()
+    {
+      return Response.ok(map.keySet().toString()).build();
+    }
+
+    @GET
+    @Path("/values")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getValues()
+    {
+      return Response.ok(map.values().toString()).build();
+    }
+
+    @GET
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getMap()
+    {return Response.ok(map).build();}
+  }
 }

processing/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java → server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java

@@ -26,9 +26,9 @@
 import io.druid.jackson.DefaultObjectMapper;
 import io.druid.query.extraction.ExtractionFn;
 import io.druid.query.extraction.MapLookupExtractor;
-import io.druid.query.extraction.MapLookupExtractorFactory;
 import io.druid.query.lookup.LookupExtractor;
 import io.druid.query.lookup.LookupReferencesManager;
+import io.druid.query.lookup.MapLookupExtractorFactory;
 import junitparams.JUnitParamsRunner;
 import junitparams.Parameters;
 import org.easymock.EasyMock;