Beginning to setup querying for new elasticsearch indexing template.

With a more efficient approach to querying hierarchy info.

src/api-umbrella/web-app/app/controllers/api/v1/analytics_controller.rb

@@ -22,7 +22,7 @@ def drilldown
     @search.aggregate_by_drilldown!(params[:prefix], drilldown_size)
 
     if(request.format != "csv")
-      @search.aggregate_by_drilldown_over_time!(params[:prefix])
+      @search.aggregate_by_drilldown_over_time!
     end
 
     @result = @search.result
@@ -55,9 +55,17 @@ def drilldown
 
         if @result.aggregations
           @result.aggregations["top_path_hits_over_time"]["buckets"].each do |bucket|
+            if ApiUmbrellaConfig[:elasticsearch][:template_version] < 2
+              id = bucket["key"]
+              label = bucket["key"].split("/", 2).last
+            else
+              id = File.join([@search.drilldown_depth.to_s, @search.drilldown_parent, bucket["key"]].compact)
+              label = File.join([@search.drilldown_parent, bucket["key"]].compact)
+            end
+
             @hits_over_time[:cols] << {
-              :id => bucket["key"],
-              :label => bucket["key"].split("/", 2).last,
+              :id => id,
+              :label => label,
               :type => "number",
             }
           end

src/api-umbrella/web-app/app/models/log_result/base.rb

@@ -37,13 +37,19 @@ def drilldown
       @drilldown = []
 
       if(aggregations && aggregations["drilldown"])
+        depth = @search.drilldown_depth
+        descendent_depth = depth + 1
+
         aggregations["drilldown"]["buckets"].each do |bucket|
-          depth, path = bucket["key"].split("/", 2)
-          terminal = !path.end_with?("/")
+          if ApiUmbrellaConfig[:elasticsearch][:template_version] < 2
+            parts = bucket["key"].split("/", 2)
+            path = parts[1]
+          else
+            path = File.join([@search.drilldown_parent, bucket["key"]].compact)
+          end
 
-          depth = depth.to_i
-          descendent_depth = depth + 1
           descendent_prefix = File.join(descendent_depth.to_s, path)
+          terminal = !path.end_with?("/")
 
           @drilldown << {
             :depth => depth,

src/api-umbrella/web-app/app/models/log_search/elastic_search.rb

@@ -1,7 +1,7 @@
 require("json")
 
 class LogSearch::ElasticSearch < LogSearch::Base
-  attr_reader :client
+  attr_reader :client, :drilldown_depth, :drilldown_parent
 
   def initialize(options = {})
     super
@@ -294,28 +294,62 @@ def filter_by_user_ids!(user_ids)
   end
 
   def aggregate_by_drilldown!(prefix, size = nil)
+    prefix_parts = prefix.split("/")
+    @drilldown_prefix = prefix
+    @drilldown_depth = prefix[0].to_i
+    @drilldown_parent = []
+    @drilldown_path_segments = []
+
+    prefix_parts.each_with_index do |value, index|
+      if index > 0
+        @drilldown_path_segments << {
+          :level => index - 1,
+          :value => value,
+        }
+
+        if index <= @drilldown_depth
+          @drilldown_parent << value
+        end
+      end
+    end
+    @drilldown_parent = File.join(@drilldown_parent)
+    if @drilldown_parent == ""
+      @drilldown_parent = nil
+    end
+
     size ||= 1_000_000
     @query[:aggregations][:drilldown] = {
       :terms => {
-        :field => "request_hierarchy",
         :size => size,
-        :include => "#{Regexp.escape(prefix)}.*",
       },
     }
-  end
 
-  def aggregate_by_drilldown_over_time!(prefix)
-    @query[:query][:bool][:filter][:bool][:must] << {
-      :prefix => {
-        :request_hierarchy => prefix,
-      },
-    }
+    if ApiUmbrellaConfig[:elasticsearch][:template_version] < 2
+      @query[:query][:bool][:filter][:bool][:must] << {
+        :prefix => {
+          :request_hierarchy => @drilldown_prefix,
+        },
+      }
+
+      @query[:aggregations][:drilldown][:terms][:field] = "request_hierarchy"
+      @query[:aggregations][:drilldown][:terms][:include] = "#{Regexp.escape(@drilldown_prefix)}.*"
+    else
+      @drilldown_path_segments.each do |segment|
+        @query[:query][:bool][:filter][:bool][:must] << {
+          :term => {
+            "request_url_hierarchy_level#{segment[:level]}" => "#{segment[:value]}/",
+          },
+        }
+      end
 
+      @query[:aggregations][:drilldown][:terms][:field] = "request_url_hierarchy_level#{@drilldown_depth}"
+    end
+  end
+
+  def aggregate_by_drilldown_over_time!
     @query[:aggregations][:top_path_hits_over_time] = {
       :terms => {
-        :field => "request_hierarchy",
         :size => 10,
-        :include => "#{Regexp.escape(prefix)}.*",
       },
       :aggregations => {
         :drilldown_over_time => {
@@ -334,6 +368,13 @@ def aggregate_by_drilldown_over_time!(prefix)
       },
     }
 
+    if ApiUmbrellaConfig[:elasticsearch][:template_version] < 2
+      @query[:aggregations][:top_path_hits_over_time][:terms][:field] = "request_hierarchy"
+      @query[:aggregations][:top_path_hits_over_time][:terms][:include] = "#{Regexp.escape(@drilldown_prefix)}.*"
+    else
+      @query[:aggregations][:top_path_hits_over_time][:terms][:field] = "request_url_hierarchy_level#{@drilldown_depth}"
+    end
+
     @query[:aggregations][:hits_over_time] = {
       :date_histogram => {
         :field => "request_at",

templates/etc/rsyslog.d/analytics.conf.mustache

@@ -39,9 +39,11 @@ if($!raw!request_connection != "") then {
 if($!raw!request_content_type != "") then {
   set $!usr!es!request_content_type = $!raw!request_content_type;
 }
+{{#elasticsearch._template_version_v1?}}
 if($!raw!request_url_hierarchy != "") then {
   set $!usr!es!request_hierarchy = $!raw!request_url_hierarchy;
 }
+{{/elasticsearch._template_version_v1?}}
 {{#elasticsearch._template_version_v2?}}
 if($!raw!request_url_hierarchy_level0 != "") then {
   set $!usr!es!request_url_hierarchy_level0 = $!raw!request_url_hierarchy_level0;