From a816733119da845dfa80c753a9c9fb38dd9e3aaf Mon Sep 17 00:00:00 2001 From: ppisljar Date: Mon, 23 Jan 2017 22:05:04 +0100 Subject: [PATCH 1/6] updating agg_config to allow adding parent aggregations --- src/ui/public/vis/agg_config.js | 7 +++++++ src/ui/public/vis/agg_configs.js | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/ui/public/vis/agg_config.js b/src/ui/public/vis/agg_config.js index d2852f2b2af88e..2be2834bd078ed 100644 --- a/src/ui/public/vis/agg_config.js +++ b/src/ui/public/vis/agg_config.js @@ -217,6 +217,13 @@ export default function AggConfigFactory(Private, fieldTypeFilter) { }); } + if (output.parentAggs) { + const subDslLvl = configDsl.parentAggs || (configDsl.parentAggs = {}); + output.parentAggs.forEach(function nestAdhocSubAggs(subAggConfig) { + subDslLvl[subAggConfig.id] = subAggConfig.toDsl(); + }); + } + return configDsl; }; diff --git a/src/ui/public/vis/agg_configs.js b/src/ui/public/vis/agg_configs.js index 08cbc266a0efc1..e6a7fe493b1036 100644 --- a/src/ui/public/vis/agg_configs.js +++ b/src/ui/public/vis/agg_configs.js @@ -114,6 +114,15 @@ export default function AggConfigsFactory(Private) { const dsl = dslLvlCursor[config.id] = config.toDsl(); let subAggs; + ((function parseParentAggs(dsl) { + if (dsl.parentAggs) { + _.each(dsl.parentAggs, (agg, key) => { + dslLvlCursor[key] = agg; + parseParentAggs(agg); + }); + } + })(dsl)); + if (config.schema.group === 'buckets' && i < list.length - 1) { // buckets that are not the last item in the list accept sub-aggs subAggs = dsl.aggs || (dsl.aggs = {}); @@ -126,6 +135,15 @@ export default function AggConfigsFactory(Private) { } }); + function removeParentAggs(obj) { + for(const prop in obj) { + if (prop === 'parentAggs') delete obj[prop]; + else if (typeof obj[prop] === 'object') removeParentAggs(obj[prop]); + } + } + + removeParentAggs(dslTopLvl); + return dslTopLvl; }; From 65043472c0875869fbc290aacb4c1c961fdca4d2 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Mon, 23 Jan 2017 22:05:51 +0100 Subject: [PATCH 2/6] derivative metric aggregation --- src/ui/public/agg_types/controls/sub_agg.html | 28 ++++ src/ui/public/agg_types/index.js | 4 +- src/ui/public/agg_types/metrics/derivative.js | 142 ++++++++++++++++++ 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 src/ui/public/agg_types/controls/sub_agg.html create mode 100644 src/ui/public/agg_types/metrics/derivative.js diff --git a/src/ui/public/agg_types/controls/sub_agg.html b/src/ui/public/agg_types/controls/sub_agg.html new file mode 100644 index 00000000000000..bd552880e911f5 --- /dev/null +++ b/src/ui/public/agg_types/controls/sub_agg.html @@ -0,0 +1,28 @@ +
+
+ + +
+ + + + +
diff --git a/src/ui/public/agg_types/index.js b/src/ui/public/agg_types/index.js index 4409b6a1fbda3c..25140ba77daf08 100644 --- a/src/ui/public/agg_types/index.js +++ b/src/ui/public/agg_types/index.js @@ -11,6 +11,7 @@ import AggTypesMetricsStdDeviationProvider from 'ui/agg_types/metrics/std_deviat import AggTypesMetricsCardinalityProvider from 'ui/agg_types/metrics/cardinality'; import AggTypesMetricsPercentilesProvider from 'ui/agg_types/metrics/percentiles'; import AggTypesMetricsPercentileRanksProvider from 'ui/agg_types/metrics/percentile_ranks'; +import AggTypesMetricsDerivativeProvider from 'ui/agg_types/metrics/derivative'; import AggTypesBucketsDateHistogramProvider from 'ui/agg_types/buckets/date_histogram'; import AggTypesBucketsHistogramProvider from 'ui/agg_types/buckets/histogram'; import AggTypesBucketsRangeProvider from 'ui/agg_types/buckets/range'; @@ -34,7 +35,8 @@ export default function AggTypeService(Private) { Private(AggTypesMetricsCardinalityProvider), Private(AggTypesMetricsPercentilesProvider), Private(AggTypesMetricsPercentileRanksProvider), - Private(AggTypesMetricsTopHitProvider) + Private(AggTypesMetricsTopHitProvider), + Private(AggTypesMetricsDerivativeProvider), ], buckets: [ Private(AggTypesBucketsDateHistogramProvider), diff --git a/src/ui/public/agg_types/metrics/derivative.js b/src/ui/public/agg_types/metrics/derivative.js new file mode 100644 index 00000000000000..33700b8356437e --- /dev/null +++ b/src/ui/public/agg_types/metrics/derivative.js @@ -0,0 +1,142 @@ +import AggTypesMetricsMetricAggTypeProvider from 'ui/agg_types/metrics/metric_agg_type'; +import orderAggTemplate from 'ui/agg_types/controls/sub_agg.html'; +import _ from 'lodash'; +import $ from 'jquery'; +import VisAggConfigProvider from 'ui/vis/agg_config'; +import VisSchemasProvider from 'ui/vis/schemas'; + +export default function AggTypeMetricDerivativeProvider(Private) { + const DerivativeAggType = Private(AggTypesMetricsMetricAggTypeProvider); + const AggConfig = Private(VisAggConfigProvider); + const Schemas = Private(VisSchemasProvider); + + const aggFilter = ['!top_hits', '!percentiles', '!median', '!std_dev']; + const orderAggSchema = (new Schemas([ + { + group: 'none', + name: 'orderAgg', + title: 'Order Agg', + aggFilter: aggFilter + } + ])).all[0]; + + return new DerivativeAggType({ + name: 'derivative', + title: 'Derivative', + makeLabel: function (aggConfig) { + if (aggConfig.params.customMetric) { + let label = aggConfig.params.customMetric.makeLabel(); + if (label.includes('Derivative of ')) { + label = '2. derivative of ' + label.substring('Derivative of '.length); + } + else if (label.includes('derivative of ')) { + label = (parseInt(label.substring(0, 1)) + 1) + label.substring(1); + } + else { + label = 'Derivative of ' + label; + } + return label; + } + const metric = aggConfig.vis.aggs.find(agg => agg.id === aggConfig.params.buckets_path); + return 'Derivative of ' + metric.makeLabel(); + }, + params: [ + { + name: 'customMetric', + type: AggConfig, + default: null, + serialize: function (customMetric) { + return customMetric.toJSON(); + }, + deserialize: function (state, agg) { + return this.makeAgg(agg, state); + }, + makeAgg: function (termsAgg, state) { + state = state || {}; + state.schema = orderAggSchema; + const orderAgg = new AggConfig(termsAgg.vis, state); + orderAgg.id = termsAgg.id + '-orderAgg'; + return orderAgg; + }, + write: _.noop + }, + { + name: 'buckets_path', + editor: orderAggTemplate, + controller: function ($scope, $element) { + + $scope.safeMakeLabel = function (agg) { + try { + return agg.makeLabel(); + } catch (e) { + return '- agg not valid -'; + } + }; + + $scope.$watch('responseValueAggs', updateOrderAgg); + $scope.$watch('agg.params.buckets_path', updateOrderAgg); + + $scope.$on('$destroy', function () { + if ($scope.aggForm && $scope.aggForm.agg) { + $scope.aggForm.agg.$setValidity('bucket', true); + } + }); + + // Returns true if the agg is not compatible with the terms bucket + $scope.rejectAgg = function (agg) { + // aggFilter elements all starts with a '!' + // so the index of agg.type.name in a filter is 1 if it is included + return Boolean(aggFilter.find((filter) => filter.indexOf(agg.type.name) === 1)); + }; + + function checkBuckets() { + const buckets = $scope.vis.aggs.filter(agg => agg.schema.group === 'buckets'); + const bucketIsHistogram = ['date_histogram', 'histogram'].includes(buckets[0].type.name); + const canUseDerivative = buckets.length === 1 && bucketIsHistogram; + if ($scope.aggForm.agg) $scope.aggForm.agg.$setValidity('bucket', canUseDerivative); + if (canUseDerivative) { + if (buckets[0].type.name === 'histogram') { + buckets[0].params.min_doc_count = 1; + } + else { + buckets[0].params.min_doc_count = 0; + } + } + } + + function updateOrderAgg() { + const agg = $scope.agg; + const params = agg.params; + const bucketsPath = params.buckets_path; + const paramDef = agg.type.params.byName.customMetric; + + checkBuckets(); + + // we aren't creating a custom aggConfig + if (bucketsPath !== 'custom') { + params.customMetric = null; + return; + } + + params.customMetric = params.customMetric || paramDef.makeAgg(agg); + } + }, + write: function (agg, output) { + const vis = agg.vis; + const orderAgg = agg.params.customMetric || vis.aggs.getResponseAggById(agg.params.buckets_path); + + if (agg.params.customMetric && agg.params.customMetric.type.name !== 'count') { + output.parentAggs = (output.parentAggs || []).concat(orderAgg); + } + + output.params = {}; + if (orderAgg.type.name === 'count') { + output.params.buckets_path = '_count'; + } else { + output.params.buckets_path = orderAgg.id; + } + } + } + ] + }); +} From 734cef1a1c52e080af909d3c32f793fff273c367 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Mon, 23 Jan 2017 22:06:13 +0100 Subject: [PATCH 3/6] disable on metric and tagcloud --- src/core_plugins/metric_vis/public/metric_vis.js | 1 + src/core_plugins/tagcloud/public/tag_cloud_vis.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core_plugins/metric_vis/public/metric_vis.js b/src/core_plugins/metric_vis/public/metric_vis.js index cd8eaef53a99f1..46f77abfd6152c 100644 --- a/src/core_plugins/metric_vis/public/metric_vis.js +++ b/src/core_plugins/metric_vis/public/metric_vis.js @@ -38,6 +38,7 @@ function MetricVisProvider(Private) { name: 'metric', title: 'Metric', min: 1, + aggFilter: ['!derivative'], defaults: [ { type: 'count', schema: 'metric' } ] diff --git a/src/core_plugins/tagcloud/public/tag_cloud_vis.js b/src/core_plugins/tagcloud/public/tag_cloud_vis.js index 9766d7934f81c9..1a5e4d79143dde 100644 --- a/src/core_plugins/tagcloud/public/tag_cloud_vis.js +++ b/src/core_plugins/tagcloud/public/tag_cloud_vis.js @@ -37,7 +37,7 @@ visTypes.register(function TagCloudProvider(Private) { title: 'Tag Size', min: 1, max: 1, - aggFilter: ['!std_dev', '!percentiles', '!percentile_ranks'], + aggFilter: ['!std_dev', '!percentiles', '!percentile_ranks', '!derivative'], defaults: [ { schema: 'metric', type: 'count' } ] From e6286fc873f6f3ddcec28be1d5dac664de260fa8 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Mon, 23 Jan 2017 22:06:39 +0100 Subject: [PATCH 4/6] fixing point series to correctly handle missing y values --- .../vislib/visualizations/point_series/area_chart.js | 12 ++++++++---- .../visualizations/point_series/column_chart.js | 5 +++-- .../vislib/visualizations/point_series/line_chart.js | 10 +++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/ui/public/vislib/visualizations/point_series/area_chart.js b/src/ui/public/vislib/visualizations/point_series/area_chart.js index 9fffb6b7113737..fc073de6c5ddfe 100644 --- a/src/ui/public/vislib/visualizations/point_series/area_chart.js +++ b/src/ui/public/vislib/visualizations/point_series/area_chart.js @@ -96,7 +96,8 @@ export default function AreaChartFactory(Private) { function y1(d) { const y0 = d.y0 || 0; - return yScale(y0 + d.y); + const y = d.y || 0; + return yScale(y0 + y); } function y0(d) { @@ -125,7 +126,9 @@ export default function AreaChartFactory(Private) { return !_.isNull(d.y); }) .interpolate(interpolate); - return area(data.values); + return area(data.values.filter(function (d) { + return !_.isNull(d.y); + })); }); return path; @@ -185,10 +188,11 @@ export default function AreaChartFactory(Private) { } function cy(d) { + const y = d.y || 0; if (isOverlapping) { - return yScale(d.y); + return yScale(y); } - return yScale(d.y0 + d.y); + return yScale(d.y0 + y); } // update diff --git a/src/ui/public/vislib/visualizations/point_series/column_chart.js b/src/ui/public/vislib/visualizations/point_series/column_chart.js index ba35923d9049cd..74b74212e554d9 100644 --- a/src/ui/public/vislib/visualizations/point_series/column_chart.js +++ b/src/ui/public/vislib/visualizations/point_series/column_chart.js @@ -1,5 +1,4 @@ import _ from 'lodash'; -import moment from 'moment'; import errors from 'ui/errors'; import VislibVisualizationsPointSeriesProvider from './_point_series'; export default function ColumnChartFactory(Private) { @@ -39,7 +38,9 @@ export default function ColumnChartFactory(Private) { .attr('clip-path', 'url(#' + this.baseChart.clipPathId + ')'); const bars = layer.selectAll('rect') - .data(data.values); + .data(data.values.filter(function (d) { + return !_.isNull(d.y); + })); bars .exit() diff --git a/src/ui/public/vislib/visualizations/point_series/line_chart.js b/src/ui/public/vislib/visualizations/point_series/line_chart.js index 37ddad59f4900d..9e629c429c126b 100644 --- a/src/ui/public/vislib/visualizations/point_series/line_chart.js +++ b/src/ui/public/vislib/visualizations/point_series/line_chart.js @@ -69,7 +69,8 @@ export default function LineChartFactory(Private) { } function cy(d) { - return yScale(d.y); + const y = d.y || 0; + return yScale(y); } function cColor(d) { @@ -156,7 +157,8 @@ export default function LineChartFactory(Private) { } function cy(d) { - return yScale(d.y); + const y = d.y || 0; + return yScale(y); } line.append('path') @@ -169,7 +171,9 @@ export default function LineChartFactory(Private) { .interpolate(interpolate) .x(isHorizontal ? cx : cy) .y(isHorizontal ? cy : cx); - return d3Line(data.values); + return d3Line(data.values.filter(function (d) { + return !_.isNull(d.y); + })); }) .attr('fill', 'none') .attr('stroke', () => { From 409def364638038eaec570d2cdc1d3b861fccc76 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Mon, 23 Jan 2017 22:07:01 +0100 Subject: [PATCH 5/6] adding unit tests --- .../agg_types/__tests__/metrics/derivative.js | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 src/ui/public/agg_types/__tests__/metrics/derivative.js diff --git a/src/ui/public/agg_types/__tests__/metrics/derivative.js b/src/ui/public/agg_types/__tests__/metrics/derivative.js new file mode 100644 index 00000000000000..4ccaa8bd428e4d --- /dev/null +++ b/src/ui/public/agg_types/__tests__/metrics/derivative.js @@ -0,0 +1,126 @@ +import _ from 'lodash'; +import expect from 'expect.js'; +import ngMock from 'ng_mock'; +import DerivativeProvider from 'ui/agg_types/metrics/derivative'; +import VisProvider from 'ui/vis'; +import StubbedIndexPattern from 'fixtures/stubbed_logstash_index_pattern'; + +describe('Derivative metric', function () { + let aggDsl; + let derivativeMetric; + let aggConfig; + + function init(settings) { + ngMock.module('kibana'); + ngMock.inject(function (Private) { + const Vis = Private(VisProvider); + const indexPattern = Private(StubbedIndexPattern); + derivativeMetric = Private(DerivativeProvider); + + const params = settings || { + buckets_path: '1', + customMetric: null + }; + + const vis = new Vis(indexPattern, { + title: 'New Visualization', + type: 'metric', + params: { + fontSize: 60, + handleNoResults: true + }, + aggs: [ + { + id: '1', + type: 'count', + schema: 'metric' + }, + { + id: '2', + type: 'derivative', + schema: 'metric', + params + } + ], + listeners: {} + }); + + // Grab the aggConfig off the vis (we don't actually use the vis for anything else) + aggConfig = vis.aggs[1]; + aggDsl = aggConfig.toDsl(); + }); + } + + it('should return a label prefixed with Derivative of', function () { + init(); + expect(derivativeMetric.makeLabel(aggConfig)).to.eql('Derivative of Count'); + }); + + it('should return a label Derivative of max bytes', function () { + init({ + buckets_path: 'custom', + customMetric: { + id:'1-orderAgg', + type: 'max', + params: { field: 'bytes' }, + schema: 'orderAgg' + } + }); + expect(derivativeMetric.makeLabel(aggConfig)).to.eql('Derivative of Max bytes'); + }); + + it('should return a label prefixed with number of derivative', function () { + init({ + buckets_path: 'custom', + customMetric: { + id:'2-orderAgg', + type: 'derivative', + params: { + buckets_path: 'custom', + customMetric: { + id:'2-orderAgg-orderAgg', + type: 'count', + schema: 'orderAgg' + } + }, + schema: 'orderAgg' + } + }); + expect(derivativeMetric.makeLabel(aggConfig)).to.eql('2. derivative of Count'); + }); + + it('should set parent aggs', function () { + init({ + buckets_path: 'custom', + customMetric: { + id:'2-orderAgg', + type: 'max', + params: { field: 'bytes' }, + schema: 'orderAgg' + } + }); + expect(aggDsl.parentAggs['2-orderAgg'].max.field).to.be('bytes'); + }); + + it('should set nested parent aggs', function () { + init({ + buckets_path: 'custom', + customMetric: { + id:'2-orderAgg', + type: 'derivative', + params: { + buckets_path: 'custom', + customMetric: { + id:'2-orderAgg-orderAgg', + type: 'max', + params: { field: 'bytes' }, + schema: 'orderAgg' + } + }, + schema: 'orderAgg' + } + }); + expect(aggDsl.parentAggs['2-orderAgg'].derivative.buckets_path).to.be('2-orderAgg-orderAgg'); + }); + +}); From 886f69b4c81d604e2ac8e61c424a6c665b7ac60d Mon Sep 17 00:00:00 2001 From: ppisljar Date: Tue, 24 Jan 2017 09:24:13 +0100 Subject: [PATCH 6/6] moving average metric agg --- src/ui/public/agg_types/controls/sub_agg.html | 2 +- src/ui/public/agg_types/index.js | 2 + src/ui/public/agg_types/metrics/moving_avg.js | 142 ++++++++++++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 src/ui/public/agg_types/metrics/moving_avg.js diff --git a/src/ui/public/agg_types/controls/sub_agg.html b/src/ui/public/agg_types/controls/sub_agg.html index bd552880e911f5..f8c399dc5ae5ac 100644 --- a/src/ui/public/agg_types/controls/sub_agg.html +++ b/src/ui/public/agg_types/controls/sub_agg.html @@ -9,7 +9,7 @@