Merge pull request elastic#7372 from elastic/feature/ingest

Add Data - CSV

package.json

@@ -68,6 +68,7 @@
   },
   "dependencies": {
     "@bigfunger/decompress-zip": "0.2.0-stripfix2",
+    "@bigfunger/jsondiffpatch": "0.1.38-webpack",
     "@elastic/datemath": "2.3.0",
     "@spalger/angular-bootstrap": "0.12.1",
     "@spalger/filesaver": "1.1.2",
@@ -76,6 +77,7 @@
     "@spalger/numeral": "^2.0.0",
     "@spalger/test-subj-selector": "0.2.1",
     "@spalger/ui-ace": "0.2.3",
+    "JSONStream": "1.1.1",
     "angular": "1.4.7",
     "angular-bootstrap-colorpicker": "3.0.19",
     "angular-elastic": "2.5.0",
@@ -95,6 +97,7 @@
     "clipboard": "1.5.5",
     "commander": "2.8.1",
     "css-loader": "0.17.0",
+    "csv-parse": "1.1.0",
     "d3": "3.5.6",
     "dragula": "3.7.0",
     "elasticsearch": "10.1.2",
@@ -110,6 +113,7 @@
     "good-squeeze": "2.1.0",
     "gridster": "0.5.6",
     "hapi": "8.8.1",
+    "highland": "2.7.2",
     "httpolyglot": "0.1.1",
     "imports-loader": "0.6.4",
     "jade": "1.11.0",
@@ -130,6 +134,7 @@
     "moment": "2.13.0",
     "moment-timezone": "0.5.4",
     "node-uuid": "1.4.7",
+    "papaparse": "4.1.2",
     "raw-loader": "0.5.1",
     "request": "2.61.0",
     "rimraf": "2.4.3",

src/cli/cluster/base_path_proxy.js

@@ -34,6 +34,9 @@ export default class BasePathProxy {
       config.set('server.basePath', this.basePath);
     }
 
+    const ONE_GIGABYTE = 1024 * 1024 * 1024;
+    config.set('server.maxPayloadBytes', ONE_GIGABYTE);
+
     setupLogging(null, this.server, config);
     setupConnection(null, this.server, config);
     this.setupRoutes();

src/plugins/kibana/common/lib/__tests__/convert_pattern_and_ingest_name.js

@@ -0,0 +1,31 @@
+import expect from 'expect.js';
+import {patternToIngest, ingestToPattern} from '../convert_pattern_and_ingest_name';
+
+describe('convertPatternAndTemplateName', function () {
+
+  describe('ingestToPattern', function () {
+
+    it('should convert an index template\'s name to its matching index pattern\'s title', function () {
+      expect(ingestToPattern('kibana-logstash-*')).to.be('logstash-*');
+    });
+
+    it('should throw an error if the template name isn\'t a valid kibana namespaced name', function () {
+      expect(ingestToPattern).withArgs('logstash-*').to.throwException('not a valid kibana namespaced template name');
+      expect(ingestToPattern).withArgs('').to.throwException(/not a valid kibana namespaced template name/);
+    });
+
+  });
+
+  describe('patternToIngest', function () {
+
+    it('should convert an index pattern\'s title to its matching index template\'s name', function () {
+      expect(patternToIngest('logstash-*')).to.be('kibana-logstash-*');
+    });
+
+    it('should throw an error if the pattern is empty', function () {
+      expect(patternToIngest).withArgs('').to.throwException(/pattern must not be empty/);
+    });
+
+  });
+
+});

src/plugins/kibana/server/lib/convert_pattern_and_template_name.js → src/plugins/kibana/common/lib/convert_pattern_and_ingest_name.js

@@ -4,15 +4,15 @@
 // This module provides utility functions for easily converting between template and pattern names.
 
 module.exports = {
-  templateToPattern: (templateName) => {
+  ingestToPattern: (templateName) => {
     if (templateName.indexOf('kibana-') === -1) {
       throw new Error('not a valid kibana namespaced template name');
     }
 
     return templateName.slice(templateName.indexOf('-') + 1);
   },
 
-  patternToTemplate: (patternName) => {
+  patternToIngest: (patternName) => {
     if (patternName === '') {
       throw new Error('pattern must not be empty');
     }

src/plugins/kibana/public/management/sections/indices/_create.html

@@ -62,6 +62,7 @@ <h4>Time-interval based index patterns are deprecated!</h4>
               ng-attr-placeholder="{{index.defaultName}}"
               ng-model-options="{ updateOn: 'default blur', debounce: {'default': 2500, 'blur': 0} }"
               validate-index-name
+              allow-wildcard
               name="name"
               required
               type="text"
@@ -167,6 +168,7 @@ <h4>Time-interval based index patterns are deprecated!</h4>
           </div>
 
           <button
+            data-test-subj="submitCreateIndexPatternFromExistingForm"
             ng-disabled="form.$invalid || index.fetchFieldsError"
             ng-class="index.fetchFieldsError ? 'btn-default' : 'btn-success'"
             type="submit"

src/plugins/kibana/public/management/sections/indices/_edit.html

@@ -1,6 +1,6 @@
 <kbn-management-app section="kibana">
   <kbn-management-indices>
-    <div ng-controller="managementIndicesEdit">
+    <div ng-controller="managementIndicesEdit" data-test-subj="editIndexPattern">
       <div class="page-header">
         <kbn-management-index-header
           index-pattern="indexPattern"

src/plugins/kibana/public/management/sections/indices/_edit.js

@@ -8,6 +8,7 @@ import IndicesFieldTypesProvider from 'plugins/kibana/management/sections/indice
 import uiRoutes from 'ui/routes';
 import uiModules from 'ui/modules';
 import editTemplate from 'plugins/kibana/management/sections/indices/_edit.html';
+import IngestProvider from 'ui/ingest';
 
 uiRoutes
 .when('/management/kibana/indices/:indexPatternId?', {
@@ -32,6 +33,7 @@ uiModules.get('apps/management')
   const notify = new Notifier();
   const $state = $scope.state = new AppState();
   const refreshKibanaIndex = Private(RefreshKibanaIndex);
+  const ingest = Private(IngestProvider);
 
   $scope.kbnUrl = Private(UrlProvider);
   $scope.indexPattern = $route.current.locals.indexPattern;
@@ -68,8 +70,8 @@ uiModules.get('apps/management')
       }
     }
 
-    courier.indexPatterns.delete($scope.indexPattern)
-    .then(refreshKibanaIndex)
+    ingest.delete($scope.indexPattern.id)
+    .then($scope.indexPattern.destroy.bind($scope.indexPattern))
     .then(function () {
       $location.url('/management/data/index');
     })

src/plugins/kibana/public/management/sections/indices/add_data_steps/install_filebeat_step/install_filebeat_step.html

@@ -0,0 +1,84 @@
+<h2><em>Follow these instructions to install Filebeat.</em>
+Now that you've got a fresh pipeline and index pattern, let's throw some data at it!
+</h2>
+
+<div class="install-filebeat">
+  <ol>
+    <li>
+      <span>
+        <strong>Install Filebeat</strong> on all servers on which you want to tail logs &nbsp;
+        <a target="_blank" ng-href="{{installStep.docLinks.installation}}">
+            <i aria-hidden="true" class="fa fa-info-circle"></i> instructions
+        </a>
+      </span>
+    </li>
+
+    <li>
+      <span>
+        <strong>Point Filebeat</strong> at the log files you want to tail &nbsp;
+        <a target="_blank" ng-href="{{installStep.docLinks.configuration}}">
+          <i aria-hidden="true" class="fa fa-info-circle"></i> instructions
+        </a>
+      </span>
+    </li>
+
+    <li ng-if="installStep.results.pipeline.processors.length">
+      <span>
+        <strong>Configure Filebeat</strong> to send data through your new Elasticsearch pipeline &nbsp;
+        <a target="_blank" ng-href="{{installStep.docLinks.elasticsearchOutput}}">
+          <i aria-hidden="true" class="fa fa-info-circle"></i> instructions
+        </a><br/>
+        At minimum you'll need to configure Filebeat's Elasticsearch output with a hostname, an index name, and a
+        <a target="_blank"
+           ng-href="{{installStep.docLinks.elasticsearchOutputAnchorParameters}}">
+          <i aria-hidden="true" class="fa fa-info-circle"></i> paramaters
+        </a> block. Your config should end up looking something like this:<br/>
+      <pre>
+output:
+ elasticsearch:
+   hosts: ["your-elasticsearch-host"]
+   index: "your-base-index-name"
+   parameters:
+     pipeline: "{{installStep.pipelineId}}"</pre>
+        <em>NOTE</em>: The Filebeat config takes a base index name and automatically rotates the target index by appending "-{date}"
+        to the end, so if your pattern was "filebeat-*" you would make the index name "filebeat" in filebeat.yml.<br />
+      </span>
+    </li>
+
+
+    <li ng-if="!installStep.results.pipeline.processors.length">
+      <span>
+        <strong>Configure Filebeat</strong> to send data to Elasticsearch &nbsp;
+        <a target="_blank" ng-href="{{installStep.docLinks.elasticsearchOutput}}">
+          <i aria-hidden="true" class="fa fa-info-circle"></i> instructions
+        </a><br/>
+        At minimum you'll need to configure Filebeat's Elasticsearch output with a hostname and an index name.
+        Your config should end up looking something like this:<br />
+        <pre>
+output:
+ elasticsearch:
+   hosts: ["your-elasticsearch-host"]
+   index: "your-base-index-name"</pre>
+      <em>NOTE</em>: The Filebeat config takes a base index name and automatically rotates the target index by appending "-{date}"
+      to the end, so if your pattern was "filebeat-*" you would make the index name "filebeat" in filebeat.yml.<br />
+      </span>
+    </li>
+
+    <li>
+      <span>
+        <strong>Run Filebeat</strong> on each server &nbsp;
+        <a target="_blank" ng-href="{{installStep.docLinks.startup}}">
+          <i aria-hidden="true" class="fa fa-info-circle"></i> instructions
+        </a>
+      </span>
+    </li>
+    <li>
+      <span>
+        <strong>Verify your filebeat installation below.</strong> We'll poll your new index pattern for documents and let you know when
+        they show up. If you'd like to skip this step, simply click Done now.
+      </span>
+    </li>
+  </ol>
+</div>
+
+<pattern-checker pattern="installStep.results.indexPattern.id"/>

src/plugins/kibana/public/management/sections/indices/add_data_steps/install_filebeat_step/install_filebeat_step.js

@@ -0,0 +1,23 @@
+import modules from 'ui/modules';
+import template from './install_filebeat_step.html';
+import 'ui/pattern_checker';
+import { patternToIngest } from '../../../../../../common/lib/convert_pattern_and_ingest_name';
+import { filebeat as docLinks } from '../../../../../../../../ui/public/documentation_links/documentation_links';
+import './styles/_add_data_install_filebeat_step.less';
+
+modules.get('apps/management')
+  .directive('installFilebeatStep', function () {
+    return {
+      template: template,
+      scope: {
+        results: '='
+      },
+      bindToController: true,
+      controllerAs: 'installStep',
+      controller: function ($scope) {
+        this.pipelineId = patternToIngest(this.results.indexPattern.id);
+        this.docLinks = docLinks;
+      }
+    };
+  });
+

src/plugins/kibana/public/management/sections/indices/add_data_steps/install_filebeat_step/styles/_add_data_install_filebeat_step.less

@@ -0,0 +1,22 @@
+install-filebeat-step {
+
+  .install-filebeat {
+    > ol {
+      padding-left: 1em;
+
+      > li {
+        padding: 4px 0;
+        font-weight: bold;
+
+        > span {
+          font-weight: normal;
+
+          > pre {
+            margin: 7px 0;
+          }
+        }
+      }
+    }
+  }
+
+}

src/plugins/kibana/public/management/sections/indices/add_data_steps/parse_csv_step/parse_csv_step.html

@@ -0,0 +1,63 @@
+<file-upload ng-if="!wizard.file" on-locate="wizard.file = file" upload-selector="button.upload">
+  <h2><em>Pick a CSV file to get started.</em>
+    Please follow the instructions below.
+  </h2>
+
+  <div class="upload-wizard-file-upload-container">
+    <div class="upload-instructions">Drop your file here</div>
+    <div class="upload-instructions-separator">or</div>
+    <button class="btn btn-primary btn-lg controls upload" ng-click>
+      Select File
+    </button>
+    <div>Maximum upload file size: 1 GB</div>
+  </div>
+</file-upload>
+
+<div class="upload-wizard-file-preview-container" ng-if="wizard.file">
+  <h2><em>Review the sample below.</em>
+    Click next if it looks like we parsed your file correctly.
+  </h2>
+
+  <div ng-if="!!wizard.formattedErrors.length" class="alert alert-danger parse-error">
+    <ul>
+      <li ng-repeat="error in wizard.formattedErrors track by $index">{{ error }}</li>
+    </ul>
+  </div>
+
+  <div ng-if="!!wizard.formattedWarnings.length" class="alert alert-warning">
+    <ul>
+      <li ng-repeat="warning in wizard.formattedWarnings track by $index">{{ warning }}</li>
+    </ul>
+  </div>
+
+  <div class="advanced-options form-inline">
+    <span class="form-group">
+      <label>Delimiter</label>
+      <select ng-model="wizard.parseOptions.delimiter"
+              ng-options="option.value as option.label for option in wizard.delimiterOptions"
+              class="form-control">
+      </select>
+    </span>
+    <span class="form-group">
+      <label>Filename:</label>
+      {{ wizard.file.name }}
+    </span>
+  </div>
+
+  <div class="preview">
+    <table class="table table-condensed">
+      <thead>
+      <tr>
+        <th ng-repeat="col in wizard.columns track by $index">
+          <span title="{{ col }}">{{ col | limitTo:12 }}{{ col.length > 12 ? '...' : '' }}</span>
+        </th>
+      </tr>
+      </thead>
+      <tbody>
+      <tr ng-repeat="row in wizard.rows">
+        <td ng-repeat="cell in row track by $index">{{ cell }}</td>
+      </tr>
+      </tbody>
+    </table>
+  </div>
+</div>