diff --git a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts
index 7514e482783b36..0eaf3143eaac02 100644
--- a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts
+++ b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts
@@ -44,9 +44,10 @@ export function postProcess(parsedFiles: any[]): void {
*/
function updateBlockParameters(docEntries: DocEntry[], block: Block, paramsGroup: string): void {
if (!block.local.parameter) {
- block.local.parameter = {
- fields: {},
- };
+ block.local.parameter = {};
+ }
+ if (!block.local.parameter.fields) {
+ block.local.parameter.fields = {};
}
if (!block.local.parameter.fields![paramsGroup]) {
diff --git a/x-pack/plugins/ml/server/routes/modules.ts b/x-pack/plugins/ml/server/routes/modules.ts
index 622ae66ede4264..ade3d3eca90ea6 100644
--- a/x-pack/plugins/ml/server/routes/modules.ts
+++ b/x-pack/plugins/ml/server/routes/modules.ts
@@ -4,13 +4,18 @@
* you may not use this file except in compliance with the Elastic License.
*/
-import { schema, TypeOf } from '@kbn/config-schema';
+import { TypeOf } from '@kbn/config-schema';
import { RequestHandlerContext } from 'kibana/server';
import { DatafeedOverride, JobOverride } from '../../common/types/modules';
import { wrapError } from '../client/error_wrapper';
import { DataRecognizer } from '../models/data_recognizer';
-import { getModuleIdParamSchema, setupModuleBodySchema } from './schemas/modules';
+import {
+ moduleIdParamSchema,
+ optionalModuleIdParamSchema,
+ modulesIndexPatternTitleSchema,
+ setupModuleBodySchema,
+} from './schemas/modules';
import { RouteInitialization } from '../types';
function recognize(context: RequestHandlerContext, indexPatternTitle: string) {
@@ -85,17 +90,33 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
*
* @api {get} /api/ml/modules/recognize/:indexPatternTitle Recognize index pattern
* @apiName RecognizeIndex
- * @apiDescription Returns the list of modules that matching the index pattern.
- *
- * @apiParam {String} indexPatternTitle Index pattern title.
+ * @apiDescription By supplying an index pattern, discover if any of the modules are a match for data in that index.
+ * @apiSchema (params) modulesIndexPatternTitleSchema
+ * @apiSuccess {object[]} modules Array of objects describing the modules which match the index pattern.
+ * @apiSuccessExample {json} Success-Response:
+ * [{
+ * "id": "nginx_ecs",
+ * "query": {
+ * "bool": {
+ * "filter": [
+ * { "term": { "event.dataset": "nginx.access" } },
+ * { "exists": { "field": "source.address" } },
+ * { "exists": { "field": "url.original" } },
+ * { "exists": { "field": "http.response.status_code" } }
+ * ]
+ * }
+ * },
+ * "description": "Find unusual activity in HTTP access logs from filebeat (ECS)",
+ * "logo": {
+ * "icon": "logoNginx"
+ * }
+ * }]
*/
router.get(
{
path: '/api/ml/modules/recognize/{indexPatternTitle}',
validate: {
- params: schema.object({
- indexPatternTitle: schema.string(),
- }),
+ params: modulesIndexPatternTitleSchema,
},
options: {
tags: ['access:ml:canCreateJob'],
@@ -118,17 +139,114 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
*
* @api {get} /api/ml/modules/get_module/:moduleId Get module
* @apiName GetModule
- * @apiDescription Returns module by id.
- *
- * @apiParam {String} [moduleId] Module id
+ * @apiDescription Retrieve a whole ML module, containing jobs, datafeeds and saved objects. If
+ * no module ID is supplied, returns all modules.
+ * @apiSchema (params) moduleIdParamSchema
+ * @apiSuccess {object} module When a module ID is specified, returns a module object containing
+ * all of the jobs, datafeeds and saved objects which will be created when the module is setup.
+ * @apiSuccess {object[]} modules If no module ID is supplied, an array of all modules will be returned.
+ * @apiSuccessExample {json} Success-Response:
+ * {
+ * "id":"sample_data_ecommerce",
+ * "title":"Kibana sample data eCommerce",
+ * "description":"Find anomalies in eCommerce total sales data",
+ * "type":"Sample Dataset",
+ * "logoFile":"logo.json",
+ * "defaultIndexPattern":"kibana_sample_data_ecommerce",
+ * "query":{
+ * "bool":{
+ * "filter":[
+ * {
+ * "term":{
+ * "_index":"kibana_sample_data_ecommerce"
+ * }
+ * }
+ * ]
+ * }
+ * },
+ * "jobs":[
+ * {
+ * "id":"high_sum_total_sales",
+ * "config":{
+ * "groups":[
+ * "kibana_sample_data",
+ * "kibana_sample_ecommerce"
+ * ],
+ * "description":"Find customers spending an unusually high amount in an hour",
+ * "analysis_config":{
+ * "bucket_span":"1h",
+ * "detectors":[
+ * {
+ * "detector_description":"High total sales",
+ * "function":"high_sum",
+ * "field_name":"taxful_total_price",
+ * "over_field_name":"customer_full_name.keyword"
+ * }
+ * ],
+ * "influencers":[
+ * "customer_full_name.keyword",
+ * "category.keyword"
+ * ]
+ * },
+ * "analysis_limits":{
+ * "model_memory_limit":"10mb"
+ * },
+ * "data_description":{
+ * "time_field":"order_date"
+ * },
+ * "model_plot_config":{
+ * "enabled":true
+ * },
+ * "custom_settings":{
+ * "created_by":"ml-module-sample",
+ * "custom_urls":[
+ * {
+ * "url_name":"Raw data",
+ * "url_value":"kibana#/discover?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a
+ * (index:ff959d40-b880-11e8-a6d9-e546fe2bba5f,query:(language:kuery,query:'customer_full_name
+ * keyword:\"$customer_full_name.keyword$\"'),sort:!('@timestamp',desc))"
+ * },
+ * {
+ * "url_name":"Data dashboard",
+ * "url_value":"kibana#/dashboard/722b74f0-b882-11e8-a6d9-e546fe2bba5f?_g=(filters:!(),time:(from:'$earliest$',
+ * mode:absolute,to:'$latest$'))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f
+ * index:'INDEX_PATTERN_ID', key:customer_full_name.keyword,negate:!f,params:(query:'$customer_full_name.keyword$')
+ * type:phrase,value:'$customer_full_name.keyword$'),query:(match:(customer_full_name.keyword:
+ * (query:'$customer_full_name.keyword$',type:phrase))))),query:(language:kuery, query:''))"
+ * }
+ * ]
+ * }
+ * }
+ * }
+ * ],
+ * "datafeeds":[
+ * {
+ * "id":"datafeed-high_sum_total_sales",
+ * "config":{
+ * "job_id":"high_sum_total_sales",
+ * "indexes":[
+ * "INDEX_PATTERN_NAME"
+ * ],
+ * "query":{
+ * "bool":{
+ * "filter":[
+ * {
+ * "term":{ "_index":"kibana_sample_data_ecommerce" }
+ * }
+ * ]
+ * }
+ * }
+ * }
+ * }
+ * ],
+ * "kibana":{}
+ * }
*/
router.get(
{
path: '/api/ml/modules/get_module/{moduleId?}',
validate: {
- params: schema.object({
- ...getModuleIdParamSchema(true),
- }),
+ params: optionalModuleIdParamSchema,
},
options: {
tags: ['access:ml:canGetJobs'],
@@ -154,17 +272,148 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
/**
* @apiGroup Modules
*
- * @api {post} /api/ml/modules/setup/:moduleId Setup module
+ * @api {post} /api/ml/modules/setup/:moduleId Set up module
* @apiName SetupModule
- * @apiDescription Created module items.
- *
+ * @apiDescription Runs the module setup process.
+ * This creates jobs, datafeeds and kibana saved objects. It allows for customization of the module,
+ * overriding the default configuration. It also allows the user to start the datafeed.
+ * @apiSchema (params) moduleIdParamSchema
* @apiSchema (body) setupModuleBodySchema
+ * @apiParamExample {json} jobOverrides-no-job-ID:
+ * "jobOverrides": {
+ * "analysis_limits": {
+ * "model_memory_limit": "13mb"
+ * }
+ * }
+ * @apiParamExample {json} jobOverrides-with-job-ID:
+ * "jobOverrides": [
+ * {
+ * "analysis_limits": {
+ * "job_id": "foo"
+ * "model_memory_limit": "13mb"
+ * }
+ * }
+ * ]
+ * @apiParamExample {json} datafeedOverrides:
+ * "datafeedOverrides": [
+ * {
+ * "scroll_size": 1001
+ * },
+ * {
+ * "job_id": "visitor_rate_ecs",
+ * "frequency": "30m"
+ * }
+ * ]
+ * @apiParamExample {json} query-overrrides-datafeedOverrides-query:
+ * {
+ * "query": {"bool":{"must":[{"match_all":{}}]}}
+ * "datafeedOverrides": {
+ * "query": {}
+ * }
+ * }
+ * @apiSuccess {object} results An object containing the results of creating the items in a module,
+ * i.e. the jobs, datafeeds and saved objects. Each item is listed by id with a success flag
+ * signifying whether the creation was successful. If the item creation failed, an error object
+ * with also be supplied containing the error.
+ * @apiSuccessExample {json} Success-Response:
+ * {
+ * "jobs": [{
+ * "id": "test-visitor_rate_ecs",
+ * "success": true
+ * }, {
+ * "id": "test-status_code_rate_ecs",
+ * "success": true
+ * }, {
+ * "id": "test-source_ip_url_count_ecs",
+ * "success": true
+ * }, {
+ * "id": "test-source_ip_request_rate_ecs",
+ * "success": true
+ * }, {
+ * "id": "test-low_request_rate_ecs",
+ * "success": true
+ * }],
+ * "datafeeds": [{
+ * "id": "datafeed-test-visitor_rate_ecs",
+ * "success": true,
+ * "started": false
+ * }, {
+ * "id": "datafeed-test-status_code_rate_ecs",
+ * "success": true,
+ * "started": false
+ * }, {
+ * "id": "datafeed-test-source_ip_url_count_ecs",
+ * "success": true,
+ * "started": false
+ * }, {
+ * "id": "datafeed-test-low_request_rate_ecs",
+ * "success": true,
+ * "started": false
+ * }, {
+ * "id": "datafeed-test-source_ip_request_rate_ecs",
+ * "success": true,
+ * "started": false
+ * }],
+ * "kibana": {
+ * "dashboard": [{
+ * "id": "ml_http_access_explorer_ecs",
+ * "success": true
+ * }],
+ * "search": [{
+ * "id": "ml_http_access_filebeat_ecs",
+ * "success": true
+ * }],
+ * "visualization": [{
+ * "id": "ml_http_access_map_ecs",
+ * "success": true
+ * }, {
+ * "id": "ml_http_access_source_ip_timechart_ecs",
+ * "success": true
+ * }, {
+ * "id": "ml_http_access_status_code_timechart_ecs",
+ * "success": true
+ * }, {
+ * "id": "ml_http_access_top_source_ips_table_ecs",
+ * "success": true
+ * }, {
+ * "id": "ml_http_access_top_urls_table_ecs",
+ * "success": true
+ * }, {
+ * "id": "ml_http_access_events_timechart_ecs",
+ * "success": true
+ * }, {
+ * "id": "ml_http_access_unique_count_url_timechart_ecs",
+ * "success": true
+ * }]
+ * }
+ * }
+ * @apiSuccessExample {json} Error-Response:
+ * {
+ * "jobs": [{
+ * "id": "test-status_code_rate_ecs",
+ * "success": false,
+ * "error": {
+ * "msg": "[resource_already_exists_exception] The job cannot be created with the Id 'test-status_code_rate_ecs'. The Id is
+ * already used.",
+ * "path": "/_ml/anomaly_detectors/test-status_code_rate_ecs",
+ * "query": {},
+ * "body": "{...}",
+ * "statusCode": 400,
+ * "response": "{\"error\":{\"root_cause\":[{\"type\":\"resource_already_exists_exception\",\"reason\":\"The job cannot be created
+ * with the Id 'test-status_code_rate_ecs'. The Id is already used.\"}],\"type\":\"resource_already_exists_exception\",
+ * \"reason\":\"The job cannot be created with the Id 'test-status_code_rate_ecs'. The Id is already used.\"},\"status\":400}"
+ * }
+ * },
+ * },
+ * ...
+ * }]
+ * }
*/
router.post(
{
path: '/api/ml/modules/setup/{moduleId}',
validate: {
- params: schema.object(getModuleIdParamSchema()),
+ params: moduleIdParamSchema,
body: setupModuleBodySchema,
},
options: {
@@ -217,15 +466,58 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) {
*
* @api {post} /api/ml/modules/jobs_exist/:moduleId Check if module jobs exist
* @apiName CheckExistingModuleJobs
- * @apiDescription Checks if the jobs in the module have been created.
- *
- * @apiParam {String} moduleId Module id
+ * @apiDescription Check whether the jobs in the module with the specified ID exist in the
+ * current list of jobs. The check runs a test to see if any of the jobs in existence
+ * have an ID which ends with the ID of each job in the module. This is done as a prefix
+ * may be supplied in the setup endpoint which is added to the start of the ID of every job in the module.
+ * @apiSchema (params) moduleIdParamSchema
+ * @apiSuccess {boolean} jobsExist true
if all the jobs in the module have a matching job with an
+ * ID which ends with the job ID specified in the module, false
otherwise.
+ * @apiSuccess {Object[]} jobs present if the jobs do all exist, with each object having keys of id
,
+ * and optionally earliestTimestampMs
, latestTimestampMs
, latestResultsTimestampMs
+ * properties if the job has processed any data.
+ * @apiSuccessExample {json} Success-Response:
+ * {
+ * "jobsExist":true,
+ * "jobs":[
+ * {
+ * "id":"nginx_low_request_rate_ecs",
+ * "earliestTimestampMs":1547016291000,
+ * "latestTimestampMs":1548256497000
+ * "latestResultsTimestampMs":1548255600000
+ * },
+ * {
+ * "id":"nginx_source_ip_request_rate_ecs",
+ * "earliestTimestampMs":1547015109000,
+ * "latestTimestampMs":1548257222000
+ * "latestResultsTimestampMs":1548255600000
+ * },
+ * {
+ * "id":"nginx_source_ip_url_count_ecs",
+ * "earliestTimestampMs":1547015109000,
+ * "latestTimestampMs":1548257222000
+ * "latestResultsTimestampMs":1548255600000
+ * },
+ * {
+ * "id":"nginx_status_code_rate_ecs",
+ * "earliestTimestampMs":1547015109000,
+ * "latestTimestampMs":1548257222000
+ * "latestResultsTimestampMs":1548255600000
+ * },
+ * {
+ * "id":"nginx_visitor_rate_ecs",
+ * "earliestTimestampMs":1547016291000,
+ * "latestTimestampMs":1548256497000
+ * "latestResultsTimestampMs":1548255600000
+ * }
+ * ]
+ * }
*/
router.get(
{
path: '/api/ml/modules/jobs_exist/{moduleId}',
validate: {
- params: schema.object(getModuleIdParamSchema()),
+ params: moduleIdParamSchema,
},
options: {
tags: ['access:ml:canGetJobs'],
diff --git a/x-pack/plugins/ml/server/routes/schemas/modules.ts b/x-pack/plugins/ml/server/routes/schemas/modules.ts
index 98e3d80f0ff842..23148c14c734e0 100644
--- a/x-pack/plugins/ml/server/routes/schemas/modules.ts
+++ b/x-pack/plugins/ml/server/routes/schemas/modules.ts
@@ -7,24 +7,89 @@
import { schema } from '@kbn/config-schema';
export const setupModuleBodySchema = schema.object({
+ /**
+ * Job ID prefix. This will be added to the start of the ID every job created by the module (optional).
+ */
prefix: schema.maybe(schema.string()),
+ /**
+ * List of group IDs. This will override the groups assigned to each job created by the module (optional).
+ */
groups: schema.maybe(schema.arrayOf(schema.string())),
+ /**
+ * Name of kibana index pattern. Overrides the index used in each datafeed and each index pattern
+ * used in the custom urls and saved objects created by the module. A matching index pattern must
+ * exist in kibana if the module contains custom urls or saved objects which rely on an index pattern ID.
+ * If the module does not contain custom urls or saved objects which require an index pattern ID, the
+ * indexPatternName can be any index name or pattern that will match an ES index. It can also be a comma
+ * separated list of names. If no indexPatternName is supplied, the default index pattern specified in
+ * the manifest.json will be used (optional).
+ */
indexPatternName: schema.maybe(schema.string()),
+ /**
+ * ES Query DSL object. Overrides the query object for each datafeed created by the module (optional).
+ */
query: schema.maybe(schema.any()),
+ /**
+ * Flag to specify that each job created by the module uses a dedicated index (optional).
+ */
useDedicatedIndex: schema.maybe(schema.boolean()),
+ /**
+ * Flag to specify that each datafeed created by the module is started once saved. Defaults to false
(optional).
+ */
startDatafeed: schema.maybe(schema.boolean()),
+ /**
+ * Start date for datafeed. Specified in epoch seconds. Only used if startDatafeed is true.
+ * If not specified, a value of 0 is used i.e. start at the beginning of the data (optional).
+ */
start: schema.maybe(schema.number()),
+ /**
+ * End date for datafeed. Specified in epoch seconds. Only used if startDatafeed is true.
+ * If not specified, the datafeed will continue to run in real time (optional).
+ */
end: schema.maybe(schema.number()),
+ /**
+ * Partial job configuration which will override jobs contained in the module. Can be an array of objects.
+ * If a job_id
is specified, only that job in the module will be overridden.
+ * Applied before any of the existing
+ * overridable options (e.g. useDedicatedIndex, groups, indexPatternName etc)
+ * and so can be overridden themselves (optional).
+ */
jobOverrides: schema.maybe(schema.any()),
+ /**
+ * Partial datafeed configuration which will override datafeeds contained in the module.
+ * Can be an array of objects.
+ * If a datafeed_id or a job_id is specified,
+ * only that datafeed in the module will be overridden. Applied before any of the existing
+ * overridable options (e.g. useDedicatedIndex, groups, indexPatternName etc)
+ * and so can be overridden themselves (optional).
+ */
datafeedOverrides: schema.maybe(schema.any()),
/**
* Indicates whether an estimate of the model memory limit
- * should be made by checking the cardinality of fields in the job configurations.
+ * should be made by checking the cardinality of fields in the job configurations (optional).
*/
estimateModelMemory: schema.maybe(schema.boolean()),
});
export const getModuleIdParamSchema = (optional = false) => {
const stringType = schema.string();
- return { moduleId: optional ? schema.maybe(stringType) : stringType };
+ return schema.object({
+ /**
+ * ID of the module.
+ */
+ moduleId: optional ? schema.maybe(stringType) : stringType,
+ });
};
+
+export const optionalModuleIdParamSchema = getModuleIdParamSchema(true);
+
+export const moduleIdParamSchema = getModuleIdParamSchema(false);
+
+export const modulesIndexPatternTitleSchema = schema.object({
+ /**
+ * Index pattern to recognize. Note that this does not need to be a Kibana
+ * index pattern, and can be the name of a single Elasticsearch index,
+ * or include a wildcard (*) to match multiple indices.
+ */
+ indexPatternTitle: schema.string(),
+});