diff --git a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts index 7514e482783b36..0eaf3143eaac02 100644 --- a/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts +++ b/x-pack/plugins/ml/server/routes/apidoc_scripts/schema_worker.ts @@ -44,9 +44,10 @@ export function postProcess(parsedFiles: any[]): void { */ function updateBlockParameters(docEntries: DocEntry[], block: Block, paramsGroup: string): void { if (!block.local.parameter) { - block.local.parameter = { - fields: {}, - }; + block.local.parameter = {}; + } + if (!block.local.parameter.fields) { + block.local.parameter.fields = {}; } if (!block.local.parameter.fields![paramsGroup]) { diff --git a/x-pack/plugins/ml/server/routes/modules.ts b/x-pack/plugins/ml/server/routes/modules.ts index 622ae66ede4264..ade3d3eca90ea6 100644 --- a/x-pack/plugins/ml/server/routes/modules.ts +++ b/x-pack/plugins/ml/server/routes/modules.ts @@ -4,13 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -import { schema, TypeOf } from '@kbn/config-schema'; +import { TypeOf } from '@kbn/config-schema'; import { RequestHandlerContext } from 'kibana/server'; import { DatafeedOverride, JobOverride } from '../../common/types/modules'; import { wrapError } from '../client/error_wrapper'; import { DataRecognizer } from '../models/data_recognizer'; -import { getModuleIdParamSchema, setupModuleBodySchema } from './schemas/modules'; +import { + moduleIdParamSchema, + optionalModuleIdParamSchema, + modulesIndexPatternTitleSchema, + setupModuleBodySchema, +} from './schemas/modules'; import { RouteInitialization } from '../types'; function recognize(context: RequestHandlerContext, indexPatternTitle: string) { @@ -85,17 +90,33 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { * * @api {get} /api/ml/modules/recognize/:indexPatternTitle Recognize index pattern * @apiName RecognizeIndex - * @apiDescription Returns the list of modules that matching the index pattern. - * - * @apiParam {String} indexPatternTitle Index pattern title. + * @apiDescription By supplying an index pattern, discover if any of the modules are a match for data in that index. + * @apiSchema (params) modulesIndexPatternTitleSchema + * @apiSuccess {object[]} modules Array of objects describing the modules which match the index pattern. + * @apiSuccessExample {json} Success-Response: + * [{ + * "id": "nginx_ecs", + * "query": { + * "bool": { + * "filter": [ + * { "term": { "event.dataset": "nginx.access" } }, + * { "exists": { "field": "source.address" } }, + * { "exists": { "field": "url.original" } }, + * { "exists": { "field": "http.response.status_code" } } + * ] + * } + * }, + * "description": "Find unusual activity in HTTP access logs from filebeat (ECS)", + * "logo": { + * "icon": "logoNginx" + * } + * }] */ router.get( { path: '/api/ml/modules/recognize/{indexPatternTitle}', validate: { - params: schema.object({ - indexPatternTitle: schema.string(), - }), + params: modulesIndexPatternTitleSchema, }, options: { tags: ['access:ml:canCreateJob'], @@ -118,17 +139,114 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { * * @api {get} /api/ml/modules/get_module/:moduleId Get module * @apiName GetModule - * @apiDescription Returns module by id. - * - * @apiParam {String} [moduleId] Module id + * @apiDescription Retrieve a whole ML module, containing jobs, datafeeds and saved objects. If + * no module ID is supplied, returns all modules. + * @apiSchema (params) moduleIdParamSchema + * @apiSuccess {object} module When a module ID is specified, returns a module object containing + * all of the jobs, datafeeds and saved objects which will be created when the module is setup. + * @apiSuccess {object[]} modules If no module ID is supplied, an array of all modules will be returned. + * @apiSuccessExample {json} Success-Response: + * { + * "id":"sample_data_ecommerce", + * "title":"Kibana sample data eCommerce", + * "description":"Find anomalies in eCommerce total sales data", + * "type":"Sample Dataset", + * "logoFile":"logo.json", + * "defaultIndexPattern":"kibana_sample_data_ecommerce", + * "query":{ + * "bool":{ + * "filter":[ + * { + * "term":{ + * "_index":"kibana_sample_data_ecommerce" + * } + * } + * ] + * } + * }, + * "jobs":[ + * { + * "id":"high_sum_total_sales", + * "config":{ + * "groups":[ + * "kibana_sample_data", + * "kibana_sample_ecommerce" + * ], + * "description":"Find customers spending an unusually high amount in an hour", + * "analysis_config":{ + * "bucket_span":"1h", + * "detectors":[ + * { + * "detector_description":"High total sales", + * "function":"high_sum", + * "field_name":"taxful_total_price", + * "over_field_name":"customer_full_name.keyword" + * } + * ], + * "influencers":[ + * "customer_full_name.keyword", + * "category.keyword" + * ] + * }, + * "analysis_limits":{ + * "model_memory_limit":"10mb" + * }, + * "data_description":{ + * "time_field":"order_date" + * }, + * "model_plot_config":{ + * "enabled":true + * }, + * "custom_settings":{ + * "created_by":"ml-module-sample", + * "custom_urls":[ + * { + * "url_name":"Raw data", + * "url_value":"kibana#/discover?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a + * (index:ff959d40-b880-11e8-a6d9-e546fe2bba5f,query:(language:kuery,query:'customer_full_name + * keyword:\"$customer_full_name.keyword$\"'),sort:!('@timestamp',desc))" + * }, + * { + * "url_name":"Data dashboard", + * "url_value":"kibana#/dashboard/722b74f0-b882-11e8-a6d9-e546fe2bba5f?_g=(filters:!(),time:(from:'$earliest$', + * mode:absolute,to:'$latest$'))&_a=(filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f + * index:'INDEX_PATTERN_ID', key:customer_full_name.keyword,negate:!f,params:(query:'$customer_full_name.keyword$') + * type:phrase,value:'$customer_full_name.keyword$'),query:(match:(customer_full_name.keyword: + * (query:'$customer_full_name.keyword$',type:phrase))))),query:(language:kuery, query:''))" + * } + * ] + * } + * } + * } + * ], + * "datafeeds":[ + * { + * "id":"datafeed-high_sum_total_sales", + * "config":{ + * "job_id":"high_sum_total_sales", + * "indexes":[ + * "INDEX_PATTERN_NAME" + * ], + * "query":{ + * "bool":{ + * "filter":[ + * { + * "term":{ "_index":"kibana_sample_data_ecommerce" } + * } + * ] + * } + * } + * } + * } + * ], + * "kibana":{} + * } */ router.get( { path: '/api/ml/modules/get_module/{moduleId?}', validate: { - params: schema.object({ - ...getModuleIdParamSchema(true), - }), + params: optionalModuleIdParamSchema, }, options: { tags: ['access:ml:canGetJobs'], @@ -154,17 +272,148 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { /** * @apiGroup Modules * - * @api {post} /api/ml/modules/setup/:moduleId Setup module + * @api {post} /api/ml/modules/setup/:moduleId Set up module * @apiName SetupModule - * @apiDescription Created module items. - * + * @apiDescription Runs the module setup process. + * This creates jobs, datafeeds and kibana saved objects. It allows for customization of the module, + * overriding the default configuration. It also allows the user to start the datafeed. + * @apiSchema (params) moduleIdParamSchema * @apiSchema (body) setupModuleBodySchema + * @apiParamExample {json} jobOverrides-no-job-ID: + * "jobOverrides": { + * "analysis_limits": { + * "model_memory_limit": "13mb" + * } + * } + * @apiParamExample {json} jobOverrides-with-job-ID: + * "jobOverrides": [ + * { + * "analysis_limits": { + * "job_id": "foo" + * "model_memory_limit": "13mb" + * } + * } + * ] + * @apiParamExample {json} datafeedOverrides: + * "datafeedOverrides": [ + * { + * "scroll_size": 1001 + * }, + * { + * "job_id": "visitor_rate_ecs", + * "frequency": "30m" + * } + * ] + * @apiParamExample {json} query-overrrides-datafeedOverrides-query: + * { + * "query": {"bool":{"must":[{"match_all":{}}]}} + * "datafeedOverrides": { + * "query": {} + * } + * } + * @apiSuccess {object} results An object containing the results of creating the items in a module, + * i.e. the jobs, datafeeds and saved objects. Each item is listed by id with a success flag + * signifying whether the creation was successful. If the item creation failed, an error object + * with also be supplied containing the error. + * @apiSuccessExample {json} Success-Response: + * { + * "jobs": [{ + * "id": "test-visitor_rate_ecs", + * "success": true + * }, { + * "id": "test-status_code_rate_ecs", + * "success": true + * }, { + * "id": "test-source_ip_url_count_ecs", + * "success": true + * }, { + * "id": "test-source_ip_request_rate_ecs", + * "success": true + * }, { + * "id": "test-low_request_rate_ecs", + * "success": true + * }], + * "datafeeds": [{ + * "id": "datafeed-test-visitor_rate_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-status_code_rate_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-source_ip_url_count_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-low_request_rate_ecs", + * "success": true, + * "started": false + * }, { + * "id": "datafeed-test-source_ip_request_rate_ecs", + * "success": true, + * "started": false + * }], + * "kibana": { + * "dashboard": [{ + * "id": "ml_http_access_explorer_ecs", + * "success": true + * }], + * "search": [{ + * "id": "ml_http_access_filebeat_ecs", + * "success": true + * }], + * "visualization": [{ + * "id": "ml_http_access_map_ecs", + * "success": true + * }, { + * "id": "ml_http_access_source_ip_timechart_ecs", + * "success": true + * }, { + * "id": "ml_http_access_status_code_timechart_ecs", + * "success": true + * }, { + * "id": "ml_http_access_top_source_ips_table_ecs", + * "success": true + * }, { + * "id": "ml_http_access_top_urls_table_ecs", + * "success": true + * }, { + * "id": "ml_http_access_events_timechart_ecs", + * "success": true + * }, { + * "id": "ml_http_access_unique_count_url_timechart_ecs", + * "success": true + * }] + * } + * } + * @apiSuccessExample {json} Error-Response: + * { + * "jobs": [{ + * "id": "test-status_code_rate_ecs", + * "success": false, + * "error": { + * "msg": "[resource_already_exists_exception] The job cannot be created with the Id 'test-status_code_rate_ecs'. The Id is + * already used.", + * "path": "/_ml/anomaly_detectors/test-status_code_rate_ecs", + * "query": {}, + * "body": "{...}", + * "statusCode": 400, + * "response": "{\"error\":{\"root_cause\":[{\"type\":\"resource_already_exists_exception\",\"reason\":\"The job cannot be created + * with the Id 'test-status_code_rate_ecs'. The Id is already used.\"}],\"type\":\"resource_already_exists_exception\", + * \"reason\":\"The job cannot be created with the Id 'test-status_code_rate_ecs'. The Id is already used.\"},\"status\":400}" + * } + * }, + * }, + * ... + * }] + * } */ router.post( { path: '/api/ml/modules/setup/{moduleId}', validate: { - params: schema.object(getModuleIdParamSchema()), + params: moduleIdParamSchema, body: setupModuleBodySchema, }, options: { @@ -217,15 +466,58 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { * * @api {post} /api/ml/modules/jobs_exist/:moduleId Check if module jobs exist * @apiName CheckExistingModuleJobs - * @apiDescription Checks if the jobs in the module have been created. - * - * @apiParam {String} moduleId Module id + * @apiDescription Check whether the jobs in the module with the specified ID exist in the + * current list of jobs. The check runs a test to see if any of the jobs in existence + * have an ID which ends with the ID of each job in the module. This is done as a prefix + * may be supplied in the setup endpoint which is added to the start of the ID of every job in the module. + * @apiSchema (params) moduleIdParamSchema + * @apiSuccess {boolean} jobsExist true if all the jobs in the module have a matching job with an + * ID which ends with the job ID specified in the module, false otherwise. + * @apiSuccess {Object[]} jobs present if the jobs do all exist, with each object having keys of id, + * and optionally earliestTimestampMs, latestTimestampMs, latestResultsTimestampMs + * properties if the job has processed any data. + * @apiSuccessExample {json} Success-Response: + * { + * "jobsExist":true, + * "jobs":[ + * { + * "id":"nginx_low_request_rate_ecs", + * "earliestTimestampMs":1547016291000, + * "latestTimestampMs":1548256497000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_source_ip_request_rate_ecs", + * "earliestTimestampMs":1547015109000, + * "latestTimestampMs":1548257222000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_source_ip_url_count_ecs", + * "earliestTimestampMs":1547015109000, + * "latestTimestampMs":1548257222000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_status_code_rate_ecs", + * "earliestTimestampMs":1547015109000, + * "latestTimestampMs":1548257222000 + * "latestResultsTimestampMs":1548255600000 + * }, + * { + * "id":"nginx_visitor_rate_ecs", + * "earliestTimestampMs":1547016291000, + * "latestTimestampMs":1548256497000 + * "latestResultsTimestampMs":1548255600000 + * } + * ] + * } */ router.get( { path: '/api/ml/modules/jobs_exist/{moduleId}', validate: { - params: schema.object(getModuleIdParamSchema()), + params: moduleIdParamSchema, }, options: { tags: ['access:ml:canGetJobs'], diff --git a/x-pack/plugins/ml/server/routes/schemas/modules.ts b/x-pack/plugins/ml/server/routes/schemas/modules.ts index 98e3d80f0ff842..23148c14c734e0 100644 --- a/x-pack/plugins/ml/server/routes/schemas/modules.ts +++ b/x-pack/plugins/ml/server/routes/schemas/modules.ts @@ -7,24 +7,89 @@ import { schema } from '@kbn/config-schema'; export const setupModuleBodySchema = schema.object({ + /** + * Job ID prefix. This will be added to the start of the ID every job created by the module (optional). + */ prefix: schema.maybe(schema.string()), + /** + * List of group IDs. This will override the groups assigned to each job created by the module (optional). + */ groups: schema.maybe(schema.arrayOf(schema.string())), + /** + * Name of kibana index pattern. Overrides the index used in each datafeed and each index pattern + * used in the custom urls and saved objects created by the module. A matching index pattern must + * exist in kibana if the module contains custom urls or saved objects which rely on an index pattern ID. + * If the module does not contain custom urls or saved objects which require an index pattern ID, the + * indexPatternName can be any index name or pattern that will match an ES index. It can also be a comma + * separated list of names. If no indexPatternName is supplied, the default index pattern specified in + * the manifest.json will be used (optional). + */ indexPatternName: schema.maybe(schema.string()), + /** + * ES Query DSL object. Overrides the query object for each datafeed created by the module (optional). + */ query: schema.maybe(schema.any()), + /** + * Flag to specify that each job created by the module uses a dedicated index (optional). + */ useDedicatedIndex: schema.maybe(schema.boolean()), + /** + * Flag to specify that each datafeed created by the module is started once saved. Defaults to false (optional). + */ startDatafeed: schema.maybe(schema.boolean()), + /** + * Start date for datafeed. Specified in epoch seconds. Only used if startDatafeed is true. + * If not specified, a value of 0 is used i.e. start at the beginning of the data (optional). + */ start: schema.maybe(schema.number()), + /** + * End date for datafeed. Specified in epoch seconds. Only used if startDatafeed is true. + * If not specified, the datafeed will continue to run in real time (optional). + */ end: schema.maybe(schema.number()), + /** + * Partial job configuration which will override jobs contained in the module. Can be an array of objects. + * If a job_id is specified, only that job in the module will be overridden. + * Applied before any of the existing + * overridable options (e.g. useDedicatedIndex, groups, indexPatternName etc) + * and so can be overridden themselves (optional). + */ jobOverrides: schema.maybe(schema.any()), + /** + * Partial datafeed configuration which will override datafeeds contained in the module. + * Can be an array of objects. + * If a datafeed_id or a job_id is specified, + * only that datafeed in the module will be overridden. Applied before any of the existing + * overridable options (e.g. useDedicatedIndex, groups, indexPatternName etc) + * and so can be overridden themselves (optional). + */ datafeedOverrides: schema.maybe(schema.any()), /** * Indicates whether an estimate of the model memory limit - * should be made by checking the cardinality of fields in the job configurations. + * should be made by checking the cardinality of fields in the job configurations (optional). */ estimateModelMemory: schema.maybe(schema.boolean()), }); export const getModuleIdParamSchema = (optional = false) => { const stringType = schema.string(); - return { moduleId: optional ? schema.maybe(stringType) : stringType }; + return schema.object({ + /** + * ID of the module. + */ + moduleId: optional ? schema.maybe(stringType) : stringType, + }); }; + +export const optionalModuleIdParamSchema = getModuleIdParamSchema(true); + +export const moduleIdParamSchema = getModuleIdParamSchema(false); + +export const modulesIndexPatternTitleSchema = schema.object({ + /** + * Index pattern to recognize. Note that this does not need to be a Kibana + * index pattern, and can be the name of a single Elasticsearch index, + * or include a wildcard (*) to match multiple indices. + */ + indexPatternTitle: schema.string(), +});