diff --git a/.bazelignore b/.bazelignore index ac7a2d15a7aa4a..ec61c2f3e7e750 100644 --- a/.bazelignore +++ b/.bazelignore @@ -8,7 +8,10 @@ .idea .teamcity .yarn-local-mirror -/bazel +bazel-bin +bazel-kibana +bazel-out +bazel-testlogs build node_modules target diff --git a/.bazelrc.common b/.bazelrc.common index fb8e8e86b9ef59..20a41c4cde9a0d 100644 --- a/.bazelrc.common +++ b/.bazelrc.common @@ -18,17 +18,16 @@ build --disk_cache=~/.bazel-cache/disk-cache build --repository_cache=~/.bazel-cache/repository-cache # Bazel will create symlinks from the workspace directory to output artifacts. -# Build results will be placed in a directory called "bazel/bin" +# Build results will be placed in a directory called "bazel-bin" # This will still create a bazel-out symlink in # the project directory, which must be excluded from the # editor's search path. -build --symlink_prefix=bazel/ # To disable the symlinks altogether (including bazel-out) we can use # build --symlink_prefix=/ # however this makes it harder to find outputs. # Prevents the creation of bazel-out dir -build --experimental_no_product_name_out_symlink +# build --experimental_no_product_name_out_symlink # Make direct file system calls to create symlink trees build --experimental_inprocess_symlink_creation @@ -83,7 +82,7 @@ test:debug --test_output=streamed --test_strategy=exclusive --test_timeout=9999 run:debug --define=VERBOSE_LOGS=1 -- --node_options=--inspect-brk # The following option will change the build output of certain rules such as terser and may not be desirable in all cases # It will also output both the repo cache and action cache to a folder inside the repo -build:debug --compilation_mode=dbg --show_result=1 --disk_cache=bazel/disk-cache --repository_cache=bazel/repository-cache +build:debug --compilation_mode=dbg --show_result=1 # Turn off legacy external runfiles # This prevents accidentally depending on this feature, which Bazel will remove. diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 5317b2c500b493..a63c2825816bdd 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -2,8 +2,10 @@ set -e -# cache image used by kibana-load-testing project -docker pull "maven:3.6.3-openjdk-8-slim" +if [[ "$(which docker)" != "" && "$(command uname -m)" != "aarch64" ]]; then + # cache image used by kibana-load-testing project + docker pull "maven:3.6.3-openjdk-8-slim" +fi ./.ci/packer_cache_for_branch.sh master ./.ci/packer_cache_for_branch.sh 7.x diff --git a/.eslintignore b/.eslintignore index 4559711bb9dd31..bbd8e3f88a3788 100644 --- a/.eslintignore +++ b/.eslintignore @@ -48,4 +48,4 @@ snapshots.js /packages/kbn-monaco/src/painless/antlr # Bazel -/bazel +/bazel-* diff --git a/.gitignore b/.gitignore index fbe28b8f1e77cc..ce8fd38b18a929 100644 --- a/.gitignore +++ b/.gitignore @@ -75,5 +75,6 @@ report.asciidoc .yarn-local-mirror # Bazel -/bazel -/.bazelrc.user +bazel +bazel-* +.bazelrc.user diff --git a/.stylelintignore b/.stylelintignore index a48b3adfa36321..72d9d5104a0e99 100644 --- a/.stylelintignore +++ b/.stylelintignore @@ -1,3 +1,4 @@ x-pack/plugins/canvas/shareable_runtime/**/*.s+(a|c)ss build target +bazel-* diff --git a/BUILD.bazel b/BUILD.bazel index 38a478565a4af7..4502daeaacb597 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,6 +2,7 @@ # other packages builds and need to be included as inputs exports_files( [ + "tsconfig.base.json", "tsconfig.json", "package.json" ], diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md index 6ca7a83ac0a030..860f7c3c748924 100644 --- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md +++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.links.md @@ -144,6 +144,7 @@ readonly links: { putComponentTemplateMetadata: string; putSnapshotLifecyclePolicy: string; putWatch: string; + simulatePipeline: string; updateTransform: string; }>; readonly observability: Record; diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md index 3847ab0c6183a4..a9cb6729b214e6 100644 --- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md +++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md @@ -17,5 +17,5 @@ export interface DocLinksStart | --- | --- | --- | | [DOC\_LINK\_VERSION](./kibana-plugin-core-public.doclinksstart.doc_link_version.md) | string | | | [ELASTIC\_WEBSITE\_URL](./kibana-plugin-core-public.doclinksstart.elastic_website_url.md) | string | | -| [links](./kibana-plugin-core-public.doclinksstart.links.md) | {
readonly dashboard: {
readonly guide: string;
readonly drilldowns: string;
readonly drilldownsTriggerPicker: string;
readonly urlDrilldownTemplateSyntax: string;
readonly urlDrilldownVariables: string;
};
readonly discover: Record<string, string>;
readonly filebeat: {
readonly base: string;
readonly installation: string;
readonly configuration: string;
readonly elasticsearchOutput: string;
readonly elasticsearchModule: string;
readonly startup: string;
readonly exportedFields: string;
};
readonly auditbeat: {
readonly base: string;
};
readonly metricbeat: {
readonly base: string;
readonly configure: string;
readonly httpEndpoint: string;
readonly install: string;
readonly start: string;
};
readonly enterpriseSearch: {
readonly base: string;
readonly appSearchBase: string;
readonly workplaceSearchBase: string;
};
readonly heartbeat: {
readonly base: string;
};
readonly logstash: {
readonly base: string;
};
readonly functionbeat: {
readonly base: string;
};
readonly winlogbeat: {
readonly base: string;
};
readonly aggs: {
readonly composite: string;
readonly composite_missing_bucket: string;
readonly date_histogram: string;
readonly date_range: string;
readonly date_format_pattern: string;
readonly filter: string;
readonly filters: string;
readonly geohash_grid: string;
readonly histogram: string;
readonly ip_range: string;
readonly range: string;
readonly significant_terms: string;
readonly terms: string;
readonly avg: string;
readonly avg_bucket: string;
readonly max_bucket: string;
readonly min_bucket: string;
readonly sum_bucket: string;
readonly cardinality: string;
readonly count: string;
readonly cumulative_sum: string;
readonly derivative: string;
readonly geo_bounds: string;
readonly geo_centroid: string;
readonly max: string;
readonly median: string;
readonly min: string;
readonly moving_avg: string;
readonly percentile_ranks: string;
readonly serial_diff: string;
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
};
readonly runtimeFields: {
readonly mapping: string;
};
readonly scriptedFields: {
readonly scriptFields: string;
readonly scriptAggs: string;
readonly painless: string;
readonly painlessApi: string;
readonly painlessLangSpec: string;
readonly painlessSyntax: string;
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
readonly indexPatterns: {
readonly introduction: string;
readonly fieldFormattersNumber: string;
readonly fieldFormattersString: string;
};
readonly addData: string;
readonly kibana: string;
readonly elasticsearch: Record<string, string>;
readonly siem: {
readonly guide: string;
readonly gettingStarted: string;
};
readonly query: {
readonly eql: string;
readonly luceneQuerySyntax: string;
readonly queryDsl: string;
readonly kueryQuerySyntax: string;
};
readonly date: {
readonly dateMath: string;
readonly dateMathIndexNames: string;
};
readonly management: Record<string, string>;
readonly ml: Record<string, string>;
readonly transforms: Record<string, string>;
readonly visualize: Record<string, string>;
readonly apis: Readonly<{
createIndex: string;
createSnapshotLifecyclePolicy: string;
createRoleMapping: string;
createRoleMappingTemplates: string;
createApiKey: string;
createPipeline: string;
createTransformRequest: string;
cronExpressions: string;
executeWatchActionModes: string;
indexExists: string;
openIndex: string;
putComponentTemplate: string;
painlessExecute: string;
painlessExecuteAPIContexts: string;
putComponentTemplateMetadata: string;
putSnapshotLifecyclePolicy: string;
putWatch: string;
updateTransform: string;
}>;
readonly observability: Record<string, string>;
readonly alerting: Record<string, string>;
readonly maps: Record<string, string>;
readonly monitoring: Record<string, string>;
readonly security: Readonly<{
apiKeyServiceSettings: string;
clusterPrivileges: string;
elasticsearchSettings: string;
elasticsearchEnableSecurity: string;
indicesPrivileges: string;
kibanaTLS: string;
kibanaPrivileges: string;
mappingRoles: string;
mappingRolesFieldRules: string;
runAsPrivilege: string;
}>;
readonly watcher: Record<string, string>;
readonly ccs: Record<string, string>;
readonly plugins: Record<string, string>;
readonly snapshotRestore: Record<string, string>;
readonly ingest: Record<string, string>;
} | | +| [links](./kibana-plugin-core-public.doclinksstart.links.md) | {
readonly dashboard: {
readonly guide: string;
readonly drilldowns: string;
readonly drilldownsTriggerPicker: string;
readonly urlDrilldownTemplateSyntax: string;
readonly urlDrilldownVariables: string;
};
readonly discover: Record<string, string>;
readonly filebeat: {
readonly base: string;
readonly installation: string;
readonly configuration: string;
readonly elasticsearchOutput: string;
readonly elasticsearchModule: string;
readonly startup: string;
readonly exportedFields: string;
};
readonly auditbeat: {
readonly base: string;
};
readonly metricbeat: {
readonly base: string;
readonly configure: string;
readonly httpEndpoint: string;
readonly install: string;
readonly start: string;
};
readonly enterpriseSearch: {
readonly base: string;
readonly appSearchBase: string;
readonly workplaceSearchBase: string;
};
readonly heartbeat: {
readonly base: string;
};
readonly logstash: {
readonly base: string;
};
readonly functionbeat: {
readonly base: string;
};
readonly winlogbeat: {
readonly base: string;
};
readonly aggs: {
readonly composite: string;
readonly composite_missing_bucket: string;
readonly date_histogram: string;
readonly date_range: string;
readonly date_format_pattern: string;
readonly filter: string;
readonly filters: string;
readonly geohash_grid: string;
readonly histogram: string;
readonly ip_range: string;
readonly range: string;
readonly significant_terms: string;
readonly terms: string;
readonly avg: string;
readonly avg_bucket: string;
readonly max_bucket: string;
readonly min_bucket: string;
readonly sum_bucket: string;
readonly cardinality: string;
readonly count: string;
readonly cumulative_sum: string;
readonly derivative: string;
readonly geo_bounds: string;
readonly geo_centroid: string;
readonly max: string;
readonly median: string;
readonly min: string;
readonly moving_avg: string;
readonly percentile_ranks: string;
readonly serial_diff: string;
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
};
readonly runtimeFields: {
readonly mapping: string;
};
readonly scriptedFields: {
readonly scriptFields: string;
readonly scriptAggs: string;
readonly painless: string;
readonly painlessApi: string;
readonly painlessLangSpec: string;
readonly painlessSyntax: string;
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
readonly indexPatterns: {
readonly introduction: string;
readonly fieldFormattersNumber: string;
readonly fieldFormattersString: string;
};
readonly addData: string;
readonly kibana: string;
readonly elasticsearch: Record<string, string>;
readonly siem: {
readonly guide: string;
readonly gettingStarted: string;
};
readonly query: {
readonly eql: string;
readonly luceneQuerySyntax: string;
readonly queryDsl: string;
readonly kueryQuerySyntax: string;
};
readonly date: {
readonly dateMath: string;
readonly dateMathIndexNames: string;
};
readonly management: Record<string, string>;
readonly ml: Record<string, string>;
readonly transforms: Record<string, string>;
readonly visualize: Record<string, string>;
readonly apis: Readonly<{
createIndex: string;
createSnapshotLifecyclePolicy: string;
createRoleMapping: string;
createRoleMappingTemplates: string;
createApiKey: string;
createPipeline: string;
createTransformRequest: string;
cronExpressions: string;
executeWatchActionModes: string;
indexExists: string;
openIndex: string;
putComponentTemplate: string;
painlessExecute: string;
painlessExecuteAPIContexts: string;
putComponentTemplateMetadata: string;
putSnapshotLifecyclePolicy: string;
putWatch: string;
simulatePipeline: string;
updateTransform: string;
}>;
readonly observability: Record<string, string>;
readonly alerting: Record<string, string>;
readonly maps: Record<string, string>;
readonly monitoring: Record<string, string>;
readonly security: Readonly<{
apiKeyServiceSettings: string;
clusterPrivileges: string;
elasticsearchSettings: string;
elasticsearchEnableSecurity: string;
indicesPrivileges: string;
kibanaTLS: string;
kibanaPrivileges: string;
mappingRoles: string;
mappingRolesFieldRules: string;
runAsPrivilege: string;
}>;
readonly watcher: Record<string, string>;
readonly ccs: Record<string, string>;
readonly plugins: Record<string, string>;
readonly snapshotRestore: Record<string, string>;
readonly ingest: Record<string, string>;
} | | diff --git a/docs/maps/maps-aggregations.asciidoc b/docs/maps/maps-aggregations.asciidoc index 265bf6bfaea304..7f4af952653e7c 100644 --- a/docs/maps/maps-aggregations.asciidoc +++ b/docs/maps/maps-aggregations.asciidoc @@ -76,9 +76,8 @@ then accumulates the most relevant documents based on sort order for each entry To enable top hits: -. Click *Add layer*, then select the *Documents* layer. +. Click *Add layer*, then select the *Top hits per entity* layer. . Configure *Index pattern* and *Geospatial field*. -. In *Scaling*, select *Show top hits per entity*. . Set *Entity* to the field that identifies entities in your documents. This field will be used in the terms aggregation to group your documents into entity buckets. . Set *Documents per entity* to configure the maximum number of documents accumulated per entity. diff --git a/docs/maps/vector-layer.asciidoc b/docs/maps/vector-layer.asciidoc index 6a2228161845ef..2115c16a889c63 100644 --- a/docs/maps/vector-layer.asciidoc +++ b/docs/maps/vector-layer.asciidoc @@ -23,8 +23,6 @@ Select the appropriate *Scaling* option for your use case. * *Limit results to 10000.* The layer displays features from the first `index.max_result_window` documents. Results exceeding `index.max_result_window` are not displayed. -* *Show top hits per entity.* The layer displays the <>. - * *Show clusters when results exceed 10000.* When results exceed `index.max_result_window`, the layer uses {ref}/search-aggregations-bucket-geotilegrid-aggregation.html[GeoTile grid aggregation] to group your documents into clusters and displays metrics for each cluster. When results are less then `index.max_result_window`, the layer displays features from individual documents. * *Use vector tiles.* Vector tiles partition your map into 6 to 8 tiles. @@ -36,6 +34,9 @@ Tiles exceeding `index.max_result_window` have a visual indicator when there are *Point to point*:: Aggregated data paths between the source and destination. The index must contain at least 2 fields mapped as {ref}/geo-point.html[geo_point], source and destination. +*Top hits per entity*:: The layer displays the <>. +The index must contain at least one field mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape]. + *Tracks*:: Create lines from points. The index must contain at least one field mapped as {ref}/geo-point.html[geo_point]. diff --git a/package.json b/package.json index 05a8e450793d6b..deb8bc24319f6b 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,7 @@ "@elastic/apm-rum": "^5.6.1", "@elastic/apm-rum-react": "^1.2.5", "@elastic/charts": "26.1.0", - "@elastic/datemath": "link:packages/elastic-datemath", + "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath/npm_module", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.4", "@elastic/ems-client": "7.12.0", "@elastic/eui": "31.10.0", @@ -441,6 +441,7 @@ "@babel/traverse": "^7.12.12", "@babel/types": "^7.12.12", "@bazel/ibazel": "^0.14.0", + "@bazel/typescript": "^3.2.3", "@cypress/snapshot": "^2.1.7", "@cypress/webpack-preprocessor": "^5.5.0", "@elastic/apm-rum": "^5.6.1", diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel index 1f1eba0747ab7f..31894fcb1bb5db 100644 --- a/packages/BUILD.bazel +++ b/packages/BUILD.bazel @@ -2,5 +2,7 @@ # targets so we can build them all at once filegroup( name = "build", - srcs = [], + srcs = [ + "//packages/elastic-datemath:build" + ], ) diff --git a/packages/elastic-datemath/.npmignore b/packages/elastic-datemath/.npmignore index 591be7afd16696..cb8c40d17ea043 100644 --- a/packages/elastic-datemath/.npmignore +++ b/packages/elastic-datemath/.npmignore @@ -1,2 +1,3 @@ +/index.test.js +/jest.config.js /tsconfig.json -/__tests__ diff --git a/packages/elastic-datemath/BUILD.bazel b/packages/elastic-datemath/BUILD.bazel new file mode 100644 index 00000000000000..6a80556d4eed51 --- /dev/null +++ b/packages/elastic-datemath/BUILD.bazel @@ -0,0 +1,76 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm") + +PKG_BASE_NAME = "elastic-datemath" +PKG_REQUIRE_NAME = "@elastic/datemath" + +SOURCE_FILES = [ + "src/index.ts", +] + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = glob(SOURCE_FILES), +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", + "README.md", +] + +SRC_DEPS = [ + "@npm//moment", +] + +TYPES_DEPS = [ + "@npm//@types/node", +] + +DEPS = SRC_DEPS + TYPES_DEPS + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + ], +) + +ts_project( + name = "tsc", + srcs = SRCS, + deps = DEPS, + declaration = True, + declaration_map = True, + incremental = True, + out_dir = "target", + source_map = True, + root_dir = "src", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_BASE_NAME, + srcs = [], + deps = [":tsc"] + DEPS, + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + srcs = NPM_MODULE_EXTRA_FILES, + deps = [ + ":%s" % PKG_BASE_NAME, + ] +) + +filegroup( + name = "build", + srcs = [ + ":npm_module", + ], + visibility = ["//visibility:public"], +) diff --git a/packages/elastic-datemath/package.json b/packages/elastic-datemath/package.json index 4dc9c4f24d5678..67fbb74eb223cb 100644 --- a/packages/elastic-datemath/package.json +++ b/packages/elastic-datemath/package.json @@ -5,8 +5,7 @@ "license": "Apache-2.0", "main": "./target/index.js", "types": "./target/index.d.ts", - "scripts": { - "build": "../../node_modules/.bin/tsc", - "kbn:bootstrap": "yarn build" + "peerDependencies": { + "moment": "^2.24.0" } } \ No newline at end of file diff --git a/packages/elastic-datemath/tsconfig.json b/packages/elastic-datemath/tsconfig.json index 6f04bee983a9e5..d0fa806ed411b4 100644 --- a/packages/elastic-datemath/tsconfig.json +++ b/packages/elastic-datemath/tsconfig.json @@ -1,10 +1,10 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "incremental": false, - "outDir": "./target", "declaration": true, "declarationMap": true, + "outDir": "target", + "rootDir": "src", "sourceMap": true, "sourceRoot": "../../../../packages/elastic-datemath/src", "types": [ diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index bcb0b6da2a2f80..509ce89f8c02cb 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -209,7 +209,7 @@ async function run(argv) { }, default: { cache: true, - 'force-install': true, + 'force-install': false, offline: false, validate: true }, @@ -8910,8 +8910,11 @@ const BootstrapCommand = { const nonBazelProjectsOnly = await Object(_utils_projects__WEBPACK_IMPORTED_MODULE_4__["getNonBazelProjectsOnly"])(projects); const batchedNonBazelProjects = Object(_utils_projects__WEBPACK_IMPORTED_MODULE_4__["topologicallyBatchProjects"])(nonBazelProjectsOnly, projectGraph); const kibanaProjectPath = ((_projects$get = projects.get('kibana')) === null || _projects$get === void 0 ? void 0 : _projects$get.path) || ''; - const runOffline = (options === null || options === void 0 ? void 0 : options.offline) === true; - const forceInstall = !!options && options['force-install'] === true; // Ensure we have a `node_modules/.yarn-integrity` file as we depend on it + const runOffline = (options === null || options === void 0 ? void 0 : options.offline) === true; // Force install is set in case a flag is passed or + // if the `.yarn-integrity` file is not found which + // will be indicated by the return of yarnIntegrityFileExists. + + const forceInstall = !!options && options['force-install'] === true || !(await Object(_utils_bazel__WEBPACK_IMPORTED_MODULE_9__["yarnIntegrityFileExists"])(Object(path__WEBPACK_IMPORTED_MODULE_0__["resolve"])(kibanaProjectPath, 'node_modules'))); // Ensure we have a `node_modules/.yarn-integrity` file as we depend on it // for bazel to know it has to re-install the node_modules after a reset or a clean await Object(_utils_bazel__WEBPACK_IMPORTED_MODULE_9__["ensureYarnIntegrityFileExists"])(Object(path__WEBPACK_IMPORTED_MODULE_0__["resolve"])(kibanaProjectPath, 'node_modules')); // Install bazel machinery tools if needed @@ -8925,9 +8928,6 @@ const BootstrapCommand = { // That way non bazel projects could depend on bazel projects but not the other way around // That is only intended during the migration process while non Bazel projects are not removed at all. // - // Until we have our first package build within Bazel we will always need to directly call the yarn rule - // otherwise yarn install won't trigger as we don't have any npm dependency within Bazel - // TODO: Change CLI default in order to not force install as soon as we have our first Bazel package being built if (forceInstall) { await Object(_utils_bazel__WEBPACK_IMPORTED_MODULE_9__["runBazel"])(['run', '@nodejs//:yarn'], runOffline); @@ -9105,6 +9105,7 @@ __webpack_require__.r(__webpack_exports__); /* harmony export (binding) */ __webpack_require__.d(__webpack_exports__, "isDirectory", function() { return isDirectory; }); /* harmony export (binding) */ __webpack_require__.d(__webpack_exports__, "isFile", function() { return isFile; }); /* harmony export (binding) */ __webpack_require__.d(__webpack_exports__, "createSymlink", function() { return createSymlink; }); +/* harmony export (binding) */ __webpack_require__.d(__webpack_exports__, "tryRealpath", function() { return tryRealpath; }); /* harmony import */ var cmd_shim__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(132); /* harmony import */ var cmd_shim__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(cmd_shim__WEBPACK_IMPORTED_MODULE_0__); /* harmony import */ var del__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(143); @@ -9137,6 +9138,7 @@ const symlink = Object(util__WEBPACK_IMPORTED_MODULE_5__["promisify"])(fs__WEBPA const chmod = Object(util__WEBPACK_IMPORTED_MODULE_5__["promisify"])(fs__WEBPACK_IMPORTED_MODULE_2___default.a.chmod); const cmdShim = Object(util__WEBPACK_IMPORTED_MODULE_5__["promisify"])(cmd_shim__WEBPACK_IMPORTED_MODULE_0___default.a); const mkdir = Object(util__WEBPACK_IMPORTED_MODULE_5__["promisify"])(fs__WEBPACK_IMPORTED_MODULE_2___default.a.mkdir); +const realpathNative = Object(util__WEBPACK_IMPORTED_MODULE_5__["promisify"])(fs__WEBPACK_IMPORTED_MODULE_2___default.a.realpath.native); const mkdirp = async path => await mkdir(path, { recursive: true }); @@ -9220,6 +9222,20 @@ async function forceCreate(src, dest, type) { await symlink(src, dest, type); } +async function tryRealpath(path) { + let calculatedPath = path; + + try { + calculatedPath = await realpathNative(path); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + return calculatedPath; +} + /***/ }), /* 132 */ /***/ (function(module, exports, __webpack_require__) { @@ -22981,11 +22997,11 @@ class Project { ensureValidProjectDependency(project) { const relativePathToProject = normalizePath(path__WEBPACK_IMPORTED_MODULE_1___default.a.relative(this.path, project.path)); - const relativePathToProjectIfBazelPkg = normalizePath(path__WEBPACK_IMPORTED_MODULE_1___default.a.relative(this.path, `bazel/bin/packages/${path__WEBPACK_IMPORTED_MODULE_1___default.a.basename(project.path)}`)); + const relativePathToProjectIfBazelPkg = normalizePath(path__WEBPACK_IMPORTED_MODULE_1___default.a.relative(this.path, `${__dirname}/../../../bazel-bin/packages/${path__WEBPACK_IMPORTED_MODULE_1___default.a.basename(project.path)}/npm_module`)); const versionInPackageJson = this.allDependencies[project.name]; const expectedVersionInPackageJson = `link:${relativePathToProject}`; const expectedVersionInPackageJsonIfBazelPkg = `link:${relativePathToProjectIfBazelPkg}`; // TODO: after introduce bazel to build all the packages and completely remove the support for kbn packages - // do not allow child projects to hold dependencies + // do not allow child projects to hold dependencies, unless they are meant to be published externally if (versionInPackageJson === expectedVersionInPackageJson || versionInPackageJson === expectedVersionInPackageJsonIfBazelPkg) { return; @@ -23143,7 +23159,7 @@ const createProductionPackageJson = pkgJson => _objectSpread(_objectSpread({}, p dependencies: transformDependencies(pkgJson.dependencies) }); const isLinkDependency = depVersion => depVersion.startsWith('link:'); -const isBazelPackageDependency = depVersion => depVersion.startsWith('link:bazel/bin/'); +const isBazelPackageDependency = depVersion => depVersion.startsWith('link:bazel-bin/'); /** * Replaces `link:` dependencies with `file:` dependencies. When installing * dependencies, these `file:` dependencies will be copied into `node_modules` @@ -23153,7 +23169,7 @@ const isBazelPackageDependency = depVersion => depVersion.startsWith('link:bazel * will then _copy_ the `file:` dependencies into `node_modules` instead of * symlinking like we do in development. * - * Additionally it also taken care of replacing `link:bazel/bin/` with + * Additionally it also taken care of replacing `link:bazel-bin/` with * `file:` so we can also support the copy of the Bazel packages dist already into * build/packages to be copied into the node_modules */ @@ -23170,7 +23186,7 @@ function transformDependencies(dependencies = {}) { } if (isBazelPackageDependency(depVersion)) { - newDeps[name] = depVersion.replace('link:bazel/bin/', 'file:'); + newDeps[name] = depVersion.replace('link:bazel-bin/', 'file:').replace('/npm_module', ''); continue; } @@ -48065,8 +48081,10 @@ function addProjectToTree(tree, pathParts, project) { "use strict"; __webpack_require__.r(__webpack_exports__); -/* harmony import */ var _ensure_yarn_integrity_exists__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(373); -/* harmony reexport (safe) */ __webpack_require__.d(__webpack_exports__, "ensureYarnIntegrityFileExists", function() { return _ensure_yarn_integrity_exists__WEBPACK_IMPORTED_MODULE_0__["ensureYarnIntegrityFileExists"]; }); +/* harmony import */ var _yarn_integrity__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(373); +/* harmony reexport (safe) */ __webpack_require__.d(__webpack_exports__, "yarnIntegrityFileExists", function() { return _yarn_integrity__WEBPACK_IMPORTED_MODULE_0__["yarnIntegrityFileExists"]; }); + +/* harmony reexport (safe) */ __webpack_require__.d(__webpack_exports__, "ensureYarnIntegrityFileExists", function() { return _yarn_integrity__WEBPACK_IMPORTED_MODULE_0__["ensureYarnIntegrityFileExists"]; }); /* harmony import */ var _get_cache_folders__WEBPACK_IMPORTED_MODULE_1__ = __webpack_require__(374); /* harmony reexport (safe) */ __webpack_require__.d(__webpack_exports__, "getBazelDiskCacheFolder", function() { return _get_cache_folders__WEBPACK_IMPORTED_MODULE_1__["getBazelDiskCacheFolder"]; }); @@ -48099,6 +48117,7 @@ __webpack_require__.r(__webpack_exports__); "use strict"; __webpack_require__.r(__webpack_exports__); +/* harmony export (binding) */ __webpack_require__.d(__webpack_exports__, "yarnIntegrityFileExists", function() { return yarnIntegrityFileExists; }); /* harmony export (binding) */ __webpack_require__.d(__webpack_exports__, "ensureYarnIntegrityFileExists", function() { return ensureYarnIntegrityFileExists; }); /* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0__ = __webpack_require__(4); /* harmony import */ var path__WEBPACK_IMPORTED_MODULE_0___default = /*#__PURE__*/__webpack_require__.n(path__WEBPACK_IMPORTED_MODULE_0__); @@ -48112,9 +48131,27 @@ __webpack_require__.r(__webpack_exports__); */ +async function yarnIntegrityFileExists(nodeModulesPath) { + try { + const nodeModulesRealPath = await Object(_fs__WEBPACK_IMPORTED_MODULE_1__["tryRealpath"])(nodeModulesPath); + const yarnIntegrityFilePath = Object(path__WEBPACK_IMPORTED_MODULE_0__["join"])(nodeModulesRealPath, '.yarn-integrity'); // check if the file already exists + + if (await Object(_fs__WEBPACK_IMPORTED_MODULE_1__["isFile"])(yarnIntegrityFilePath)) { + return true; + } + } catch {// no-op + } + + return false; +} async function ensureYarnIntegrityFileExists(nodeModulesPath) { try { - await Object(_fs__WEBPACK_IMPORTED_MODULE_1__["writeFile"])(Object(path__WEBPACK_IMPORTED_MODULE_0__["join"])(nodeModulesPath, '.yarn-integrity'), '', { + const nodeModulesRealPath = await Object(_fs__WEBPACK_IMPORTED_MODULE_1__["tryRealpath"])(nodeModulesPath); + const yarnIntegrityFilePath = Object(path__WEBPACK_IMPORTED_MODULE_0__["join"])(nodeModulesRealPath, '.yarn-integrity'); // ensure node_modules folder is created + + await Object(_fs__WEBPACK_IMPORTED_MODULE_1__["mkdirp"])(nodeModulesRealPath); // write a blank file in case it doesn't exists + + await Object(_fs__WEBPACK_IMPORTED_MODULE_1__["writeFile"])(yarnIntegrityFilePath, '', { flag: 'wx' }); } catch {// no-op @@ -63656,7 +63693,7 @@ async function buildBazelProductionProjects({ const projectNames = [...projects.values()].map(project => project.name); _utils_log__WEBPACK_IMPORTED_MODULE_6__["log"].info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`); await Object(_utils_bazel__WEBPACK_IMPORTED_MODULE_4__["runBazel"])(['build', '//packages:build']); - _utils_log__WEBPACK_IMPORTED_MODULE_6__["log"].info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete}]`); + _utils_log__WEBPACK_IMPORTED_MODULE_6__["log"].info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete`); for (const project of projects.values()) { await copyToBuild(project, kibanaRoot, buildRoot); @@ -63680,7 +63717,7 @@ async function copyToBuild(project, kibanaRoot, buildRoot) { const relativeProjectPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["relative"])(kibanaRoot, project.path); const buildProjectPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(buildRoot, relativeProjectPath); await cpy__WEBPACK_IMPORTED_MODULE_0___default()(['**/*'], buildProjectPath, { - cwd: Object(path__WEBPACK_IMPORTED_MODULE_2__["join"])(kibanaRoot, 'bazel', 'bin', 'packages', Object(path__WEBPACK_IMPORTED_MODULE_2__["basename"])(buildProjectPath), 'npm_module'), + cwd: Object(path__WEBPACK_IMPORTED_MODULE_2__["join"])(kibanaRoot, 'bazel-bin', 'packages', Object(path__WEBPACK_IMPORTED_MODULE_2__["basename"])(buildProjectPath), 'npm_module'), dot: true, onlyFiles: true, parents: true @@ -63702,12 +63739,12 @@ async function applyCorrectPermissions(project, kibanaRoot, buildRoot) { const buildProjectPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(buildRoot, relativeProjectPath); const allPluginPaths = await globby__WEBPACK_IMPORTED_MODULE_1___default()([`**/*`], { onlyFiles: false, - cwd: Object(path__WEBPACK_IMPORTED_MODULE_2__["join"])(kibanaRoot, 'bazel', 'bin', 'packages', Object(path__WEBPACK_IMPORTED_MODULE_2__["basename"])(buildProjectPath), 'npm_module'), + cwd: buildProjectPath, dot: true }); for (const pluginPath of allPluginPaths) { - const resolvedPluginPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(buildRoot, pluginPath); + const resolvedPluginPath = Object(path__WEBPACK_IMPORTED_MODULE_2__["resolve"])(buildProjectPath, pluginPath); if (await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["isFile"])(resolvedPluginPath)) { await Object(_utils_fs__WEBPACK_IMPORTED_MODULE_5__["chmod"])(resolvedPluginPath, 0o644); diff --git a/packages/kbn-pm/src/cli.ts b/packages/kbn-pm/src/cli.ts index 6d033b4121d992..f6ea4d7124ab27 100644 --- a/packages/kbn-pm/src/cli.ts +++ b/packages/kbn-pm/src/cli.ts @@ -75,7 +75,7 @@ export async function run(argv: string[]) { }, default: { cache: true, - 'force-install': true, + 'force-install': false, offline: false, validate: true, }, diff --git a/packages/kbn-pm/src/commands/bootstrap.ts b/packages/kbn-pm/src/commands/bootstrap.ts index 4a6a43ff2d91f3..b383a52be63f50 100644 --- a/packages/kbn-pm/src/commands/bootstrap.ts +++ b/packages/kbn-pm/src/commands/bootstrap.ts @@ -17,7 +17,12 @@ import { getAllChecksums } from '../utils/project_checksums'; import { BootstrapCacheFile } from '../utils/bootstrap_cache_file'; import { readYarnLock } from '../utils/yarn_lock'; import { validateDependencies } from '../utils/validate_dependencies'; -import { ensureYarnIntegrityFileExists, installBazelTools, runBazel } from '../utils/bazel'; +import { + ensureYarnIntegrityFileExists, + installBazelTools, + runBazel, + yarnIntegrityFileExists, +} from '../utils/bazel'; export const BootstrapCommand: ICommand = { description: 'Install dependencies and crosslink projects', @@ -33,7 +38,13 @@ export const BootstrapCommand: ICommand = { const batchedNonBazelProjects = topologicallyBatchProjects(nonBazelProjectsOnly, projectGraph); const kibanaProjectPath = projects.get('kibana')?.path || ''; const runOffline = options?.offline === true; - const forceInstall = !!options && options['force-install'] === true; + + // Force install is set in case a flag is passed or + // if the `.yarn-integrity` file is not found which + // will be indicated by the return of yarnIntegrityFileExists. + const forceInstall = + (!!options && options['force-install'] === true) || + !(await yarnIntegrityFileExists(resolve(kibanaProjectPath, 'node_modules'))); // Ensure we have a `node_modules/.yarn-integrity` file as we depend on it // for bazel to know it has to re-install the node_modules after a reset or a clean @@ -51,9 +62,6 @@ export const BootstrapCommand: ICommand = { // That way non bazel projects could depend on bazel projects but not the other way around // That is only intended during the migration process while non Bazel projects are not removed at all. // - // Until we have our first package build within Bazel we will always need to directly call the yarn rule - // otherwise yarn install won't trigger as we don't have any npm dependency within Bazel - // TODO: Change CLI default in order to not force install as soon as we have our first Bazel package being built if (forceInstall) { await runBazel(['run', '@nodejs//:yarn'], runOffline); } diff --git a/packages/kbn-pm/src/production/build_bazel_production_projects.ts b/packages/kbn-pm/src/production/build_bazel_production_projects.ts index 313622d44276a4..07c0b651f5ad13 100644 --- a/packages/kbn-pm/src/production/build_bazel_production_projects.ts +++ b/packages/kbn-pm/src/production/build_bazel_production_projects.ts @@ -37,7 +37,7 @@ export async function buildBazelProductionProjects({ log.info(`Preparing Bazel projects production build for [${projectNames.join(', ')}]`); await runBazel(['build', '//packages:build']); - log.info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete}]`); + log.info(`All Bazel projects production builds for [${projectNames.join(', ')}] are complete`); for (const project of projects.values()) { await copyToBuild(project, kibanaRoot, buildRoot); @@ -62,7 +62,7 @@ async function copyToBuild(project: Project, kibanaRoot: string, buildRoot: stri const buildProjectPath = resolve(buildRoot, relativeProjectPath); await copy(['**/*'], buildProjectPath, { - cwd: join(kibanaRoot, 'bazel', 'bin', 'packages', basename(buildProjectPath), 'npm_module'), + cwd: join(kibanaRoot, 'bazel-bin', 'packages', basename(buildProjectPath), 'npm_module'), dot: true, onlyFiles: true, parents: true, @@ -88,12 +88,12 @@ async function applyCorrectPermissions(project: Project, kibanaRoot: string, bui const buildProjectPath = resolve(buildRoot, relativeProjectPath); const allPluginPaths = await globby([`**/*`], { onlyFiles: false, - cwd: join(kibanaRoot, 'bazel', 'bin', 'packages', basename(buildProjectPath), 'npm_module'), + cwd: buildProjectPath, dot: true, }); for (const pluginPath of allPluginPaths) { - const resolvedPluginPath = resolve(buildRoot, pluginPath); + const resolvedPluginPath = resolve(buildProjectPath, pluginPath); if (await isFile(resolvedPluginPath)) { await chmod(resolvedPluginPath, 0o644); } diff --git a/packages/kbn-pm/src/utils/__snapshots__/link_project_executables.test.ts.snap b/packages/kbn-pm/src/utils/__snapshots__/link_project_executables.test.ts.snap index c037c2a4976b43..8aeae04c265cf5 100644 --- a/packages/kbn-pm/src/utils/__snapshots__/link_project_executables.test.ts.snap +++ b/packages/kbn-pm/src/utils/__snapshots__/link_project_executables.test.ts.snap @@ -11,6 +11,7 @@ Object { "mkdirp": Array [], "readFile": Array [], "rmdirp": Array [], + "tryRealpath": Array [], "unlink": Array [], "writeFile": Array [], } @@ -27,6 +28,7 @@ Object { "mkdirp": Array [], "readFile": Array [], "rmdirp": Array [], + "tryRealpath": Array [], "unlink": Array [], "writeFile": Array [], } diff --git a/packages/kbn-pm/src/utils/bazel/ensure_yarn_integrity_exists.ts b/packages/kbn-pm/src/utils/bazel/ensure_yarn_integrity_exists.ts deleted file mode 100644 index 90786bc0ea55e8..00000000000000 --- a/packages/kbn-pm/src/utils/bazel/ensure_yarn_integrity_exists.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { join } from 'path'; -import { writeFile } from '../fs'; - -export async function ensureYarnIntegrityFileExists(nodeModulesPath: string) { - try { - await writeFile(join(nodeModulesPath, '.yarn-integrity'), '', { flag: 'wx' }); - } catch { - // no-op - } -} diff --git a/packages/kbn-pm/src/utils/bazel/index.ts b/packages/kbn-pm/src/utils/bazel/index.ts index 0b755ba2446a04..a3651039161b86 100644 --- a/packages/kbn-pm/src/utils/bazel/index.ts +++ b/packages/kbn-pm/src/utils/bazel/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export * from './ensure_yarn_integrity_exists'; +export * from './yarn_integrity'; export * from './get_cache_folders'; export * from './install_tools'; export * from './run'; diff --git a/packages/kbn-pm/src/utils/bazel/yarn_integrity.ts b/packages/kbn-pm/src/utils/bazel/yarn_integrity.ts new file mode 100644 index 00000000000000..3a72f5ca080b8e --- /dev/null +++ b/packages/kbn-pm/src/utils/bazel/yarn_integrity.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { join } from 'path'; +import { isFile, mkdirp, tryRealpath, writeFile } from '../fs'; + +export async function yarnIntegrityFileExists(nodeModulesPath: string) { + try { + const nodeModulesRealPath = await tryRealpath(nodeModulesPath); + const yarnIntegrityFilePath = join(nodeModulesRealPath, '.yarn-integrity'); + + // check if the file already exists + if (await isFile(yarnIntegrityFilePath)) { + return true; + } + } catch { + // no-op + } + + return false; +} + +export async function ensureYarnIntegrityFileExists(nodeModulesPath: string) { + try { + const nodeModulesRealPath = await tryRealpath(nodeModulesPath); + const yarnIntegrityFilePath = join(nodeModulesRealPath, '.yarn-integrity'); + + // ensure node_modules folder is created + await mkdirp(nodeModulesRealPath); + + // write a blank file in case it doesn't exists + await writeFile(yarnIntegrityFilePath, '', { flag: 'wx' }); + } catch { + // no-op + } +} diff --git a/packages/kbn-pm/src/utils/fs.ts b/packages/kbn-pm/src/utils/fs.ts index dd961b83214464..5739d319e08e7d 100644 --- a/packages/kbn-pm/src/utils/fs.ts +++ b/packages/kbn-pm/src/utils/fs.ts @@ -20,6 +20,7 @@ const symlink = promisify(fs.symlink); export const chmod = promisify(fs.chmod); const cmdShim = promisify(cmdShimCb); const mkdir = promisify(fs.mkdir); +const realpathNative = promisify(fs.realpath.native); export const mkdirp = async (path: string) => await mkdir(path, { recursive: true }); export const rmdirp = async (path: string) => await del(path, { force: true }); export const unlink = promisify(fs.unlink); @@ -96,3 +97,17 @@ async function forceCreate(src: string, dest: string, type: string) { await symlink(src, dest, type); } + +export async function tryRealpath(path: string): Promise { + let calculatedPath = path; + + try { + calculatedPath = await realpathNative(path); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + return calculatedPath; +} diff --git a/packages/kbn-pm/src/utils/package_json.ts b/packages/kbn-pm/src/utils/package_json.ts index b405b544ab800c..e635c2566e65ac 100644 --- a/packages/kbn-pm/src/utils/package_json.ts +++ b/packages/kbn-pm/src/utils/package_json.ts @@ -35,7 +35,7 @@ export const createProductionPackageJson = (pkgJson: IPackageJson) => ({ export const isLinkDependency = (depVersion: string) => depVersion.startsWith('link:'); export const isBazelPackageDependency = (depVersion: string) => - depVersion.startsWith('link:bazel/bin/'); + depVersion.startsWith('link:bazel-bin/'); /** * Replaces `link:` dependencies with `file:` dependencies. When installing @@ -46,7 +46,7 @@ export const isBazelPackageDependency = (depVersion: string) => * will then _copy_ the `file:` dependencies into `node_modules` instead of * symlinking like we do in development. * - * Additionally it also taken care of replacing `link:bazel/bin/` with + * Additionally it also taken care of replacing `link:bazel-bin/` with * `file:` so we can also support the copy of the Bazel packages dist already into * build/packages to be copied into the node_modules */ @@ -61,7 +61,7 @@ export function transformDependencies(dependencies: IPackageDependencies = {}) { } if (isBazelPackageDependency(depVersion)) { - newDeps[name] = depVersion.replace('link:bazel/bin/', 'file:'); + newDeps[name] = depVersion.replace('link:bazel-bin/', 'file:').replace('/npm_module', ''); continue; } diff --git a/packages/kbn-pm/src/utils/project.ts b/packages/kbn-pm/src/utils/project.ts index 797a9a36df78f7..5d2a0547b25772 100644 --- a/packages/kbn-pm/src/utils/project.ts +++ b/packages/kbn-pm/src/utils/project.ts @@ -92,7 +92,10 @@ export class Project { public ensureValidProjectDependency(project: Project) { const relativePathToProject = normalizePath(Path.relative(this.path, project.path)); const relativePathToProjectIfBazelPkg = normalizePath( - Path.relative(this.path, `bazel/bin/packages/${Path.basename(project.path)}`) + Path.relative( + this.path, + `${__dirname}/../../../bazel-bin/packages/${Path.basename(project.path)}/npm_module` + ) ); const versionInPackageJson = this.allDependencies[project.name]; @@ -100,7 +103,7 @@ export class Project { const expectedVersionInPackageJsonIfBazelPkg = `link:${relativePathToProjectIfBazelPkg}`; // TODO: after introduce bazel to build all the packages and completely remove the support for kbn packages - // do not allow child projects to hold dependencies + // do not allow child projects to hold dependencies, unless they are meant to be published externally if ( versionInPackageJson === expectedVersionInPackageJson || versionInPackageJson === expectedVersionInPackageJsonIfBazelPkg diff --git a/packages/kbn-test/jest-preset.js b/packages/kbn-test/jest-preset.js index 4949d6d1f9fad4..225f93d4878238 100644 --- a/packages/kbn-test/jest-preset.js +++ b/packages/kbn-test/jest-preset.js @@ -107,4 +107,7 @@ module.exports = { '!**/*.d.ts', '!**/index.{js,ts}', ], + + // A custom resolver to preserve symlinks by default + resolver: '/packages/kbn-test/target/jest/setup/preserve_symlinks_resolver.js', }; diff --git a/packages/kbn-test/src/jest/setup/preserve_symlinks_resolver.js b/packages/kbn-test/src/jest/setup/preserve_symlinks_resolver.js new file mode 100644 index 00000000000000..711bf2c9aa189c --- /dev/null +++ b/packages/kbn-test/src/jest/setup/preserve_symlinks_resolver.js @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +// Inspired in a discussion found at https://github.com/facebook/jest/issues/5356 as Jest currently doesn't +// offer any other option to preserve symlinks. +// +// It would be available once https://github.com/facebook/jest/pull/9976 got merged. + +const resolve = require('resolve'); + +module.exports = (request, options) => { + try { + return resolve.sync(request, { + basedir: options.basedir, + extensions: options.extensions, + preserveSymlinks: true, + }); + } catch (error) { + if (error.code === 'MODULE_NOT_FOUND') { + return options.defaultResolver(request, options); + } + + throw error; + } +}; diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts index ef3172b620b232..a946640f58b0d1 100644 --- a/src/core/public/doc_links/doc_links_service.ts +++ b/src/core/public/doc_links/doc_links_service.ts @@ -271,8 +271,10 @@ export class DocLinksService { painlessExecute: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/painless/${DOC_LINK_VERSION}/painless-execute-api.html`, painlessExecuteAPIContexts: `${ELASTIC_WEBSITE_URL}guide/en/elasticsearch/painless/${DOC_LINK_VERSION}/painless-execute-api.html#_contexts`, putComponentTemplateMetadata: `${ELASTICSEARCH_DOCS}indices-component-template.html#component-templates-metadata`, + putEnrichPolicy: `${ELASTICSEARCH_DOCS}put-enrich-policy-api.html`, putSnapshotLifecyclePolicy: `${ELASTICSEARCH_DOCS}slm-api-put-policy.html`, - putWatch: `${ELASTICSEARCH_DOCS}/watcher-api-put-watch.html`, + putWatch: `${ELASTICSEARCH_DOCS}watcher-api-put-watch.html`, + simulatePipeline: `${ELASTICSEARCH_DOCS}simulate-pipeline-api.html`, updateTransform: `${ELASTICSEARCH_DOCS}update-transform.html`, }, plugins: { @@ -293,9 +295,47 @@ export class DocLinksService { restoreSnapshotApi: `${ELASTICSEARCH_DOCS}restore-snapshot-api.html#restore-snapshot-api-request-body`, }, ingest: { + append: `${ELASTICSEARCH_DOCS}append-processor.html`, + bytes: `${ELASTICSEARCH_DOCS}bytes-processor.html`, + circle: `${ELASTICSEARCH_DOCS}ingest-circle-processor.html`, + convert: `${ELASTICSEARCH_DOCS}convert-processor.html`, + csv: `${ELASTICSEARCH_DOCS}csv-processor.html`, + date: `${ELASTICSEARCH_DOCS}date-processor.html`, + dateIndexName: `${ELASTICSEARCH_DOCS}date-index-name-processor.html`, + dissect: `${ELASTICSEARCH_DOCS}dissect-processor.html`, + dissectKeyModifiers: `${ELASTICSEARCH_DOCS}dissect-processor.html#dissect-key-modifiers`, + dotExpander: `${ELASTICSEARCH_DOCS}dot-expand-processor.html`, + drop: `${ELASTICSEARCH_DOCS}drop-processor.html`, + enrich: `${ELASTICSEARCH_DOCS}ingest-enriching-data.html`, + fail: `${ELASTICSEARCH_DOCS}fail-processor.html`, + foreach: `${ELASTICSEARCH_DOCS}foreach-processor.html`, + geoIp: `${ELASTICSEARCH_DOCS}geoip-processor.html`, + grok: `${ELASTICSEARCH_DOCS}grok-processor.html`, + gsub: `${ELASTICSEARCH_DOCS}gsub-processor.html`, + htmlString: `${ELASTICSEARCH_DOCS}htmlstrip-processor.html`, + inference: `${ELASTICSEARCH_DOCS}inference-processor.html`, + inferenceClassification: `${ELASTICSEARCH_DOCS}inference-processor.html#inference-processor-classification-opt`, + inferenceRegression: `${ELASTICSEARCH_DOCS}inference-processor.html#inference-processor-regression-opt`, + join: `${ELASTICSEARCH_DOCS}join-processor.html`, + json: `${ELASTICSEARCH_DOCS}json-processor.html`, + kv: `${ELASTICSEARCH_DOCS}kv-processor.html`, + lowercase: `${ELASTICSEARCH_DOCS}lowercase-processor.html`, + pipeline: `${ELASTICSEARCH_DOCS}pipeline-processor.html`, pipelines: `${ELASTICSEARCH_DOCS}ingest.html`, pipelineFailure: `${ELASTICSEARCH_DOCS}ingest.html#handling-pipeline-failures`, processors: `${ELASTICSEARCH_DOCS}processors.html`, + remove: `${ELASTICSEARCH_DOCS}remove-processor.html`, + rename: `${ELASTICSEARCH_DOCS}rename-processor.html`, + script: `${ELASTICSEARCH_DOCS}script-processor.html`, + set: `${ELASTICSEARCH_DOCS}set-processor.html`, + setSecurityUser: `${ELASTICSEARCH_DOCS}ingest-node-set-security-user-processor.html`, + sort: `${ELASTICSEARCH_DOCS}sort-processor.html`, + split: `${ELASTICSEARCH_DOCS}split-processor.html`, + trim: `${ELASTICSEARCH_DOCS}trim-processor.html`, + uppercase: `${ELASTICSEARCH_DOCS}uppercase-processor.html`, + uriParts: `${ELASTICSEARCH_DOCS}uri-parts-processor.html`, + urlDecode: `${ELASTICSEARCH_DOCS}urldecode-processor.html`, + userAgent: `${ELASTICSEARCH_DOCS}user-agent-processor.html`, }, }, }); @@ -443,6 +483,7 @@ export interface DocLinksStart { putComponentTemplateMetadata: string; putSnapshotLifecyclePolicy: string; putWatch: string; + simulatePipeline: string; updateTransform: string; }>; readonly observability: Record; diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 0a1c7a9b0fa360..8327428991e13b 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -627,6 +627,7 @@ export interface DocLinksStart { putComponentTemplateMetadata: string; putSnapshotLifecyclePolicy: string; putWatch: string; + simulatePipeline: string; updateTransform: string; }>; readonly observability: Record; diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts index 61de31e825d33b..530203e659086f 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.mock.ts @@ -35,13 +35,14 @@ const createMigrator = ( ) => { const mockMigrator: jest.Mocked = { kibanaVersion: '8.0.0-testing', - savedObjectsConfig: { + soMigrationsConfig: { batchSize: 100, scrollDuration: '15m', pollInterval: 1500, skip: false, - // TODO migrationsV2: remove/deprecate once we release migrations v2 + // TODO migrationsV2: remove/deprecate once we remove migrations v1 enableV2: false, + retryAttempts: 10, }, runMigrations: jest.fn(), getActiveMappings: jest.fn(), diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts index 7ead37699980a5..40d18c3b5063a6 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts @@ -414,12 +414,13 @@ const mockOptions = ({ enableV2 }: { enableV2: boolean } = { enableV2: false }) enabled: true, index: '.my-index', } as KibanaMigratorOptions['kibanaConfig'], - savedObjectsConfig: { + soMigrationsConfig: { batchSize: 20, pollInterval: 20000, scrollDuration: '10m', skip: false, enableV2, + retryAttempts: 20, }, client: elasticsearchClientMock.createElasticsearchClient(), }; diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts index e5c64914e4c96d..29852f8ac64452 100644 --- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts +++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts @@ -41,7 +41,7 @@ import { MigrationLogger } from '../core/migration_logger'; export interface KibanaMigratorOptions { client: ElasticsearchClient; typeRegistry: ISavedObjectTypeRegistry; - savedObjectsConfig: SavedObjectsMigrationConfigType; + soMigrationsConfig: SavedObjectsMigrationConfigType; kibanaConfig: KibanaConfigType; kibanaVersion: string; logger: Logger; @@ -72,10 +72,10 @@ export class KibanaMigrator { }); private readonly activeMappings: IndexMapping; private migrationsRetryDelay?: number; - // TODO migrationsV2: make private once we release migrations v2 - public kibanaVersion: string; - // TODO migrationsV2: make private once we release migrations v2 - public readonly savedObjectsConfig: SavedObjectsMigrationConfigType; + // TODO migrationsV2: make private once we remove migrations v1 + public readonly kibanaVersion: string; + // TODO migrationsV2: make private once we remove migrations v1 + public readonly soMigrationsConfig: SavedObjectsMigrationConfigType; /** * Creates an instance of KibanaMigrator. @@ -84,14 +84,14 @@ export class KibanaMigrator { client, typeRegistry, kibanaConfig, - savedObjectsConfig, + soMigrationsConfig, kibanaVersion, logger, migrationsRetryDelay, }: KibanaMigratorOptions) { this.client = client; this.kibanaConfig = kibanaConfig; - this.savedObjectsConfig = savedObjectsConfig; + this.soMigrationsConfig = soMigrationsConfig; this.typeRegistry = typeRegistry; this.serializer = new SavedObjectsSerializer(this.typeRegistry); this.mappingProperties = mergeTypes(this.typeRegistry.getAllTypes()); @@ -175,7 +175,7 @@ export class KibanaMigrator { const migrators = Object.keys(indexMap).map((index) => { // TODO migrationsV2: remove old migrations algorithm - if (this.savedObjectsConfig.enableV2) { + if (this.soMigrationsConfig.enableV2) { return { migrate: (): Promise => { return runResilientMigrator({ @@ -193,20 +193,21 @@ export class KibanaMigrator { ), migrationVersionPerType: this.documentMigrator.migrationVersion, indexPrefix: index, + migrationsConfig: this.soMigrationsConfig, }); }, }; } else { return new IndexMigrator({ - batchSize: this.savedObjectsConfig.batchSize, + batchSize: this.soMigrationsConfig.batchSize, client: createMigrationEsClient(this.client, this.log, this.migrationsRetryDelay), documentMigrator: this.documentMigrator, index, kibanaVersion: this.kibanaVersion, log: this.log, mappingProperties: indexMap[index].typeMappings, - pollInterval: this.savedObjectsConfig.pollInterval, - scrollDuration: this.savedObjectsConfig.scrollDuration, + pollInterval: this.soMigrationsConfig.pollInterval, + scrollDuration: this.soMigrationsConfig.scrollDuration, serializer: this.serializer, // Only necessary for the migrator of the kibana index. obsoleteIndexTemplatePattern: diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts index 22dfb03815052d..52fa99b7248737 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts @@ -9,7 +9,7 @@ import * as Either from 'fp-ts/lib/Either'; import * as TaskEither from 'fp-ts/lib/TaskEither'; import * as Option from 'fp-ts/lib/Option'; -import { ElasticsearchClientError } from '@elastic/elasticsearch/lib/errors'; +import { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/lib/errors'; import { pipe } from 'fp-ts/lib/pipeable'; import { errors as EsErrors } from '@elastic/elasticsearch'; import { flow } from 'fp-ts/lib/function'; @@ -23,12 +23,6 @@ import { } from './catch_retryable_es_client_errors'; export type { RetryableEsClientError }; -export const isRetryableEsClientResponse = ( - res: Either.Either -): res is Either.Left => { - return Either.isLeft(res) && res.left.type === 'retryable_es_client_error'; -}; - /** * Batch size for updateByQuery, reindex & search operations. Smaller batches * reduce the memory pressure on Elasticsearch and Kibana so are less likely @@ -45,6 +39,27 @@ const INDEX_NUMBER_OF_SHARDS = 1; /** Wait for all shards to be active before starting an operation */ const WAIT_FOR_ALL_SHARDS_TO_BE_ACTIVE = 'all'; +// Map of left response 'type' string -> response interface +export interface ActionErrorTypeMap { + wait_for_task_completion_timeout: WaitForTaskCompletionTimeout; + retryable_es_client_error: RetryableEsClientError; + index_not_found_exception: IndexNotFound; + target_index_had_write_block: TargetIndexHadWriteBlock; + incompatible_mapping_exception: IncompatibleMappingException; + alias_not_found_exception: AliasNotFound; + remove_index_not_a_concrete_index: RemoveIndexNotAConcreteIndex; +} + +/** + * Type guard for narrowing the type of a left + */ +export function isLeftTypeof( + res: any, + typeString: T +): res is ActionErrorTypeMap[T] { + return res.type === typeString; +} + export type FetchIndexResponse = Record< string, { aliases: Record; mappings: IndexMapping; settings: unknown } @@ -74,6 +89,10 @@ export const fetchIndices = ( .catch(catchRetryableEsClientErrors); }; +export interface IndexNotFound { + type: 'index_not_found_exception'; + index: string; +} /** * Sets a write block in place for the given index. If the response includes * `acknowledged: true` all in-progress writes have drained and no further @@ -87,7 +106,7 @@ export const setWriteBlock = ( client: ElasticsearchClient, index: string ): TaskEither.TaskEither< - { type: 'index_not_found_exception' } | RetryableEsClientError, + IndexNotFound | RetryableEsClientError, 'set_write_block_succeeded' > => () => { return client.indices @@ -112,7 +131,7 @@ export const setWriteBlock = ( .catch((e: ElasticsearchClientError) => { if (e instanceof EsErrors.ResponseError) { if (e.message === 'index_not_found_exception') { - return Either.left({ type: 'index_not_found_exception' as const }); + return Either.left({ type: 'index_not_found_exception' as const, index }); } } throw e; @@ -170,10 +189,11 @@ export const removeWriteBlock = ( */ const waitForIndexStatusYellow = ( client: ElasticsearchClient, - index: string + index: string, + timeout: string ): TaskEither.TaskEither => () => { return client.cluster - .health({ index, wait_for_status: 'yellow', timeout: '30s' }) + .health({ index, wait_for_status: 'yellow', timeout }) .then(() => { return Either.right({}); }) @@ -189,19 +209,18 @@ export type CloneIndexResponse = AcknowledgeResponse; * This method adds some additional logic to the ES clone index API: * - it is idempotent, if it gets called multiple times subsequent calls will * wait for the first clone operation to complete (up to 60s) - * - the first call will wait up to 90s for the cluster state and all shards + * - the first call will wait up to 120s for the cluster state and all shards * to be updated. */ export const cloneIndex = ( client: ElasticsearchClient, source: string, - target: string -): TaskEither.TaskEither< - RetryableEsClientError | { type: 'index_not_found_exception'; index: string }, - CloneIndexResponse -> => { + target: string, + /** only used for testing */ + timeout = DEFAULT_TIMEOUT +): TaskEither.TaskEither => { const cloneTask: TaskEither.TaskEither< - RetryableEsClientError | { type: 'index_not_found_exception'; index: string }, + RetryableEsClientError | IndexNotFound, AcknowledgeResponse > = () => { return client.indices @@ -227,7 +246,7 @@ export const cloneIndex = ( }, }, }, - timeout: DEFAULT_TIMEOUT, + timeout, }, { maxRetries: 0 /** handle retry ourselves for now */ } ) @@ -277,7 +296,7 @@ export const cloneIndex = ( } else { // Otherwise, wait until the target index has a 'green' status. return pipe( - waitForIndexStatusYellow(client, target), + waitForIndexStatusYellow(client, target, timeout), TaskEither.map((value) => { /** When the index status is 'green' we know that all shards were started */ return { acknowledged: true, shardsAcknowledged: true }; @@ -295,6 +314,38 @@ interface WaitForTaskResponse { description?: string; } +/** + * After waiting for the specificed timeout, the task has not yet completed. + * + * When querying the tasks API we use `wait_for_completion=true` to block the + * request until the task completes. If after the `timeout`, the task still has + * not completed we return this error. This does not mean that the task itelf + * has reached a timeout, Elasticsearch will continue to run the task. + */ +export interface WaitForTaskCompletionTimeout { + /** After waiting for the specificed timeout, the task has not yet completed. */ + readonly type: 'wait_for_task_completion_timeout'; + readonly message: string; + readonly error?: Error; +} + +const catchWaitForTaskCompletionTimeout = ( + e: ResponseError +): Either.Either => { + if ( + e.body?.error?.type === 'timeout_exception' || + e.body?.error?.type === 'receive_timeout_transport_exception' + ) { + return Either.left({ + type: 'wait_for_task_completion_timeout' as const, + message: `[${e.body.error.type}] ${e.body.error.reason}`, + error: e, + }); + } else { + throw e; + } +}; + /** * Blocks for up to 60s or until a task completes. * @@ -304,7 +355,10 @@ const waitForTask = ( client: ElasticsearchClient, taskId: string, timeout: string -): TaskEither.TaskEither => () => { +): TaskEither.TaskEither< + RetryableEsClientError | WaitForTaskCompletionTimeout, + WaitForTaskResponse +> => () => { return client.tasks .get({ task_id: taskId, @@ -322,6 +376,7 @@ const waitForTask = ( description: body.task.description, }); }) + .catch(catchWaitForTaskCompletionTimeout) .catch(catchRetryableEsClientErrors); }; @@ -424,7 +479,15 @@ export const reindex = ( }; interface WaitForReindexTaskFailure { - cause: { type: string; reason: string }; + readonly cause: { type: string; reason: string }; +} + +export interface TargetIndexHadWriteBlock { + type: 'target_index_had_write_block'; +} + +export interface IncompatibleMappingException { + type: 'incompatible_mapping_exception'; } export const waitForReindexTask = flow( @@ -433,10 +496,11 @@ export const waitForReindexTask = flow( ( res ): TaskEither.TaskEither< - | { type: 'index_not_found_exception'; index: string } - | { type: 'target_index_had_write_block' } - | { type: 'incompatible_mapping_exception' } - | RetryableEsClientError, + | IndexNotFound + | TargetIndexHadWriteBlock + | IncompatibleMappingException + | RetryableEsClientError + | WaitForTaskCompletionTimeout, 'reindex_succeeded' > => { const failureIsAWriteBlock = ({ cause: { type, reason } }: WaitForReindexTaskFailure) => @@ -507,7 +571,12 @@ export const verifyReindex = ( export const waitForPickupUpdatedMappingsTask = flow( waitForTask, TaskEither.chain( - (res): TaskEither.TaskEither => { + ( + res + ): TaskEither.TaskEither< + RetryableEsClientError | WaitForTaskCompletionTimeout, + 'pickup_updated_mappings_succeeded' + > => { // We don't catch or type failures/errors because they should never // occur in our migration algorithm and we don't have any business logic // for dealing with it. If something happens we'll just crash and try @@ -529,6 +598,14 @@ export const waitForPickupUpdatedMappingsTask = flow( ) ); +export interface AliasNotFound { + type: 'alias_not_found_exception'; +} + +export interface RemoveIndexNotAConcreteIndex { + type: 'remove_index_not_a_concrete_index'; +} + export type AliasAction = | { remove_index: { index: string } } | { remove: { index: string; alias: string; must_exist: boolean } } @@ -541,10 +618,7 @@ export const updateAliases = ( client: ElasticsearchClient, aliasActions: AliasAction[] ): TaskEither.TaskEither< - | { type: 'index_not_found_exception'; index: string } - | { type: 'alias_not_found_exception' } - | { type: 'remove_index_not_a_concrete_index' } - | RetryableEsClientError, + IndexNotFound | AliasNotFound | RemoveIndexNotAConcreteIndex | RetryableEsClientError, 'update_aliases_succeeded' > => () => { return client.indices @@ -698,11 +772,11 @@ export const createIndex = ( // If the cluster state was updated and all shards ackd we're done return TaskEither.right('create_index_succeeded'); } else { - // Otherwise, wait until the target index has a 'green' status. + // Otherwise, wait until the target index has a 'yellow' status. return pipe( - waitForIndexStatusYellow(client, indexName), + waitForIndexStatusYellow(client, indexName, DEFAULT_TIMEOUT), TaskEither.map(() => { - /** When the index status is 'green' we know that all shards were started */ + /** When the index status is 'yellow' we know that all shards were started */ return 'create_index_succeeded'; }) ); diff --git a/src/core/server/saved_objects/migrationsv2/index.ts b/src/core/server/saved_objects/migrationsv2/index.ts index 0297aefdc7abdd..6e65a2e700fd30 100644 --- a/src/core/server/saved_objects/migrationsv2/index.ts +++ b/src/core/server/saved_objects/migrationsv2/index.ts @@ -14,6 +14,7 @@ import { MigrationResult } from '../migrations/core'; import { next, TransformRawDocs } from './next'; import { createInitialState, model } from './model'; import { migrationStateActionMachine } from './migrations_state_action_machine'; +import { SavedObjectsMigrationConfigType } from '../saved_objects_config'; /** * Migrates the provided indexPrefix index using a resilient algorithm that is @@ -29,6 +30,7 @@ export async function runResilientMigrator({ transformRawDocs, migrationVersionPerType, indexPrefix, + migrationsConfig, }: { client: ElasticsearchClient; kibanaVersion: string; @@ -38,6 +40,7 @@ export async function runResilientMigrator({ transformRawDocs: TransformRawDocs; migrationVersionPerType: SavedObjectsMigrationVersion; indexPrefix: string; + migrationsConfig: SavedObjectsMigrationConfigType; }): Promise { const initialState = createInitialState({ kibanaVersion, @@ -45,6 +48,7 @@ export async function runResilientMigrator({ preMigrationScript, migrationVersionPerType, indexPrefix, + migrationsConfig, }); return migrationStateActionMachine({ initialState, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts index 2c052a87d028b5..1824efa0ed8d44 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts @@ -33,6 +33,7 @@ import { } from '../actions'; import * as Either from 'fp-ts/lib/Either'; import * as Option from 'fp-ts/lib/Option'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; const { startES } = kbnTestServer.createTestServers({ adjustTimeout: (t: number) => jest.setTimeout(t), @@ -162,6 +163,7 @@ describe('migration actions', () => { Object { "_tag": "Left", "left": Object { + "index": "no_such_index", "type": "index_not_found_exception", }, } @@ -291,6 +293,45 @@ describe('migration actions', () => { } `); }); + it('resolves left with a retryable_es_client_error if clone target already exists but takes longer than the specified timeout before turning yellow', async () => { + // Create a red index + await client.indices + .create({ + index: 'clone_red_index', + timeout: '5s', + body: { + mappings: { properties: {} }, + settings: { + // Allocate 1 replica so that this index stays yellow + number_of_replicas: '1', + // Disable all shard allocation so that the index status is red + 'index.routing.allocation.enable': 'none', + }, + }, + }) + .catch((e) => {}); + + // Call clone even though the index already exists + const cloneIndexPromise = cloneIndex( + client, + 'existing_index_with_write_block', + 'clone_red_index', + '0s' + )(); + + await cloneIndexPromise.then((res) => { + expect(res).toMatchInlineSnapshot(` + Object { + "_tag": "Left", + "left": Object { + "error": [ResponseError: Response Error], + "message": "Response Error", + "type": "retryable_es_client_error", + }, + } + `); + }); + }); }); // Reindex doesn't return any errors on it's own, so we have to test @@ -587,6 +628,28 @@ describe('migration actions', () => { } `); }); + it('resolves left wait_for_task_completion_timeout when the task does not finish within the timeout', async () => { + const res = (await reindex( + client, + 'existing_index_with_docs', + 'reindex_target', + Option.none, + false + )()) as Either.Right; + + const task = waitForReindexTask(client, res.right.taskId, '0s'); + + await expect(task()).resolves.toMatchObject({ + _tag: 'Left', + left: { + error: expect.any(ResponseError), + message: expect.stringMatching( + /\[timeout_exception\] Timed out waiting for completion of \[org.elasticsearch.index.reindex.BulkByScrollTask/ + ), + type: 'wait_for_task_completion_timeout', + }, + }); + }); }); describe('verifyReindex', () => { @@ -702,6 +765,25 @@ describe('migration actions', () => { {"type":"index_not_found_exception","reason":"no such index [no_such_index]","resource.type":"index_or_alias","resource.id":"no_such_index","index_uuid":"_na_","index":"no_such_index"}] `); }); + it('resolves left wait_for_task_completion_timeout when the task does not complete within the timeout', async () => { + const res = (await pickupUpdatedMappings( + client, + 'existing_index_with_docs' + )()) as Either.Right; + + const task = waitForPickupUpdatedMappingsTask(client, res.right.taskId, '0s'); + + await expect(task()).resolves.toMatchObject({ + _tag: 'Left', + left: { + error: expect.any(ResponseError), + message: expect.stringMatching( + /\[timeout_exception\] Timed out waiting for completion of \[org.elasticsearch.index.reindex.BulkByScrollTask/ + ), + type: 'wait_for_task_completion_timeout', + }, + }); + }); it('resolves right when successful', async () => { const res = (await pickupUpdatedMappings( client, diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index f7b9c4c368fa0c..99c06c0a3586ba 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -27,6 +27,14 @@ describe('migrationsStateActionMachine', () => { targetMappings: { properties: {} }, migrationVersionPerType: {}, indexPrefix: '.my-so-index', + migrationsConfig: { + batchSize: 1000, + pollInterval: 0, + scrollDuration: '0s', + skip: false, + enableV2: true, + retryAttempts: 5, + }, }); const next = jest.fn((s: State) => { @@ -221,6 +229,7 @@ describe('migrationsStateActionMachine', () => { "_tag": "None", }, "reason": "the fatal reason", + "retryAttempts": 5, "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { @@ -280,6 +289,7 @@ describe('migrationsStateActionMachine', () => { "_tag": "None", }, "reason": "the fatal reason", + "retryAttempts": 5, "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { @@ -424,6 +434,7 @@ describe('migrationsStateActionMachine', () => { "_tag": "None", }, "reason": "the fatal reason", + "retryAttempts": 5, "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { @@ -478,6 +489,7 @@ describe('migrationsStateActionMachine', () => { "_tag": "None", }, "reason": "the fatal reason", + "retryAttempts": 5, "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index 5531f847f8bb41..2813f01093e950 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -35,6 +35,7 @@ import { SavedObjectsRawDoc } from '..'; import { AliasAction, RetryableEsClientError } from './actions'; import { createInitialState, model } from './model'; import { ResponseType } from './next'; +import { SavedObjectsMigrationConfigType } from '../saved_objects_config'; describe('migrations v2 model', () => { const baseState: BaseState = { @@ -44,6 +45,7 @@ describe('migrations v2 model', () => { logs: [], retryCount: 0, retryDelay: 0, + retryAttempts: 15, indexPrefix: '.kibana', outdatedDocumentsQuery: {}, targetIndexMappings: { @@ -160,15 +162,15 @@ describe('migrations v2 model', () => { expect(newState.retryDelay).toEqual(0); }); - test('terminates to FATAL after 10 retries', () => { + test('terminates to FATAL after retryAttempts retries', () => { const newState = model( - { ...state, ...{ retryCount: 10, retryDelay: 64000 } }, + { ...state, ...{ retryCount: 15, retryDelay: 64000 } }, Either.left(retryableError) ) as FatalState; expect(newState.controlState).toEqual('FATAL'); expect(newState.reason).toMatchInlineSnapshot( - `"Unable to complete the INIT step after 10 attempts, terminating."` + `"Unable to complete the INIT step after 15 attempts, terminating."` ); }); }); @@ -610,6 +612,7 @@ describe('migrations v2 model', () => { test('LEGACY_SET_WRITE_BLOCK -> LEGACY_CREATE_REINDEX_TARGET if action fails with index_not_found_exception', () => { const res: ResponseType<'LEGACY_SET_WRITE_BLOCK'> = Either.left({ type: 'index_not_found_exception', + index: 'legacy_index_name', }); const newState = model(legacySetWriteBlockState, res); expect(newState.controlState).toEqual('LEGACY_CREATE_REINDEX_TARGET'); @@ -707,6 +710,16 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); + test('LEGACY_REINDEX_WAIT_FOR_TASK -> LEGACY_REINDEX_WAIT_FOR_TASK if action fails with wait_for_task_completion_timeout', () => { + const res: ResponseType<'LEGACY_REINDEX_WAIT_FOR_TASK'> = Either.left({ + message: '[timeout_exception] Timeout waiting for ...', + type: 'wait_for_task_completion_timeout', + }); + const newState = model(legacyReindexWaitForTaskState, res); + expect(newState.controlState).toEqual('LEGACY_REINDEX_WAIT_FOR_TASK'); + expect(newState.retryCount).toEqual(1); + expect(newState.retryDelay).toEqual(2000); + }); }); describe('LEGACY_DELETE', () => { const legacyDeleteState: LegacyDeleteState = { @@ -846,6 +859,16 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); + test('REINDEX_SOURCE_TO_TEMP_WAIT_FOR_TASK -> REINDEX_SOURCE_TO_TEMP_WAIT_FOR_TASK when response is left wait_for_task_completion_timeout', () => { + const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_WAIT_FOR_TASK'> = Either.left({ + message: '[timeout_exception] Timeout waiting for ...', + type: 'wait_for_task_completion_timeout', + }); + const newState = model(state, res); + expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_WAIT_FOR_TASK'); + expect(newState.retryCount).toEqual(1); + expect(newState.retryDelay).toEqual(2000); + }); }); describe('SET_TEMP_WRITE_BLOCK', () => { const state: SetTempWriteBlock = { @@ -1025,6 +1048,19 @@ describe('migrations v2 model', () => { expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); }); + test('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK -> UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK when response is left wait_for_task_completion_timeout', () => { + const res: ResponseType<'UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK'> = Either.left({ + message: '[timeout_exception] Timeout waiting for ...', + type: 'wait_for_task_completion_timeout', + }); + const newState = model( + updateTargetMappingsWaitForTaskState, + res + ) as UpdateTargetMappingsWaitForTaskState; + expect(newState.controlState).toEqual('UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK'); + expect(newState.retryCount).toEqual(1); + expect(newState.retryDelay).toEqual(2000); + }); }); describe('CREATE_NEW_TARGET', () => { const aliasActions = Option.some([Symbol('alias action')] as unknown) as Option.Some< @@ -1144,6 +1180,9 @@ describe('migrations v2 model', () => { }); }); describe('createInitialState', () => { + const migrationsConfig = ({ + retryAttempts: 15, + } as unknown) as SavedObjectsMigrationConfigType; it('creates the initial state for the model based on the passed in paramaters', () => { expect( createInitialState({ @@ -1154,6 +1193,7 @@ describe('migrations v2 model', () => { }, migrationVersionPerType: {}, indexPrefix: '.kibana_task_manager', + migrationsConfig, }) ).toMatchInlineSnapshot(` Object { @@ -1171,6 +1211,7 @@ describe('migrations v2 model', () => { "preMigrationScript": Object { "_tag": "None", }, + "retryAttempts": 15, "retryCount": 0, "retryDelay": 0, "targetIndexMappings": Object { @@ -1214,6 +1255,7 @@ describe('migrations v2 model', () => { preMigrationScript, migrationVersionPerType: {}, indexPrefix: '.kibana_task_manager', + migrationsConfig, }); expect(Option.isSome(initialState.preMigrationScript)).toEqual(true); @@ -1233,6 +1275,7 @@ describe('migrations v2 model', () => { preMigrationScript: undefined, migrationVersionPerType: {}, indexPrefix: '.kibana_task_manager', + migrationsConfig, }).preMigrationScript ) ).toEqual(true); @@ -1248,6 +1291,7 @@ describe('migrations v2 model', () => { preMigrationScript: "ctx._id = ctx._source.type + ':' + ctx._id", migrationVersionPerType: { my_dashboard: '7.10.1', my_viz: '8.0.0' }, indexPrefix: '.kibana_task_manager', + migrationsConfig, }).outdatedDocumentsQuery ).toMatchInlineSnapshot(` Object { diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts index 2e92f34429ea9f..5bdba980267925 100644 --- a/src/core/server/saved_objects/migrationsv2/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model.ts @@ -10,33 +10,13 @@ import { gt, valid } from 'semver'; import * as Either from 'fp-ts/lib/Either'; import * as Option from 'fp-ts/lib/Option'; import { cloneDeep } from 'lodash'; -import { AliasAction, FetchIndexResponse, RetryableEsClientError } from './actions'; +import { AliasAction, FetchIndexResponse, isLeftTypeof, RetryableEsClientError } from './actions'; import { AllActionStates, InitState, State } from './types'; import { IndexMapping } from '../mappings'; import { ResponseType } from './next'; import { SavedObjectsMigrationVersion } from '../types'; import { disableUnknownTypeMappingFields } from '../migrations/core/migration_context'; - -/** - * How many times to retry a failing step. - * - * Waiting for a task to complete will cause a failing step every time the - * wait_for_task action times out e.g. the following sequence has 3 retry - * attempts: - * LEGACY_REINDEX_WAIT_FOR_TASK (60s timeout) -> - * LEGACY_REINDEX_WAIT_FOR_TASK (2s delay, 60s timeout) -> - * LEGACY_REINDEX_WAIT_FOR_TASK (4s delay, 60s timeout) -> - * LEGACY_REINDEX_WAIT_FOR_TASK (success) -> ... - * - * This places an upper limit to how long we will wait for a task to complete. - * The duration of a step is the time it takes for the action to complete plus - * the exponential retry delay: - * max_task_runtime = 2+4+8+16+32+64*(MAX_RETRY_ATTEMPTS-5) + ACTION_DURATION*MAX_RETRY_ATTEMPTS - * - * For MAX_RETRY_ATTEMPTS=10, ACTION_DURATION=60 - * max_task_runtime = 16.46 minutes - */ -const MAX_RETRY_ATTEMPTS = 10; +import { SavedObjectsMigrationConfigType } from '../saved_objects_config'; /** * A helper function/type for ensuring that all control state's are handled. @@ -115,12 +95,17 @@ function getAliases(indices: FetchIndexResponse) { }, {} as Record); } -const delayRetryState = (state: S, left: RetryableEsClientError): S => { - if (state.retryCount === MAX_RETRY_ATTEMPTS) { +const delayRetryState = ( + state: S, + errorMessage: string, + /** How many times to retry a step that fails */ + maxRetryAttempts: number +): S => { + if (state.retryCount >= maxRetryAttempts) { return { ...state, controlState: 'FATAL', - reason: `Unable to complete the ${state.controlState} step after ${MAX_RETRY_ATTEMPTS} attempts, terminating.`, + reason: `Unable to complete the ${state.controlState} step after ${maxRetryAttempts} attempts, terminating.`, }; } else { const retryCount = state.retryCount + 1; @@ -134,9 +119,7 @@ const delayRetryState = (state: S, left: RetryableEsClientError ...state.logs, { level: 'error', - message: `Action failed with '${ - left.message - }'. Retrying attempt ${retryCount} out of ${MAX_RETRY_ATTEMPTS} in ${ + message: `Action failed with '${errorMessage}'. Retrying attempt ${retryCount} in ${ retryDelay / 1000 } seconds.`, }, @@ -175,9 +158,12 @@ export const model = (currentState: State, resW: ResponseType): // Handle retryable_es_client_errors. Other left values need to be handled // by the control state specific code below. - if (Either.isLeft(resW) && resW.left.type === 'retryable_es_client_error') { + if ( + Either.isLeft(resW) && + isLeftTypeof(resW.left, 'retryable_es_client_error') + ) { // Retry the same step after an exponentially increasing delay. - return delayRetryState(stateP, resW.left); + return delayRetryState(stateP, resW.left.message, stateP.retryAttempts); } else { // If the action didn't fail with a retryable_es_client_error, reset the // retry counter and retryDelay state @@ -333,7 +319,7 @@ export const model = (currentState: State, resW: ResponseType): // If the write block failed because the index doesn't exist, it means // another instance already completed the legacy pre-migration. Proceed // to the next step. - if (res.left.type === 'index_not_found_exception') { + if (isLeftTypeof(res.left, 'index_not_found_exception')) { return { ...stateP, controlState: 'LEGACY_CREATE_REINDEX_TARGET' }; } else { // @ts-expect-error TS doesn't correctly narrow this type to never @@ -376,8 +362,8 @@ export const model = (currentState: State, resW: ResponseType): } else { const left = res.left; if ( - (left.type === 'index_not_found_exception' && left.index === stateP.legacyIndex) || - left.type === 'target_index_had_write_block' + (isLeftTypeof(left, 'index_not_found_exception') && left.index === stateP.legacyIndex) || + isLeftTypeof(left, 'target_index_had_write_block') ) { // index_not_found_exception for the LEGACY_REINDEX source index: // another instance already complete the LEGACY_DELETE step. @@ -390,12 +376,23 @@ export const model = (currentState: State, resW: ResponseType): // step. However, by not skipping ahead we limit branches in the // control state progression and simplify the implementation. return { ...stateP, controlState: 'LEGACY_DELETE' }; - } else { + } else if (isLeftTypeof(left, 'wait_for_task_completion_timeout')) { + // After waiting for the specificed timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticeasrch task succeeds or fails. + return delayRetryState(stateP, left.message, Number.MAX_SAFE_INTEGER); + } else if ( + isLeftTypeof(left, 'index_not_found_exception') || + isLeftTypeof(left, 'incompatible_mapping_exception') + ) { // We don't handle the following errors as the algorithm will never // run into these during the LEGACY_REINDEX_WAIT_FOR_TASK step: // - index_not_found_exception for the LEGACY_REINDEX target index - // - strict_dynamic_mapping_exception + // - incompatible_mapping_exception throwBadResponse(stateP, left as never); + } else { + throwBadResponse(stateP, left); } } } else if (stateP.controlState === 'LEGACY_DELETE') { @@ -405,8 +402,8 @@ export const model = (currentState: State, resW: ResponseType): } else if (Either.isLeft(res)) { const left = res.left; if ( - left.type === 'remove_index_not_a_concrete_index' || - (left.type === 'index_not_found_exception' && left.index === stateP.legacyIndex) + isLeftTypeof(left, 'remove_index_not_a_concrete_index') || + (isLeftTypeof(left, 'index_not_found_exception') && left.index === stateP.legacyIndex) ) { // index_not_found_exception, another Kibana instance already // deleted the legacy index @@ -419,13 +416,18 @@ export const model = (currentState: State, resW: ResponseType): // step. However, by not skipping ahead we limit branches in the // control state progression and simplify the implementation. return { ...stateP, controlState: 'SET_SOURCE_WRITE_BLOCK' }; - } else { + } else if ( + isLeftTypeof(left, 'index_not_found_exception') || + isLeftTypeof(left, 'alias_not_found_exception') + ) { // We don't handle the following errors as the migration algorithm // will never cause them to occur: // - alias_not_found_exception we're not using must_exist // - index_not_found_exception for source index into which we reindex // the legacy index throwBadResponse(stateP, left as never); + } else { + throwBadResponse(stateP, left); } } else { throwBadResponse(stateP, res); @@ -438,11 +440,13 @@ export const model = (currentState: State, resW: ResponseType): ...stateP, controlState: 'CREATE_REINDEX_TEMP', }; - } else { + } else if (isLeftTypeof(res.left, 'index_not_found_exception')) { // We don't handle the following errors as the migration algorithm // will never cause them to occur: // - index_not_found_exception - return throwBadResponse(stateP, res as never); + return throwBadResponse(stateP, res.left as never); + } else { + return throwBadResponse(stateP, res.left); } } else if (stateP.controlState === 'CREATE_REINDEX_TEMP') { const res = resW as ExcludeRetryableEsError>; @@ -477,8 +481,8 @@ export const model = (currentState: State, resW: ResponseType): } else { const left = res.left; if ( - left.type === 'target_index_had_write_block' || - (left.type === 'index_not_found_exception' && left.index === stateP.tempIndex) + isLeftTypeof(left, 'target_index_had_write_block') || + (isLeftTypeof(left, 'index_not_found_exception') && left.index === stateP.tempIndex) ) { // index_not_found_exception: // another instance completed the MARK_VERSION_INDEX_READY and @@ -493,10 +497,25 @@ export const model = (currentState: State, resW: ResponseType): ...stateP, controlState: 'SET_TEMP_WRITE_BLOCK', }; - } else { - // Don't handle incompatible_mapping_exception as we will never add a write - // block to the temp index or change the mappings. + } else if (isLeftTypeof(left, 'wait_for_task_completion_timeout')) { + // After waiting for the specificed timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticeasrch task succeeds or fails. + return delayRetryState(stateP, left.message, Number.MAX_SAFE_INTEGER); + } else if ( + isLeftTypeof(left, 'index_not_found_exception') || + isLeftTypeof(left, 'incompatible_mapping_exception') + ) { + // Don't handle the following errors as the migration algorithm should + // never cause them to occur: + // - incompatible_mapping_exception the temp index has `dynamic: false` + // mappings + // - index_not_found_exception for the source index, we will never + // delete the source index throwBadResponse(stateP, left as never); + } else { + throwBadResponse(stateP, left); } } } else if (stateP.controlState === 'SET_TEMP_WRITE_BLOCK') { @@ -508,7 +527,7 @@ export const model = (currentState: State, resW: ResponseType): }; } else { const left = res.left; - if (left.type === 'index_not_found_exception') { + if (isLeftTypeof(left, 'index_not_found_exception')) { // index_not_found_exception: // another instance completed the MARK_VERSION_INDEX_READY and // removed the temp index. @@ -520,7 +539,6 @@ export const model = (currentState: State, resW: ResponseType): controlState: 'CLONE_TEMP_TO_TARGET', }; } else { - // @ts-expect-error TS doesn't correctly narrow this to never throwBadResponse(stateP, left); } } @@ -533,7 +551,7 @@ export const model = (currentState: State, resW: ResponseType): }; } else { const left = res.left; - if (left.type === 'index_not_found_exception') { + if (isLeftTypeof(left, 'index_not_found_exception')) { // index_not_found_exception means another instance alread completed // the MARK_VERSION_INDEX_READY step and removed the temp index // We still perform the OUTDATED_DOCUMENTS_* and @@ -543,8 +561,9 @@ export const model = (currentState: State, resW: ResponseType): ...stateP, controlState: 'OUTDATED_DOCUMENTS_SEARCH', }; + } else { + throwBadResponse(stateP, left); } - throwBadResponse(stateP, res as never); } } else if (stateP.controlState === 'OUTDATED_DOCUMENTS_SEARCH') { const res = resW as ExcludeRetryableEsError>; @@ -611,7 +630,16 @@ export const model = (currentState: State, resW: ResponseType): }; } } else { - throwBadResponse(stateP, res); + const left = res.left; + if (isLeftTypeof(left, 'wait_for_task_completion_timeout')) { + // After waiting for the specificed timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticeasrch task succeeds or fails. + return delayRetryState(stateP, res.left.message, Number.MAX_SAFE_INTEGER); + } else { + throwBadResponse(stateP, left); + } } } else if (stateP.controlState === 'CREATE_NEW_TARGET') { const res = resW as ExcludeRetryableEsError>; @@ -632,13 +660,13 @@ export const model = (currentState: State, resW: ResponseType): return { ...stateP, controlState: 'DONE' }; } else { const left = res.left; - if (left.type === 'alias_not_found_exception') { + if (isLeftTypeof(left, 'alias_not_found_exception')) { // the versionIndexReadyActions checks that the currentAlias is still // pointing to the source index. If this fails with an // alias_not_found_exception another instance has completed a // migration from the same source. return { ...stateP, controlState: 'MARK_VERSION_INDEX_READY_CONFLICT' }; - } else if (left.type === 'index_not_found_exception') { + } else if (isLeftTypeof(left, 'index_not_found_exception')) { if (left.index === stateP.tempIndex) { // another instance has already completed the migration and deleted // the temporary index @@ -649,7 +677,7 @@ export const model = (currentState: State, resW: ResponseType): // index handled above. throwBadResponse(stateP, left as never); } - } else if (left.type === 'remove_index_not_a_concrete_index') { + } else if (isLeftTypeof(left, 'remove_index_not_a_concrete_index')) { // We don't handle this error as the migration algorithm will never // cause it to occur (this error is only relevant to the LEGACY_DELETE // step). @@ -708,12 +736,14 @@ export const createInitialState = ({ preMigrationScript, migrationVersionPerType, indexPrefix, + migrationsConfig, }: { kibanaVersion: string; targetMappings: IndexMapping; preMigrationScript?: string; migrationVersionPerType: SavedObjectsMigrationVersion; indexPrefix: string; + migrationsConfig: SavedObjectsMigrationConfigType; }): InitState => { const outdatedDocumentsQuery = { bool: { @@ -753,6 +783,7 @@ export const createInitialState = ({ outdatedDocumentsQuery, retryCount: 0, retryDelay: 0, + retryAttempts: migrationsConfig.retryAttempts, logs: [], }; return initialState; diff --git a/src/core/server/saved_objects/migrationsv2/state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/state_action_machine.test.ts index 6625c446e22825..ebbb540c9b4fdf 100644 --- a/src/core/server/saved_objects/migrationsv2/state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/state_action_machine.test.ts @@ -89,12 +89,4 @@ describe('state action machine', () => { } `); }); - - test("rejects if control state doesn't change after 50 steps", async () => { - await expect( - stateActionMachine(state, next, countUntilModel(51)) - ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Control state didn't change after 50 steps aborting."` - ); - }); }); diff --git a/src/core/server/saved_objects/migrationsv2/state_action_machine.ts b/src/core/server/saved_objects/migrationsv2/state_action_machine.ts index c5aa4bf7c42c6c..b011ab694e1454 100644 --- a/src/core/server/saved_objects/migrationsv2/state_action_machine.ts +++ b/src/core/server/saved_objects/migrationsv2/state_action_machine.ts @@ -10,8 +10,6 @@ export interface ControlState { controlState: string; } -const MAX_STEPS_WITHOUT_CONTROL_STATE_CHANGE = 50; - /** * A state-action machine next function that returns the next action thunk * based on the passed in state. @@ -65,7 +63,6 @@ export async function stateActionMachine( model: Model ) { let state = initialState; - let controlStateStepCounter = 0; let nextAction = next(state); while (nextAction != null) { @@ -73,15 +70,6 @@ export async function stateActionMachine( const actionResponse = await nextAction(); const newState = model(state, actionResponse); - controlStateStepCounter = - newState.controlState === state.controlState ? controlStateStepCounter + 1 : 0; - if (controlStateStepCounter >= MAX_STEPS_WITHOUT_CONTROL_STATE_CHANGE) { - // This is just a fail-safe to ensure we don't get stuck in an infinite loop - throw new Error( - `Control state didn't change after ${MAX_STEPS_WITHOUT_CONTROL_STATE_CHANGE} steps aborting.` - ); - } - // Get ready for the next step state = newState; nextAction = next(state); diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts index b8d67d04b33345..dbdd5774dfa62d 100644 --- a/src/core/server/saved_objects/migrationsv2/types.ts +++ b/src/core/server/saved_objects/migrationsv2/types.ts @@ -37,6 +37,23 @@ export interface BaseState extends ControlState { readonly outdatedDocumentsQuery: Record; readonly retryCount: number; readonly retryDelay: number; + /** + * How many times to retry a step that fails with retryable_es_client_error + * such as a statusCode: 503 or a snapshot_in_progress_exception. + * + * We don't want to immediately crash Kibana and cause a reboot for these + * intermittent. However, if we're still receiving e.g. a 503 after 10 minutes + * this is probably not just a temporary problem so we stop trying and exit + * with a fatal error. + * + * Because of the exponential backoff the total time we will retry such errors + * is: + * max_retry_time = 2+4+8+16+32+64*(RETRY_ATTEMPTS-5) + ACTION_DURATION*RETRY_ATTEMPTS + * + * For RETRY_ATTEMPTS=15 (default), ACTION_DURATION=0 + * max_retry_time = 11.7 minutes + */ + readonly retryAttempts: number; readonly logs: Array<{ level: 'error' | 'info'; message: string }>; /** * The current alias e.g. `.kibana` which always points to the latest diff --git a/src/core/server/saved_objects/saved_objects_config.ts b/src/core/server/saved_objects/saved_objects_config.ts index 1806bb6e0c8954..7228cb126d286b 100644 --- a/src/core/server/saved_objects/saved_objects_config.ts +++ b/src/core/server/saved_objects/saved_objects_config.ts @@ -13,12 +13,14 @@ export type SavedObjectsMigrationConfigType = TypeOf { beforeEach(async function () { await esArchiver.load('management'); diff --git a/x-pack/package.json b/x-pack/package.json index 14c59cf89a74ef..9e963881450380 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -37,7 +37,6 @@ "@kbn/utility-types": "link:../packages/kbn-utility-types" }, "dependencies": { - "@elastic/datemath": "link:../packages/elastic-datemath", "@elastic/safer-lodash-set": "link:../packages/elastic-safer-lodash-set", "@kbn/config-schema": "link:../packages/kbn-config-schema", "@kbn/i18n": "link:../packages/kbn-i18n", diff --git a/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx b/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx index 76020d0b48073d..75f3cca05c5c53 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/correlations_table.tsx @@ -24,12 +24,10 @@ import { ImpactBar } from '../../shared/ImpactBar'; import { useUiTracker } from '../../../../../observability/public'; type CorrelationsApiResponse = - | APIReturnType<'GET /api/apm/correlations/failed_transactions'> - | APIReturnType<'GET /api/apm/correlations/slow_transactions'>; + | APIReturnType<'GET /api/apm/correlations/errors/failed_transactions'> + | APIReturnType<'GET /api/apm/correlations/latency/slow_transactions'>; -type SignificantTerm = NonNullable< - NonNullable['significantTerms'] ->[0]; +type SignificantTerm = CorrelationsApiResponse['significantTerms'][0]; export type SelectedSignificantTerm = Pick< SignificantTerm, diff --git a/x-pack/plugins/apm/public/components/app/correlations/error_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/error_correlations.tsx index c3b5f52dd84b7f..7fb7444a52f848 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/error_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/error_correlations.tsx @@ -34,8 +34,12 @@ import { useFieldNames } from './use_field_names'; import { useLocalStorage } from '../../../hooks/useLocalStorage'; import { useUiTracker } from '../../../../../observability/public'; +type OverallErrorsApiResponse = NonNullable< + APIReturnType<'GET /api/apm/correlations/errors/overall_timeseries'> +>; + type CorrelationsApiResponse = NonNullable< - APIReturnType<'GET /api/apm/correlations/failed_transactions'> + APIReturnType<'GET /api/apm/correlations/errors/failed_transactions'> >; interface Props { @@ -65,11 +69,41 @@ export function ErrorCorrelations({ onClose }: Props) { ); const hasFieldNames = fieldNames.length > 0; - const { data, status } = useFetcher( + const { data: overallData, status: overallStatus } = useFetcher( + (callApmApi) => { + if (start && end) { + return callApmApi({ + endpoint: 'GET /api/apm/correlations/errors/overall_timeseries', + params: { + query: { + environment, + kuery, + serviceName, + transactionName, + transactionType, + start, + end, + }, + }, + }); + } + }, + [ + environment, + kuery, + serviceName, + start, + end, + transactionName, + transactionType, + ] + ); + + const { data: correlationsData, status: correlationsStatus } = useFetcher( (callApmApi) => { if (start && end && hasFieldNames) { return callApmApi({ - endpoint: 'GET /api/apm/correlations/failed_transactions', + endpoint: 'GET /api/apm/correlations/errors/failed_transactions', params: { query: { environment, @@ -125,8 +159,9 @@ export function ErrorCorrelations({ onClose }: Props) { @@ -136,8 +171,12 @@ export function ErrorCorrelations({ onClose }: Props) { 'xpack.apm.correlations.error.percentageColumnName', { defaultMessage: '% of failed transactions' } )} - significantTerms={hasFieldNames ? data?.significantTerms : []} - status={status} + significantTerms={ + hasFieldNames && correlationsData?.significantTerms + ? correlationsData.significantTerms + : [] + } + status={correlationsStatus} setSelectedSignificantTerm={setSelectedSignificantTerm} onFilter={onClose} /> @@ -151,10 +190,9 @@ export function ErrorCorrelations({ onClose }: Props) { } function getSelectedTimeseries( - data: CorrelationsApiResponse, + significantTerms: CorrelationsApiResponse['significantTerms'], selectedSignificantTerm: SelectedSignificantTerm ) { - const { significantTerms } = data; if (!significantTerms) { return []; } @@ -168,11 +206,13 @@ function getSelectedTimeseries( } function ErrorTimeseriesChart({ - data, + overallData, + correlationsData, selectedSignificantTerm, status, }: { - data?: CorrelationsApiResponse; + overallData?: OverallErrorsApiResponse; + correlationsData?: CorrelationsApiResponse; selectedSignificantTerm: SelectedSignificantTerm | null; status: FETCH_STATUS; }) { @@ -180,7 +220,7 @@ function ErrorTimeseriesChart({ const dateFormatter = timeFormatter('HH:mm:ss'); return ( - + @@ -206,11 +246,11 @@ function ErrorTimeseriesChart({ yScaleType={ScaleType.Linear} xAccessor={'x'} yAccessors={['y']} - data={data?.overall?.timeseries ?? []} + data={overallData?.overall?.timeseries ?? []} curve={CurveType.CURVE_MONOTONE_X} /> - {data && selectedSignificantTerm ? ( + {correlationsData && selectedSignificantTerm ? ( ) : null} diff --git a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx index 77571421ed00e4..e65bad8088c170 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/latency_correlations.tsx @@ -32,8 +32,12 @@ import { useFieldNames } from './use_field_names'; import { useLocalStorage } from '../../../hooks/useLocalStorage'; import { useUiTracker } from '../../../../../observability/public'; +type OverallLatencyApiResponse = NonNullable< + APIReturnType<'GET /api/apm/correlations/latency/overall_distribution'> +>; + type CorrelationsApiResponse = NonNullable< - APIReturnType<'GET /api/apm/correlations/slow_transactions'> + APIReturnType<'GET /api/apm/correlations/latency/slow_transactions'> >; interface Props { @@ -71,11 +75,45 @@ export function LatencyCorrelations({ onClose }: Props) { 75 ); - const { data, status } = useFetcher( + const { data: overallData, status: overallStatus } = useFetcher( (callApmApi) => { - if (start && end && hasFieldNames) { + if (start && end) { return callApmApi({ - endpoint: 'GET /api/apm/correlations/slow_transactions', + endpoint: 'GET /api/apm/correlations/latency/overall_distribution', + params: { + query: { + environment, + kuery, + serviceName, + transactionName, + transactionType, + start, + end, + }, + }, + }); + } + }, + [ + environment, + kuery, + serviceName, + start, + end, + transactionName, + transactionType, + ] + ); + + const maxLatency = overallData?.maxLatency; + const distributionInterval = overallData?.distributionInterval; + const fieldNamesCommaSeparated = fieldNames.join(','); + + const { data: correlationsData, status: correlationsStatus } = useFetcher( + (callApmApi) => { + if (start && end && hasFieldNames && maxLatency && distributionInterval) { + return callApmApi({ + endpoint: 'GET /api/apm/correlations/latency/slow_transactions', params: { query: { environment, @@ -86,7 +124,9 @@ export function LatencyCorrelations({ onClose }: Props) { start, end, durationPercentile: durationPercentile.toString(10), - fieldNames: fieldNames.join(','), + fieldNames: fieldNamesCommaSeparated, + maxLatency: maxLatency.toString(10), + distributionInterval: distributionInterval.toString(10), }, }, }); @@ -101,8 +141,10 @@ export function LatencyCorrelations({ onClose }: Props) { transactionName, transactionType, durationPercentile, - fieldNames, + fieldNamesCommaSeparated, hasFieldNames, + maxLatency, + distributionInterval, ] ); @@ -134,8 +176,13 @@ export function LatencyCorrelations({ onClose }: Props) { @@ -147,8 +194,12 @@ export function LatencyCorrelations({ onClose }: Props) { 'xpack.apm.correlations.latency.percentageColumnName', { defaultMessage: '% of slow transactions' } )} - significantTerms={hasFieldNames ? data?.significantTerms : []} - status={status} + significantTerms={ + hasFieldNames && correlationsData + ? correlationsData?.significantTerms + : [] + } + status={correlationsStatus} setSelectedSignificantTerm={setSelectedSignificantTerm} onFilter={onClose} /> @@ -167,25 +218,23 @@ export function LatencyCorrelations({ onClose }: Props) { ); } -function getDistributionYMax(data?: CorrelationsApiResponse) { - if (!data?.overall) { - return 0; +function getAxisMaxes(data?: OverallLatencyApiResponse) { + if (!data?.overallDistribution) { + return { xMax: 0, yMax: 0 }; } - - const yValues = [ - ...data.overall.distribution.map((p) => p.y ?? 0), - ...data.significantTerms.flatMap((term) => - term.distribution.map((p) => p.y ?? 0) - ), - ]; - return Math.max(...yValues); + const { overallDistribution } = data; + const xValues = overallDistribution.map((p) => p.x ?? 0); + const yValues = overallDistribution.map((p) => p.y ?? 0); + return { + xMax: Math.max(...xValues), + yMax: Math.max(...yValues), + }; } function getSelectedDistribution( - data: CorrelationsApiResponse, + significantTerms: CorrelationsApiResponse['significantTerms'], selectedSignificantTerm: SelectedSignificantTerm ) { - const { significantTerms } = data; if (!significantTerms) { return []; } @@ -199,23 +248,22 @@ function getSelectedDistribution( } function LatencyDistributionChart({ - data, + overallData, + correlationsData, selectedSignificantTerm, status, }: { - data?: CorrelationsApiResponse; + overallData?: OverallLatencyApiResponse; + correlationsData?: CorrelationsApiResponse['significantTerms']; selectedSignificantTerm: SelectedSignificantTerm | null; status: FETCH_STATUS; }) { const theme = useTheme(); - const xMax = Math.max( - ...(data?.overall?.distribution.map((p) => p.x ?? 0) ?? []) - ); + const { xMax, yMax } = getAxisMaxes(overallData); const durationFormatter = getDurationFormatter(xMax); - const yMax = getDistributionYMax(data); return ( - + { const start = durationFormatter(obj.value); const end = durationFormatter( - obj.value + data?.distributionInterval + obj.value + overallData?.distributionInterval ); return `${start.value} - ${end.formatted}`; @@ -254,12 +302,12 @@ function LatencyDistributionChart({ xAccessor={'x'} yAccessors={['y']} color={theme.eui.euiColorVis1} - data={data?.overall?.distribution || []} + data={overallData?.overallDistribution || []} minBarHeight={5} tickFormat={(d) => `${roundFloat(d)}%`} /> - {data && selectedSignificantTerm ? ( + {correlationsData && selectedSignificantTerm ? ( `${roundFloat(d)}%`} /> diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts b/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts similarity index 63% rename from x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts rename to x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts index c668f3bb287138..8ee469c9a93c77 100644 --- a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_failed_transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { isEmpty, omit, merge } from 'lodash'; +import { isEmpty, omit } from 'lodash'; import { EventOutcome } from '../../../../common/event_outcome'; import { processSignificantTermAggs, @@ -13,65 +13,25 @@ import { } from '../process_significant_term_aggs'; import { AggregationOptionsByType } from '../../../../../../../typings/elasticsearch'; import { ESFilter } from '../../../../../../../typings/elasticsearch'; -import { - environmentQuery, - rangeQuery, - kqlQuery, -} from '../../../../server/utils/queries'; -import { - EVENT_OUTCOME, - SERVICE_NAME, - TRANSACTION_NAME, - TRANSACTION_TYPE, - PROCESSOR_EVENT, -} from '../../../../common/elasticsearch_fieldnames'; +import { EVENT_OUTCOME } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { - getOutcomeAggregation, + getTimeseriesAggregation, getTransactionErrorRateTimeSeries, } from '../../helpers/transaction_error_rate'; import { withApmSpan } from '../../../utils/with_apm_span'; +import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; -export async function getCorrelationsForFailedTransactions({ - environment, - kuery, - serviceName, - transactionType, - transactionName, - fieldNames, - setup, -}: { - environment?: string; - kuery?: string; - serviceName: string | undefined; - transactionType: string | undefined; - transactionName: string | undefined; +interface Options extends CorrelationsOptions { fieldNames: string[]; - setup: Setup & SetupTimeRange; -}) { +} +export async function getCorrelationsForFailedTransactions(options: Options) { return withApmSpan('get_correlations_for_failed_transactions', async () => { - const { start, end, apmEventClient } = setup; - - const backgroundFilters: ESFilter[] = [ - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; - - if (serviceName) { - backgroundFilters.push({ term: { [SERVICE_NAME]: serviceName } }); - } - - if (transactionType) { - backgroundFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); - } - - if (transactionName) { - backgroundFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); - } + const { fieldNames, setup } = options; + const { apmEventClient } = setup; + const filters = getCorrelationsFilters(options); const params = { apm: { events: [ProcessorEvent.transaction] }, @@ -79,7 +39,7 @@ export async function getCorrelationsForFailedTransactions({ body: { size: 0, query: { - bool: { filter: backgroundFilters }, + bool: { filter: filters }, }, aggs: { failed_transactions: { @@ -95,7 +55,7 @@ export async function getCorrelationsForFailedTransactions({ field: fieldName, background_filter: { bool: { - filter: backgroundFilters, + filter: filters, must_not: { term: { [EVENT_OUTCOME]: EventOutcome.failure }, }, @@ -112,7 +72,7 @@ export async function getCorrelationsForFailedTransactions({ const response = await apmEventClient.search(params); if (!response.aggregations) { - return {}; + return { significantTerms: [] }; } const sigTermAggs = omit( @@ -121,17 +81,17 @@ export async function getCorrelationsForFailedTransactions({ ); const topSigTerms = processSignificantTermAggs({ sigTermAggs }); - return getErrorRateTimeSeries({ setup, backgroundFilters, topSigTerms }); + return getErrorRateTimeSeries({ setup, filters, topSigTerms }); }); } export async function getErrorRateTimeSeries({ setup, - backgroundFilters, + filters, topSigTerms, }: { setup: Setup & SetupTimeRange; - backgroundFilters: ESFilter[]; + filters: ESFilter[]; topSigTerms: TopSigTerm[]; }) { return withApmSpan('get_error_rate_timeseries', async () => { @@ -139,20 +99,10 @@ export async function getErrorRateTimeSeries({ const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); if (isEmpty(topSigTerms)) { - return {}; + return { significantTerms: [] }; } - const timeseriesAgg = { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { min: start, max: end }, - }, - aggs: { - outcomes: getOutcomeAggregation(), - }, - }; + const timeseriesAgg = getTimeseriesAggregation(start, end, intervalString); const perTermAggs = topSigTerms.reduce( (acc, term, index) => { @@ -175,8 +125,8 @@ export async function getErrorRateTimeSeries({ apm: { events: [ProcessorEvent.transaction] }, body: { size: 0, - query: { bool: { filter: backgroundFilters } }, - aggs: merge({ timeseries: timeseriesAgg }, perTermAggs), + query: { bool: { filter: filters } }, + aggs: perTermAggs, }, }; @@ -184,15 +134,10 @@ export async function getErrorRateTimeSeries({ const { aggregations } = response; if (!aggregations) { - return {}; + return { significantTerms: [] }; } return { - overall: { - timeseries: getTransactionErrorRateTimeSeries( - aggregations.timeseries.buckets - ), - }, significantTerms: topSigTerms.map((topSig, index) => { const agg = aggregations[`term_${index}`]!; diff --git a/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts b/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts new file mode 100644 index 00000000000000..9387e64a51e01c --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ProcessorEvent } from '../../../../common/processor_event'; +import { getBucketSize } from '../../helpers/get_bucket_size'; +import { + getTimeseriesAggregation, + getTransactionErrorRateTimeSeries, +} from '../../helpers/transaction_error_rate'; +import { withApmSpan } from '../../../utils/with_apm_span'; +import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; + +export async function getOverallErrorTimeseries(options: CorrelationsOptions) { + return withApmSpan('get_error_rate_timeseries', async () => { + const { setup } = options; + const filters = getCorrelationsFilters(options); + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: { + timeseries: getTimeseriesAggregation(start, end, intervalString), + }, + }, + }; + + const response = await apmEventClient.search(params); + const { aggregations } = response; + + if (!aggregations) { + return { overall: null }; + } + + return { + overall: { + timeseries: getTransactionErrorRateTimeSeries( + aggregations.timeseries.buckets + ), + }, + }; + }); +} diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_latency_distribution.ts b/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_latency_distribution.ts deleted file mode 100644 index 88b1cf3a344ed6..00000000000000 --- a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_latency_distribution.ts +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isEmpty, dropRightWhile } from 'lodash'; -import { AggregationOptionsByType } from '../../../../../../../typings/elasticsearch'; -import { ESFilter } from '../../../../../../../typings/elasticsearch'; -import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import { ProcessorEvent } from '../../../../common/processor_event'; -import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { TopSigTerm } from '../process_significant_term_aggs'; -import { getMaxLatency } from './get_max_latency'; -import { withApmSpan } from '../../../utils/with_apm_span'; - -export async function getLatencyDistribution({ - setup, - backgroundFilters, - topSigTerms, -}: { - setup: Setup & SetupTimeRange; - backgroundFilters: ESFilter[]; - topSigTerms: TopSigTerm[]; -}) { - return withApmSpan('get_latency_distribution', async () => { - const { apmEventClient } = setup; - - if (isEmpty(topSigTerms)) { - return {}; - } - - const maxLatency = await getMaxLatency({ - setup, - backgroundFilters, - topSigTerms, - }); - - if (!maxLatency) { - return {}; - } - - const intervalBuckets = 15; - const distributionInterval = Math.floor(maxLatency / intervalBuckets); - - const distributionAgg = { - // filter out outliers not included in the significant term docs - filter: { range: { [TRANSACTION_DURATION]: { lte: maxLatency } } }, - aggs: { - dist_filtered_by_latency: { - histogram: { - // TODO: add support for metrics - field: TRANSACTION_DURATION, - interval: distributionInterval, - min_doc_count: 0, - extended_bounds: { - min: 0, - max: maxLatency, - }, - }, - }, - }, - }; - - const perTermAggs = topSigTerms.reduce( - (acc, term, index) => { - acc[`term_${index}`] = { - filter: { term: { [term.fieldName]: term.fieldValue } }, - aggs: { - distribution: distributionAgg, - }, - }; - return acc; - }, - {} as Record< - string, - { - filter: AggregationOptionsByType['filter']; - aggs: { - distribution: typeof distributionAgg; - }; - } - > - ); - - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { bool: { filter: backgroundFilters } }, - aggs: { - // overall aggs - distribution: distributionAgg, - - // per term aggs - ...perTermAggs, - }, - }, - }; - - const response = await withApmSpan('get_terms_distribution', () => - apmEventClient.search(params) - ); - type Agg = NonNullable; - - if (!response.aggregations) { - return {}; - } - - function formatDistribution(distribution: Agg['distribution']) { - const total = distribution.doc_count; - - // remove trailing buckets that are empty and out of bounds of the desired number of buckets - const buckets = dropRightWhile( - distribution.dist_filtered_by_latency.buckets, - (bucket, index) => bucket.doc_count === 0 && index > intervalBuckets - 1 - ); - - return buckets.map((bucket) => ({ - x: bucket.key, - y: (bucket.doc_count / total) * 100, - })); - } - - return { - distributionInterval, - overall: { - distribution: formatDistribution(response.aggregations.distribution), - }, - significantTerms: topSigTerms.map((topSig, index) => { - // @ts-expect-error - const agg = response.aggregations[`term_${index}`] as Agg; - - return { - ...topSig, - distribution: formatDistribution(agg.distribution), - }; - }), - }; - }); -} diff --git a/x-pack/plugins/apm/server/lib/correlations/get_filters.ts b/x-pack/plugins/apm/server/lib/correlations/get_filters.ts new file mode 100644 index 00000000000000..92fc9c5d9622b2 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/get_filters.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Setup, SetupTimeRange } from '../helpers/setup_request'; +import { ESFilter } from '../../../../../../typings/elasticsearch'; +import { environmentQuery, rangeQuery, kqlQuery } from '../../utils/queries'; +import { + SERVICE_NAME, + TRANSACTION_NAME, + TRANSACTION_TYPE, + PROCESSOR_EVENT, +} from '../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../common/processor_event'; + +export interface CorrelationsOptions { + setup: Setup & SetupTimeRange; + environment?: string; + kuery?: string; + serviceName: string | undefined; + transactionType: string | undefined; + transactionName: string | undefined; +} + +export function getCorrelationsFilters({ + setup, + environment, + kuery, + serviceName, + transactionType, + transactionName, +}: CorrelationsOptions) { + const { start, end } = setup; + const correlationsFilters: ESFilter[] = [ + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ]; + + if (serviceName) { + correlationsFilters.push({ term: { [SERVICE_NAME]: serviceName } }); + } + + if (transactionType) { + correlationsFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); + } + + if (transactionName) { + correlationsFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + return correlationsFilters; +} diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/index.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts similarity index 63% rename from x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/index.ts rename to x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts index 9472d385a26c60..0f93d1411a001c 100644 --- a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts @@ -6,75 +6,39 @@ */ import { AggregationOptionsByType } from '../../../../../../../typings/elasticsearch'; -import { ESFilter } from '../../../../../../../typings/elasticsearch'; -import { - environmentQuery, - rangeQuery, - kqlQuery, -} from '../../../../server/utils/queries'; -import { - SERVICE_NAME, - TRANSACTION_DURATION, - TRANSACTION_NAME, - TRANSACTION_TYPE, - PROCESSOR_EVENT, -} from '../../../../common/elasticsearch_fieldnames'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getDurationForPercentile } from './get_duration_for_percentile'; import { processSignificantTermAggs } from '../process_significant_term_aggs'; import { getLatencyDistribution } from './get_latency_distribution'; import { withApmSpan } from '../../../utils/with_apm_span'; +import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; -export async function getCorrelationsForSlowTransactions({ - environment, - kuery, - serviceName, - transactionType, - transactionName, - durationPercentile, - fieldNames, - setup, -}: { - environment?: string; - kuery?: string; - serviceName: string | undefined; - transactionType: string | undefined; - transactionName: string | undefined; +interface Options extends CorrelationsOptions { durationPercentile: number; fieldNames: string[]; - setup: Setup & SetupTimeRange; -}) { + maxLatency: number; + distributionInterval: number; +} +export async function getCorrelationsForSlowTransactions(options: Options) { return withApmSpan('get_correlations_for_slow_transactions', async () => { - const { start, end, apmEventClient } = setup; - - const backgroundFilters: ESFilter[] = [ - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; - - if (serviceName) { - backgroundFilters.push({ term: { [SERVICE_NAME]: serviceName } }); - } - - if (transactionType) { - backgroundFilters.push({ term: { [TRANSACTION_TYPE]: transactionType } }); - } - - if (transactionName) { - backgroundFilters.push({ term: { [TRANSACTION_NAME]: transactionName } }); - } - + const { + durationPercentile, + fieldNames, + setup, + maxLatency, + distributionInterval, + } = options; + const { apmEventClient } = setup; + const filters = getCorrelationsFilters(options); const durationForPercentile = await getDurationForPercentile({ durationPercentile, - backgroundFilters, + filters, setup, }); if (!durationForPercentile) { - return {}; + return { significantTerms: [] }; } const response = await withApmSpan('get_significant_terms', () => { @@ -85,7 +49,7 @@ export async function getCorrelationsForSlowTransactions({ query: { bool: { // foreground filters - filter: backgroundFilters, + filter: filters, must: { function_score: { query: { @@ -112,7 +76,7 @@ export async function getCorrelationsForSlowTransactions({ background_filter: { bool: { filter: [ - ...backgroundFilters, + ...filters, { range: { [TRANSACTION_DURATION]: { @@ -132,17 +96,21 @@ export async function getCorrelationsForSlowTransactions({ return apmEventClient.search(params); }); if (!response.aggregations) { - return {}; + return { significantTerms: [] }; } const topSigTerms = processSignificantTermAggs({ sigTermAggs: response.aggregations, }); - return getLatencyDistribution({ + const significantTerms = await getLatencyDistribution({ setup, - backgroundFilters, + filters, topSigTerms, + maxLatency, + distributionInterval, }); + + return { significantTerms }; }); } diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_duration_for_percentile.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts similarity index 86% rename from x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_duration_for_percentile.ts rename to x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts index 02141f5f9e76f6..43c261743861d8 100644 --- a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_duration_for_percentile.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts @@ -13,11 +13,11 @@ import { Setup, SetupTimeRange } from '../../helpers/setup_request'; export async function getDurationForPercentile({ durationPercentile, - backgroundFilters, + filters, setup, }: { durationPercentile: number; - backgroundFilters: ESFilter[]; + filters: ESFilter[]; setup: Setup & SetupTimeRange; }) { return withApmSpan('get_duration_for_percentiles', async () => { @@ -29,7 +29,7 @@ export async function getDurationForPercentile({ body: { size: 0, query: { - bool: { filter: backgroundFilters }, + bool: { filter: filters }, }, aggs: { percentile: { @@ -42,6 +42,9 @@ export async function getDurationForPercentile({ }, }); - return Object.values(res.aggregations?.percentile.values || {})[0]; + const duration = Object.values( + res.aggregations?.percentile.values || {} + )[0]; + return duration || 0; }); } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts new file mode 100644 index 00000000000000..6d42b26b22e42d --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AggregationOptionsByType } from '../../../../../../../typings/elasticsearch'; +import { ESFilter } from '../../../../../../../typings/elasticsearch'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { TopSigTerm } from '../process_significant_term_aggs'; +import { withApmSpan } from '../../../utils/with_apm_span'; +import { + getDistributionAggregation, + trimBuckets, +} from './get_overall_latency_distribution'; + +export async function getLatencyDistribution({ + setup, + filters, + topSigTerms, + maxLatency, + distributionInterval, +}: { + setup: Setup & SetupTimeRange; + filters: ESFilter[]; + topSigTerms: TopSigTerm[]; + maxLatency: number; + distributionInterval: number; +}) { + return withApmSpan('get_latency_distribution', async () => { + const { apmEventClient } = setup; + + const distributionAgg = getDistributionAggregation( + maxLatency, + distributionInterval + ); + + const perTermAggs = topSigTerms.reduce( + (acc, term, index) => { + acc[`term_${index}`] = { + filter: { term: { [term.fieldName]: term.fieldValue } }, + aggs: { + distribution: distributionAgg, + }, + }; + return acc; + }, + {} as Record< + string, + { + filter: AggregationOptionsByType['filter']; + aggs: { + distribution: typeof distributionAgg; + }; + } + > + ); + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: perTermAggs, + }, + }; + + const response = await withApmSpan('get_terms_distribution', () => + apmEventClient.search(params) + ); + type Agg = NonNullable; + + if (!response.aggregations) { + return []; + } + + return topSigTerms.map((topSig, index) => { + // ignore the typescript error since existence of response.aggregations is already checked: + // @ts-expect-error + const agg = response.aggregations[`term_${index}`] as Agg[string]; + const total = agg.distribution.doc_count; + const buckets = trimBuckets( + agg.distribution.dist_filtered_by_latency.buckets + ); + + return { + ...topSig, + distribution: buckets.map((bucket) => ({ + x: bucket.key, + y: (bucket.doc_count / total) * 100, + })), + }; + }); + }); +} diff --git a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_max_latency.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts similarity index 76% rename from x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_max_latency.ts rename to x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts index 5f12c86a9c70c5..8b415bf0d80a7b 100644 --- a/x-pack/plugins/apm/server/lib/correlations/get_correlations_for_slow_transactions/get_max_latency.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts @@ -14,12 +14,12 @@ import { TopSigTerm } from '../process_significant_term_aggs'; export async function getMaxLatency({ setup, - backgroundFilters, - topSigTerms, + filters, + topSigTerms = [], }: { setup: Setup & SetupTimeRange; - backgroundFilters: ESFilter[]; - topSigTerms: TopSigTerm[]; + filters: ESFilter[]; + topSigTerms?: TopSigTerm[]; }) { return withApmSpan('get_max_latency', async () => { const { apmEventClient } = setup; @@ -31,13 +31,17 @@ export async function getMaxLatency({ size: 0, query: { bool: { - filter: backgroundFilters, + filter: filters, - // only include docs containing the significant terms - should: topSigTerms.map((term) => ({ - term: { [term.fieldName]: term.fieldValue }, - })), - minimum_should_match: 1, + ...(topSigTerms.length + ? { + // only include docs containing the significant terms + should: topSigTerms.map((term) => ({ + term: { [term.fieldName]: term.fieldValue }, + })), + minimum_should_match: 1, + } + : null), }, }, aggs: { diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts new file mode 100644 index 00000000000000..c5d4def51ea54a --- /dev/null +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { dropRightWhile } from 'lodash'; +import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { getMaxLatency } from './get_max_latency'; +import { withApmSpan } from '../../../utils/with_apm_span'; +import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; + +export const INTERVAL_BUCKETS = 15; + +export function getDistributionAggregation( + maxLatency: number, + distributionInterval: number +) { + return { + filter: { range: { [TRANSACTION_DURATION]: { lte: maxLatency } } }, + aggs: { + dist_filtered_by_latency: { + histogram: { + // TODO: add support for metrics + field: TRANSACTION_DURATION, + interval: distributionInterval, + min_doc_count: 0, + extended_bounds: { + min: 0, + max: maxLatency, + }, + }, + }, + }, + }; +} + +export async function getOverallLatencyDistribution( + options: CorrelationsOptions +) { + const { setup } = options; + const filters = getCorrelationsFilters(options); + + return withApmSpan('get_overall_latency_distribution', async () => { + const { apmEventClient } = setup; + const maxLatency = await getMaxLatency({ setup, filters }); + if (!maxLatency) { + return { + maxLatency: null, + distributionInterval: null, + overallDistribution: null, + }; + } + const distributionInterval = Math.floor(maxLatency / INTERVAL_BUCKETS); + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: { + // overall distribution agg + distribution: getDistributionAggregation( + maxLatency, + distributionInterval + ), + }, + }, + }; + + const response = await withApmSpan('get_terms_distribution', () => + apmEventClient.search(params) + ); + + if (!response.aggregations) { + return { + maxLatency, + distributionInterval, + overallDistribution: null, + }; + } + + const { distribution } = response.aggregations; + const total = distribution.doc_count; + const buckets = trimBuckets(distribution.dist_filtered_by_latency.buckets); + + return { + maxLatency, + distributionInterval, + overallDistribution: buckets.map((bucket) => ({ + x: bucket.key, + y: (bucket.doc_count / total) * 100, + })), + }; + }); +} + +// remove trailing buckets that are empty and out of bounds of the desired number of buckets +export function trimBuckets(buckets: T[]) { + return dropRightWhile( + buckets, + (bucket, index) => bucket.doc_count === 0 && index > INTERVAL_BUCKETS - 1 + ); +} diff --git a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts index 11d65b7697e9a8..b60a2a071e6dcb 100644 --- a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts @@ -21,6 +21,20 @@ export const getOutcomeAggregation = () => ({ type OutcomeAggregation = ReturnType; +export const getTimeseriesAggregation = ( + start: number, + end: number, + intervalString: string +) => ({ + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + aggs: { outcomes: getOutcomeAggregation() }, +}); + export function calculateTransactionErrorPercentage( outcomeResponse: AggregationResultOf ) { diff --git a/x-pack/plugins/apm/server/routes/correlations.ts b/x-pack/plugins/apm/server/routes/correlations.ts index 48305d1a9df07e..c7c69e07748229 100644 --- a/x-pack/plugins/apm/server/routes/correlations.ts +++ b/x-pack/plugins/apm/server/routes/correlations.ts @@ -9,8 +9,10 @@ import Boom from '@hapi/boom'; import { i18n } from '@kbn/i18n'; import * as t from 'io-ts'; import { isActivePlatinumLicense } from '../../common/license_check'; -import { getCorrelationsForFailedTransactions } from '../lib/correlations/get_correlations_for_failed_transactions'; -import { getCorrelationsForSlowTransactions } from '../lib/correlations/get_correlations_for_slow_transactions'; +import { getCorrelationsForFailedTransactions } from '../lib/correlations/errors/get_correlations_for_failed_transactions'; +import { getOverallErrorTimeseries } from '../lib/correlations/errors/get_overall_error_timeseries'; +import { getCorrelationsForSlowTransactions } from '../lib/correlations/latency/get_correlations_for_slow_transactions'; +import { getOverallLatencyDistribution } from '../lib/correlations/latency/get_overall_latency_distribution'; import { setupRequest } from '../lib/helpers/setup_request'; import { createRoute } from './create_route'; import { environmentRt, kueryRt, rangeRt } from './default_api_types'; @@ -23,8 +25,47 @@ const INVALID_LICENSE = i18n.translate( } ); +export const correlationsLatencyDistributionRoute = createRoute({ + endpoint: 'GET /api/apm/correlations/latency/overall_distribution', + params: t.type({ + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + ]), + }), + options: { tags: ['access:apm'] }, + handler: async ({ context, request }) => { + if (!isActivePlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(INVALID_LICENSE); + } + const setup = await setupRequest(context, request); + const { + environment, + kuery, + serviceName, + transactionType, + transactionName, + } = context.params.query; + + return getOverallLatencyDistribution({ + environment, + kuery, + serviceName, + transactionType, + transactionName, + setup, + }); + }, +}); + export const correlationsForSlowTransactionsRoute = createRoute({ - endpoint: 'GET /api/apm/correlations/slow_transactions', + endpoint: 'GET /api/apm/correlations/latency/slow_transactions', params: t.type({ query: t.intersection([ t.partial({ @@ -35,6 +76,8 @@ export const correlationsForSlowTransactionsRoute = createRoute({ t.type({ durationPercentile: t.string, fieldNames: t.string, + maxLatency: t.string, + distributionInterval: t.string, }), environmentRt, kueryRt, @@ -55,6 +98,8 @@ export const correlationsForSlowTransactionsRoute = createRoute({ transactionName, durationPercentile, fieldNames, + maxLatency, + distributionInterval, } = context.params.query; return getCorrelationsForSlowTransactions({ @@ -66,12 +111,53 @@ export const correlationsForSlowTransactionsRoute = createRoute({ durationPercentile: parseInt(durationPercentile, 10), fieldNames: fieldNames.split(','), setup, + maxLatency: parseInt(maxLatency, 10), + distributionInterval: parseInt(distributionInterval, 10), + }); + }, +}); + +export const correlationsErrorDistributionRoute = createRoute({ + endpoint: 'GET /api/apm/correlations/errors/overall_timeseries', + params: t.type({ + query: t.intersection([ + t.partial({ + serviceName: t.string, + transactionName: t.string, + transactionType: t.string, + }), + environmentRt, + kueryRt, + rangeRt, + ]), + }), + options: { tags: ['access:apm'] }, + handler: async ({ context, request }) => { + if (!isActivePlatinumLicense(context.licensing.license)) { + throw Boom.forbidden(INVALID_LICENSE); + } + const setup = await setupRequest(context, request); + const { + environment, + kuery, + serviceName, + transactionType, + transactionName, + } = context.params.query; + + return getOverallErrorTimeseries({ + environment, + kuery, + serviceName, + transactionType, + transactionName, + setup, }); }, }); export const correlationsForFailedTransactionsRoute = createRoute({ - endpoint: 'GET /api/apm/correlations/failed_transactions', + endpoint: 'GET /api/apm/correlations/errors/failed_transactions', params: t.type({ query: t.intersection([ t.partial({ diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index 2b5fb0b516ab5f..5b74aa4347f141 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -58,7 +58,9 @@ import { rootTransactionByTraceIdRoute, } from './traces'; import { + correlationsLatencyDistributionRoute, correlationsForSlowTransactionsRoute, + correlationsErrorDistributionRoute, correlationsForFailedTransactionsRoute, } from './correlations'; import { @@ -152,7 +154,9 @@ const createApmApi = () => { .add(createOrUpdateAgentConfigurationRoute) // Correlations + .add(correlationsLatencyDistributionRoute) .add(correlationsForSlowTransactionsRoute) + .add(correlationsErrorDistributionRoute) .add(correlationsForFailedTransactionsRoute) // APM indices diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/analytics.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/analytics.tsx index 82b0d9a318f1d2..0eef9b0c688c04 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/analytics.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/analytics.tsx @@ -83,8 +83,7 @@ export const Analytics: React.FC = () => { /> - {/* TODO: Update this panel to use the bordered version once available */} - + { {RECENT_QUERIES}} subtitle={i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.analytics.recentQueriesDescription', diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx index f00c4e29a7190f..83c83aa36f1bbf 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/analytics/views/query_detail.tsx @@ -9,7 +9,7 @@ import React from 'react'; import { useValues } from 'kea'; -import { EuiSpacer } from '@elastic/eui'; +import { EuiPanel, EuiSpacer } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { SetAppSearchChrome as SetPageChrome } from '../../../../shared/kibana_chrome'; @@ -56,17 +56,19 @@ export const QueryDetail: React.FC = ({ breadcrumbs }) => { /> - + + + { {shouldShowCredentialsForm && } - +

{i18n.translate('xpack.enterpriseSearch.appSearch.credentials.apiEndpoint', { @@ -116,7 +116,9 @@ export const Credentials: React.FC = () => { - {!!dataLoading ? : } + + {!!dataLoading ? : } + ); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx index 0d6ebfe4379277..7e40eb63338bbf 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx @@ -104,7 +104,7 @@ export const EngineSelection: React.FC = () => { return ( <> - +

{i18n.translate( diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx index 0b631089c39847..f363f6978db290 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx @@ -22,7 +22,7 @@ export const FormKeyReadWriteAccess: React.FC = () => { return ( <> - +

{i18n.translate('xpack.enterpriseSearch.appSearch.credentials.formReadWrite.label', { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_creation.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_creation.tsx index b1bfc6c2ab7fae..10f1fc093e60f6 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_creation.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curation_creation.tsx @@ -25,7 +25,7 @@ export const CurationCreation: React.FC = () => { <> - +

{i18n.translate( diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx index a523a683c4f5b4..624790c8471679 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/curations/views/curations.tsx @@ -54,7 +54,7 @@ export const Curations: React.FC = () => { , ]} /> - + diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.test.tsx index c111383816e36f..8034b72d885dab 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.test.tsx @@ -94,6 +94,14 @@ describe('DataPanel', () => { expect(wrapper.find(LoadingOverlay)).toHaveLength(1); }); + it('passes hasBorder', () => { + const wrapper = shallow(Test

} />); + expect(wrapper.prop('hasBorder')).toBeFalsy(); + + wrapper.setProps({ hasBorder: true }); + expect(wrapper.prop('hasBorder')).toBeTruthy(); + }); + it('passes class names', () => { const wrapper = shallow(Test

} className="testing" />); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx index 825311fa1652a0..ce878dc3cf29a9 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/data_panel/data_panel.tsx @@ -29,6 +29,7 @@ interface Props { iconType?: string; action?: React.ReactNode; filled?: boolean; + hasBorder?: boolean; isLoading?: boolean; className?: string; } @@ -39,6 +40,7 @@ export const DataPanel: React.FC = ({ iconType, action, filled, + hasBorder, isLoading, className, children, @@ -52,6 +54,7 @@ export const DataPanel: React.FC = ({ {

- + POST diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx index dd55c26b5b298f..fefe983df33420 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/documents/document_detail.tsx @@ -91,7 +91,7 @@ export const DocumentDetail: React.FC = ({ engineBreadcrumb }) => { , ]} /> - + diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_creation/engine_creation.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_creation/engine_creation.tsx index bab31d0fccc40b..4e1d7bc3e8e482 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_creation/engine_creation.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_creation/engine_creation.tsx @@ -16,12 +16,11 @@ import { EuiFlexItem, EuiFieldText, EuiSelect, - EuiPageBody, EuiPageHeader, + EuiPageContent, EuiSpacer, EuiTitle, EuiButton, - EuiPanel, } from '@elastic/eui'; import { FlashMessages } from '../../../shared/flash_messages'; @@ -48,75 +47,73 @@ export const EngineCreation: React.FC = () => {
- - - - -
{ - e.preventDefault(); - submitEngine(); - }} - > - -

{ENGINE_CREATION_FORM_TITLE}

-
- - - - 0 && rawName !== name ? ( - <> - {SANITIZED_NAME_NOTE} {name} - - ) : ( - ALLOWED_CHARS_NOTE - ) - } + + + + { + e.preventDefault(); + submitEngine(); + }} + > + +

{ENGINE_CREATION_FORM_TITLE}

+
+ + + + 0 && rawName !== name ? ( + <> + {SANITIZED_NAME_NOTE} {name} + + ) : ( + ALLOWED_CHARS_NOTE + ) + } + fullWidth + > + setRawName(event.currentTarget.value)} + autoComplete="off" fullWidth - > - setRawName(event.currentTarget.value)} - autoComplete="off" - fullWidth - data-test-subj="EngineCreationNameInput" - placeholder={ENGINE_CREATION_FORM_ENGINE_NAME_PLACEHOLDER} - autoFocus - /> - - - - - setLanguage(event.currentTarget.value)} - /> - - - - - - {ENGINE_CREATION_FORM_SUBMIT_BUTTON_LABEL} - - -
-
-
+ data-test-subj="EngineCreationNameInput" + placeholder={ENGINE_CREATION_FORM_ENGINE_NAME_PLACEHOLDER} + autoFocus + /> + + + + + setLanguage(event.currentTarget.value)} + /> + + + + + + {ENGINE_CREATION_FORM_SUBMIT_BUTTON_LABEL} + + + +
); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/recent_api_logs.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/recent_api_logs.tsx index e77a4ad7b03487..1a8e4703d7c2ef 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/recent_api_logs.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/recent_api_logs.tsx @@ -34,6 +34,7 @@ export const RecentApiLogs: React.FC = () => { {VIEW_API_LOGS} } + hasBorder > TODO: API Logs Table {/* */} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/total_charts.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/total_charts.tsx index 77ba9ad0f9514d..136c9c6603e5cc 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/total_charts.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine_overview/components/total_charts.tsx @@ -40,6 +40,7 @@ export const TotalCharts: React.FC = () => { {VIEW_ANALYTICS} } + hasBorder > { {VIEW_API_LOGS} } + hasBorder > { <> - + {canManageEngines ? ( { <> - + diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_overview.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_overview.tsx index baf275fbe6c2ce..a09f30035bafcb 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_overview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/engines_overview.tsx @@ -84,7 +84,7 @@ export const EnginesOverview: React.FC = () => { - + diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/error_connecting/error_connecting.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/error_connecting/error_connecting.tsx index d7fde0cd5dd25f..b193e00c1d48dd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/error_connecting/error_connecting.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/error_connecting/error_connecting.tsx @@ -19,7 +19,7 @@ export const ErrorConnecting: React.FC = () => { - + diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/library/library.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/library/library.tsx index 5e268cc0fd214c..ad693628d911e4 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/library/library.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/library/library.tsx @@ -86,7 +86,7 @@ export const Library: React.FC = () => { <> - +

Result

diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/meta_engine_creation/meta_engine_creation.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/meta_engine_creation/meta_engine_creation.tsx index a3dbf7259975b1..85c24f1e42368c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/meta_engine_creation/meta_engine_creation.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/meta_engine_creation/meta_engine_creation.tsx @@ -87,7 +87,7 @@ export const MetaEngineCreation: React.FC = () => { } /> - + = ({ boost, index, name }) => { }; return ( - + + {getBoostForm()} - + = ({ name, type, boosts = [] }) => { ); return ( - + diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.scss b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.scss index 9795564da04d55..065effef9dded2 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.scss +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.scss @@ -19,7 +19,6 @@ } .relevanceTuningAccordionItem { - border: none; border-top: $euiBorderThin; border-radius: 0; } diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx index ab72f29a678c9d..39200a699b3f70 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx @@ -73,7 +73,7 @@ export const RelevanceTuningForm: React.FC = () => { )} {filteredSchemaFields.map((fieldName) => ( - + { {filteredSchemaFieldsWithConflicts.map((fieldName) => ( - +

{fieldName}

diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_preview.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_preview.tsx index 298b692ac7b80f..5e5ee2ea8d0f00 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_preview.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_preview.tsx @@ -48,7 +48,7 @@ export const RelevanceTuningPreview: React.FC = () => { const { engineName, isMetaEngine } = useValues(EngineLogic); return ( - +

{i18n.translate('xpack.enterpriseSearch.appSearch.engine.relevanceTuning.preview.title', { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mapping.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mapping.tsx index bfa3fefb2732d3..ebd034caaedb39 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mapping.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mapping.tsx @@ -166,7 +166,7 @@ export const RoleMapping: React.FC = ({ isNew }) => { - +

{ROLE_TITLE}

@@ -175,7 +175,6 @@ export const RoleMapping: React.FC = ({ isNew }) => {

{FULL_ENGINE_ACCESS_TITLE}

- export{' '} {STANDARD_ROLE_TYPES.map(({ type, description }) => ( = ({ isNew }) => {
{hasAdvancedRoles && ( - +

{ENGINE_ACCESS_TITLE}

diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mappings.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mappings.tsx index e31f5c04bdb457..2ec2b93d1e24f3 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mappings.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/role_mappings/role_mappings.tsx @@ -127,7 +127,7 @@ export const RoleMappings: React.FC = () => { pageTitle={ROLE_MAPPINGS_TITLE} description={ROLE_MAPPINGS_DESCRIPTION} /> - + {roleMappings.length === 0 ? roleMappingEmptyState : roleMappingsTable} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/sample_engine_creation_cta/sample_engine_creation_cta.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/sample_engine_creation_cta/sample_engine_creation_cta.tsx index 8de6b6030ef662..6f1ccd1ae2b532 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/sample_engine_creation_cta/sample_engine_creation_cta.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/sample_engine_creation_cta/sample_engine_creation_cta.tsx @@ -23,9 +23,9 @@ export const SampleEngineCreationCta: React.FC = () => { const { createSampleEngine } = useActions(SampleEngineCreationCtaLogic); return ( - - - + + +

{SAMPLE_ENGINE_CREATION_CTA_TITLE}

diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/settings/settings.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/settings/settings.tsx index 88b62b8ae83f70..2d5dd08f81288a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/settings/settings.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/settings/settings.tsx @@ -21,7 +21,7 @@ export const Settings: React.FC = () => { <> - + diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/not_found/not_found.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/not_found/not_found.tsx index 5699568c405587..f288961b72de43 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/not_found/not_found.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/not_found/not_found.tsx @@ -73,7 +73,7 @@ export const NotFound: React.FC = ({ product = {}, breadcrumbs }) - + } body={ diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.test.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.test.tsx index b5d1ebb899ba11..504acf9ae1c6a2 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.test.tsx @@ -41,14 +41,6 @@ describe('AttributeSelector', () => { expect(wrapper.find('[data-test-subj="AttributeSelector"]').exists()).toBe(true); }); - it('renders disabled panel with className', () => { - const wrapper = shallow(); - - expect(wrapper.find('[data-test-subj="AttributeSelector"]').prop('className')).toEqual( - 'euiPanel--disabled' - ); - }); - describe('Auth Providers', () => { const findAuthProvidersSelect = (wrapper: ShallowWrapper) => wrapper.find('[data-test-subj="AuthProviderSelect"]'); diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.tsx index 48d1447e9bd0f9..0417331be208d6 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/role_mapping/attribute_selector.tsx @@ -100,11 +100,7 @@ export const AttributeSelector: React.FC = ({ handleAuthProviderChange = () => null, }) => { return ( - +

{ATTRIBUTE_SELECTOR_TITLE}

diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/role_mappings/role_mapping.tsx b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/role_mappings/role_mapping.tsx index cf402f4525f9e5..7db1e82d29449c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/role_mappings/role_mapping.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/role_mappings/role_mapping.tsx @@ -141,7 +141,7 @@ export const RoleMapping: React.FC = ({ isNew }) => { - +

{ROLE_LABEL}

@@ -158,7 +158,7 @@ export const RoleMapping: React.FC = ({ isNew }) => {
- +

{GROUP_ASSIGNMENT_TITLE}

diff --git a/x-pack/plugins/fleet/server/services/agents/unenroll.ts b/x-pack/plugins/fleet/server/services/agents/unenroll.ts index ff243eff115702..59ec3a0a632063 100644 --- a/x-pack/plugins/fleet/server/services/agents/unenroll.ts +++ b/x-pack/plugins/fleet/server/services/agents/unenroll.ts @@ -102,7 +102,7 @@ export async function unenrollAgents( // Invalidate all API keys if (apiKeys.length) { - await APIKeyService.invalidateAPIKeys(soClient, apiKeys); + await APIKeyService.invalidateAPIKeys(apiKeys); } } else { // Create unenroll action for each agent @@ -152,10 +152,10 @@ export async function forceUnenrollAgent( await Promise.all([ agent.access_api_key_id - ? APIKeyService.invalidateAPIKeys(soClient, [agent.access_api_key_id]) + ? APIKeyService.invalidateAPIKeys([agent.access_api_key_id]) : undefined, agent.default_api_key_id - ? APIKeyService.invalidateAPIKeys(soClient, [agent.default_api_key_id]) + ? APIKeyService.invalidateAPIKeys([agent.default_api_key_id]) : undefined, ]); diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts index b3edb20d51c4f6..643caa8d3bb6f8 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts @@ -86,7 +86,7 @@ export async function deleteEnrollmentApiKey( ) { const enrollmentApiKey = await getEnrollmentAPIKey(esClient, id); - await invalidateAPIKeys(soClient, [enrollmentApiKey.api_key_id]); + await invalidateAPIKeys([enrollmentApiKey.api_key_id]); await esClient.update({ index: ENROLLMENT_API_KEYS_INDEX, diff --git a/x-pack/plugins/fleet/server/services/api_keys/security.ts b/x-pack/plugins/fleet/server/services/api_keys/security.ts index 599785cb5ff7b0..e68bc406055b0a 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/security.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/security.ts @@ -56,30 +56,14 @@ export async function createAPIKey( } } -export async function invalidateAPIKeys(soClient: SavedObjectsClientContract, ids: string[]) { - const adminUser = await outputService.getAdminUser(soClient); - if (!adminUser) { - throw new Error('No admin user configured'); - } - const request = KibanaRequest.from(({ - path: '/', - route: { settings: {} }, - url: { href: '/' }, - raw: { req: { url: '/' } }, - headers: { - authorization: `Basic ${Buffer.from(`${adminUser.username}:${adminUser.password}`).toString( - 'base64' - )}`, - }, - } as unknown) as Request); - +export async function invalidateAPIKeys(ids: string[]) { const security = appContextService.getSecurity(); if (!security) { throw new Error('Missing security plugin'); } try { - const res = await security.authc.apiKeys.invalidate(request, { + const res = await security.authc.apiKeys.invalidateAsInternalUser({ ids, }); diff --git a/x-pack/plugins/fleet/server/services/fleet_server/saved_object_migrations.ts b/x-pack/plugins/fleet/server/services/fleet_server/saved_object_migrations.ts index 7af2b791f3707f..78172e4dae3669 100644 --- a/x-pack/plugins/fleet/server/services/fleet_server/saved_object_migrations.ts +++ b/x-pack/plugins/fleet/server/services/fleet_server/saved_object_migrations.ts @@ -78,7 +78,6 @@ async function migrateAgents() { .getDecryptedAsInternalUser(AGENT_SAVED_OBJECT_TYPE, so.id); await invalidateAPIKeys( - soClient, [attributes.access_api_key_id, attributes.default_api_key_id].filter( (keyId): keyId is string => keyId !== undefined ) diff --git a/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/table.tsx b/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/table.tsx index f22e826a877ec4..90cb48df4b8d9d 100644 --- a/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/table.tsx +++ b/x-pack/plugins/index_management/public/application/components/component_templates/component_template_list/table.tsx @@ -100,7 +100,7 @@ export const ComponentTable: FunctionComponent = ({ {...reactRouterNavigate(history, '/create_component_template')} > {i18n.translate('xpack.idxMgmt.componentTemplatesList.table.createButtonLabel', { - defaultMessage: 'Create a component template', + defaultMessage: 'Create component template', })} , ], diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx index 95d4a8a653dc12..9cf807052fcb9d 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx @@ -91,7 +91,7 @@ export const PipelineTable: FunctionComponent = ({ {...reactRouterNavigate(history, '/create')} > {i18n.translate('xpack.ingestPipelines.list.table.createPipelineButtonLabel', { - defaultMessage: 'Create a pipeline', + defaultMessage: 'Create pipeline', })} , ], diff --git a/x-pack/plugins/maps/common/constants.ts b/x-pack/plugins/maps/common/constants.ts index ecdf94a076809c..7152d76afbdbe5 100644 --- a/x-pack/plugins/maps/common/constants.ts +++ b/x-pack/plugins/maps/common/constants.ts @@ -300,3 +300,7 @@ export type FieldFormatter = (value: RawValue) => string | number; export const INDEX_META_DATA_CREATED_BY = 'maps-drawing-data-ingest'; export const MAX_DRAWING_SIZE_BYTES = 10485760; // 10MB + +export const emsWorldLayerId = 'world_countries'; +export const emsRegionLayerId = 'administrative_regions_lvl2'; +export const emsUsaZipLayerId = 'usa_zip_codes'; diff --git a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts index dd01b7b596c302..e1f682678df4b9 100644 --- a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts @@ -26,6 +26,7 @@ export type MapFilters = { }; type ESSearchSourceSyncMeta = { + filterByMapBounds: boolean; sortField: string; sortOrder: SortDirection; scalingType: SCALING_TYPES; diff --git a/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts index 7b757aa9cf10b1..9c4ef3fde6d164 100644 --- a/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/source_descriptor_types.ts @@ -95,8 +95,8 @@ export type ESGeoLineSourceDescriptor = AbstractESAggSourceDescriptor & { export type ESSearchSourceDescriptor = AbstractESSourceDescriptor & { geoField: string; - filterByMapBounds?: boolean; - tooltipProperties?: string[]; + filterByMapBounds: boolean; + tooltipProperties: string[]; sortField: string; sortOrder: SortDirection; scalingType: SCALING_TYPES; diff --git a/x-pack/plugins/maps/public/api/ems.ts b/x-pack/plugins/maps/public/api/ems.ts new file mode 100644 index 00000000000000..da6e88c84e22c6 --- /dev/null +++ b/x-pack/plugins/maps/public/api/ems.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EMSTermJoinConfig, SampleValuesConfig } from '../ems_autosuggest'; +import { lazyLoadMapModules } from '../lazy_load_bundle'; + +export async function suggestEMSTermJoinConfig( + sampleValuesConfig: SampleValuesConfig +): Promise { + const mapModules = await lazyLoadMapModules(); + return await mapModules.suggestEMSTermJoinConfig(sampleValuesConfig); +} diff --git a/x-pack/plugins/maps/public/api/index.ts b/x-pack/plugins/maps/public/api/index.ts index 6aac4f6410e6d4..186fd98c90bf6d 100644 --- a/x-pack/plugins/maps/public/api/index.ts +++ b/x-pack/plugins/maps/public/api/index.ts @@ -8,3 +8,4 @@ export { MapsStartApi } from './start_api'; export { createLayerDescriptors } from './create_layer_descriptors'; export { registerLayerWizard, registerSource } from './register'; +export { suggestEMSTermJoinConfig } from './ems'; diff --git a/x-pack/plugins/maps/public/api/register.ts b/x-pack/plugins/maps/public/api/register.ts index 2be928585a936a..037eeadc48df77 100644 --- a/x-pack/plugins/maps/public/api/register.ts +++ b/x-pack/plugins/maps/public/api/register.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { SourceRegistryEntry } from '../classes/sources/source_registry'; -import { LayerWizard } from '../classes/layers/layer_wizard_registry'; +import type { SourceRegistryEntry } from '../classes/sources/source_registry'; +import type { LayerWizard } from '../classes/layers/layer_wizard_registry'; import { lazyLoadMapModules } from '../lazy_load_bundle'; export async function registerLayerWizard(layerWizard: LayerWizard): Promise { diff --git a/x-pack/plugins/maps/public/api/start_api.ts b/x-pack/plugins/maps/public/api/start_api.ts index 3005f6522df0d0..e4213fe07a49cd 100644 --- a/x-pack/plugins/maps/public/api/start_api.ts +++ b/x-pack/plugins/maps/public/api/start_api.ts @@ -5,10 +5,11 @@ * 2.0. */ -import { LayerDescriptor } from '../../common/descriptor_types'; -import { SourceRegistryEntry } from '../classes/sources/source_registry'; -import { LayerWizard } from '../classes/layers/layer_wizard_registry'; +import type { LayerDescriptor } from '../../common/descriptor_types'; +import type { SourceRegistryEntry } from '../classes/sources/source_registry'; +import type { LayerWizard } from '../classes/layers/layer_wizard_registry'; import type { CreateLayerDescriptorParams } from '../classes/sources/es_search_source'; +import type { SampleValuesConfig, EMSTermJoinConfig } from '../ems_autosuggest'; export interface MapsStartApi { createLayerDescriptors: { @@ -23,4 +24,5 @@ export interface MapsStartApi { }; registerLayerWizard(layerWizard: LayerWizard): Promise; registerSource(entry: SourceRegistryEntry): Promise; + suggestEMSTermJoinConfig(config: SampleValuesConfig): Promise; } diff --git a/x-pack/plugins/maps/public/classes/layers/choropleth_layer_wizard/layer_template.tsx b/x-pack/plugins/maps/public/classes/layers/choropleth_layer_wizard/layer_template.tsx index 909cb6909ee567..82a741e7ccdab0 100644 --- a/x-pack/plugins/maps/public/classes/layers/choropleth_layer_wizard/layer_template.tsx +++ b/x-pack/plugins/maps/public/classes/layers/choropleth_layer_wizard/layer_template.tsx @@ -24,7 +24,7 @@ import { EMSFileSelect } from '../../../components/ems_file_select'; import { GeoIndexPatternSelect } from '../../../components/geo_index_pattern_select'; import { SingleFieldSelect } from '../../../components/single_field_select'; import { getGeoFields, getSourceFields, getTermsFields } from '../../../index_pattern_util'; -import { getEmsFileLayers } from '../../../meta'; +import { getEmsFileLayers } from '../../../util'; import { getIndexPatternSelectComponent, getIndexPatternService } from '../../../kibana_services'; import { createEmsChoroplethLayerDescriptor, diff --git a/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.test.ts b/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.test.ts index 151db80da21428..1d21401778ae63 100644 --- a/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.test.ts +++ b/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -jest.mock('../../meta', () => { +jest.mock('../../util', () => { return {}; }); jest.mock('../../kibana_services', () => { @@ -33,7 +33,7 @@ import { createBasemapLayerDescriptor } from './create_basemap_layer_descriptor' describe('kibana.yml configured with map.tilemap.url', () => { beforeAll(() => { // eslint-disable-next-line @typescript-eslint/no-var-requires - require('../../meta').getKibanaTileMap = () => { + require('../../util').getKibanaTileMap = () => { return { url: 'myTileUrl', }; @@ -61,7 +61,7 @@ describe('kibana.yml configured with map.tilemap.url', () => { describe('EMS is enabled', () => { beforeAll(() => { // eslint-disable-next-line @typescript-eslint/no-var-requires - require('../../meta').getKibanaTileMap = () => { + require('../../util').getKibanaTileMap = () => { return null; }; // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -95,7 +95,7 @@ describe('EMS is enabled', () => { describe('EMS is not enabled', () => { beforeAll(() => { // eslint-disable-next-line @typescript-eslint/no-var-requires - require('../../meta').getKibanaTileMap = () => { + require('../../util').getKibanaTileMap = () => { return null; }; // eslint-disable-next-line @typescript-eslint/no-var-requires diff --git a/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.ts b/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.ts index edffa5dc7e0cbf..033b0de025ee19 100644 --- a/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.ts +++ b/x-pack/plugins/maps/public/classes/layers/create_basemap_layer_descriptor.ts @@ -7,7 +7,7 @@ import _ from 'lodash'; import { LayerDescriptor } from '../../../common/descriptor_types'; -import { getKibanaTileMap } from '../../meta'; +import { getKibanaTileMap } from '../../util'; import { getEMSSettings } from '../../kibana_services'; // @ts-expect-error import { KibanaTilemapSource } from '../sources/kibana_tilemap_source'; diff --git a/x-pack/plugins/maps/public/classes/layers/icons/top_hits_layer_icon.tsx b/x-pack/plugins/maps/public/classes/layers/icons/top_hits_layer_icon.tsx new file mode 100644 index 00000000000000..9e67c578951725 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/layers/icons/top_hits_layer_icon.tsx @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FunctionComponent } from 'react'; + +export const TopHitsLayerIcon: FunctionComponent = () => ( + + + + + + + +); diff --git a/x-pack/plugins/maps/public/classes/layers/icons/tracks_layer_icon.tsx b/x-pack/plugins/maps/public/classes/layers/icons/tracks_layer_icon.tsx index 77b3efe5d2d830..c2d275ba3133d6 100644 --- a/x-pack/plugins/maps/public/classes/layers/icons/tracks_layer_icon.tsx +++ b/x-pack/plugins/maps/public/classes/layers/icons/tracks_layer_icon.tsx @@ -13,9 +13,9 @@ export const TracksLayerIcon: FunctionComponent = () => ( className="mapLayersWizardIcon__highlight" d="M12.733 12.136h32.283v.545H12.935L4.452 19.98l-.356-.413 8.637-7.43z" /> - + - + diff --git a/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts b/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts index bed7599f890736..804352f5bede72 100644 --- a/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts +++ b/x-pack/plugins/maps/public/classes/layers/load_layer_wizards.ts @@ -7,8 +7,10 @@ import { registerLayerWizard } from './layer_wizard_registry'; import { uploadLayerWizardConfig } from './file_upload_wizard'; -// @ts-ignore -import { esDocumentsLayerWizardConfig } from '../sources/es_search_source'; +import { + esDocumentsLayerWizardConfig, + esTopHitsLayerWizardConfig, +} from '../sources/es_search_source'; import { clustersLayerWizardConfig, heatmapLayerWizardConfig } from '../sources/es_geo_grid_source'; import { geoLineLayerWizardConfig } from '../sources/es_geo_line_source'; // @ts-ignore @@ -37,13 +39,13 @@ export function registerLayerWizards() { // Registration order determines display order registerLayerWizard(uploadLayerWizardConfig); - // @ts-ignore registerLayerWizard(esDocumentsLayerWizardConfig); // @ts-ignore registerLayerWizard(choroplethLayerWizardConfig); registerLayerWizard(clustersLayerWizardConfig); // @ts-ignore registerLayerWizard(heatmapLayerWizardConfig); + registerLayerWizard(esTopHitsLayerWizardConfig); registerLayerWizard(geoLineLayerWizardConfig); // @ts-ignore registerLayerWizard(point2PointLayerWizardConfig); diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts index b9cfb0067abd2a..5dd4f240c2ba95 100644 --- a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts +++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.test.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { emsWorldLayerId } from '../../../../../common'; + jest.mock('../../../../kibana_services', () => { return { getIsDarkMode() { @@ -71,7 +73,7 @@ describe('createLayerDescriptor', () => { maxZoom: 24, minZoom: 0, sourceDescriptor: { - id: 'world_countries', + id: emsWorldLayerId, tooltipProperties: ['name', 'iso2'], type: 'EMS_FILE', }, diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts index 03870e7668189a..adf6f1d7f270d2 100644 --- a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts +++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/create_layer_descriptor.ts @@ -18,6 +18,7 @@ import { import { AGG_TYPE, COLOR_MAP_TYPE, + emsWorldLayerId, FIELD_ORIGIN, GRID_RESOLUTION, RENDER_AS, @@ -182,7 +183,7 @@ export function createLayerDescriptor({ }, ], sourceDescriptor: EMSFileSource.createDescriptor({ - id: 'world_countries', + id: emsWorldLayerId, tooltipProperties: ['name', 'iso2'], }), style: VectorStyle.createDescriptor({ diff --git a/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js b/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js index 3e7d25266821c4..5096e5e29bf238 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js +++ b/x-pack/plugins/maps/public/classes/layers/vector_tile_layer/vector_tile_layer.js @@ -8,7 +8,7 @@ import { TileLayer } from '../tile_layer/tile_layer'; import _ from 'lodash'; import { SOURCE_DATA_REQUEST_ID, LAYER_TYPE, LAYER_STYLE_TYPE } from '../../../../common/constants'; -import { isRetina } from '../../../meta'; +import { isRetina } from '../../../util'; import { addSpriteSheetToMapFromImageData, loadSpriteSheetImageData, diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx index 5881982b893c11..c19ded6c2593e3 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx @@ -18,7 +18,7 @@ import { VECTOR_SHAPE_TYPE, FORMAT_TYPE, } from '../../../../common/constants'; -import { getEmsFileLayers } from '../../../meta'; +import { fetchGeoJson, getEmsFileLayers } from '../../../util'; import { getDataSourceLabel } from '../../../../common/i18n_getters'; import { UpdateSourceEditor } from './update_source_editor'; import { EMSFileField } from '../../fields/ems_file_field'; @@ -123,12 +123,11 @@ export class EMSFileSource extends AbstractVectorSource implements IEmsFileSourc async getGeoJsonWithMeta(): Promise { const emsFileLayer = await this.getEMSFileLayer(); - // @ts-ignore - const featureCollection = await AbstractVectorSource.getGeoJson({ - format: emsFileLayer.getDefaultFormatType() as FORMAT_TYPE, - featureCollectionPath: 'data', - fetchUrl: emsFileLayer.getDefaultFormatUrl(), - }); + const featureCollection = await fetchGeoJson( + emsFileLayer.getDefaultFormatUrl(), + emsFileLayer.getDefaultFormatType() as FORMAT_TYPE, + 'data' + ); const emsIdField = emsFileLayer.getFields().find((field) => { return field.type === 'id'; diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx index 3a8bb9c083afd5..abac9cbe5d0267 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/update_source_editor.tsx @@ -9,7 +9,7 @@ import React, { Component, Fragment } from 'react'; import { EuiTitle, EuiPanel, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { TooltipSelector } from '../../../components/tooltip_selector'; -import { getEmsFileLayers } from '../../../meta'; +import { getEmsFileLayers } from '../../../util'; import { IEmsFileSource } from './ems_file_source'; import { IField } from '../../fields/field'; import { OnSourceChangeArgs } from '../../../connected_components/layer_panel/view'; diff --git a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.js b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.js index f2216f2afd2da4..97fb20b795bf6b 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.js +++ b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.js @@ -7,7 +7,7 @@ import React from 'react'; import { AbstractTMSSource } from '../tms_source'; -import { getEmsTmsServices } from '../../../meta'; +import { getEmsTmsServices } from '../../../util'; import { UpdateSourceEditor } from './update_source_editor'; import { i18n } from '@kbn/i18n'; import { getDataSourceLabel } from '../../../../common/i18n_getters'; diff --git a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.test.js b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.test.js index 1b62aea1800075..db5191e62fc049 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.test.js +++ b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_tms_source.test.js @@ -5,7 +5,7 @@ * 2.0. */ -jest.mock('../../../meta', () => { +jest.mock('../../../util', () => { return { getEmsTmsServices: () => { class MockTMSService { diff --git a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/tile_service_select.tsx b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/tile_service_select.tsx index 5f0f406d53e86c..e4a6fed934b8d4 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/tile_service_select.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/tile_service_select.tsx @@ -9,7 +9,7 @@ import React, { ChangeEvent, Component } from 'react'; import { EuiSelect, EuiSelectOption, EuiFormRow } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { getEmsTmsServices } from '../../../meta'; +import { getEmsTmsServices } from '../../../util'; import { getEmsUnavailableMessage } from '../../../components/ems_unavailable_message'; export const AUTO_SELECT = 'auto_select'; diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/scaling_form.test.tsx.snap b/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/scaling_form.test.tsx.snap index 120ff2e7adde3d..03f2594f287eaf 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/scaling_form.test.tsx.snap +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/scaling_form.test.tsx.snap @@ -31,12 +31,6 @@ exports[`scaling form should disable clusters option when clustering is not supp label="Limit results to 10000." onChange={[Function]} /> - - `; - -exports[`scaling form should render top hits form when scaling type is TOP_HITS 1`] = ` - - -
- -
-
- - -
- - - - - - - Use vector tiles for faster display of large datasets. - - } - delay="regular" - position="left" - > - - -
-
- - - - - - - -
-`; diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/update_source_editor.test.js.snap b/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/update_source_editor.test.js.snap index 3153f7b20f3305..0ff94163f9230d 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/update_source_editor.test.js.snap +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/__snapshots__/update_source_editor.test.js.snap @@ -98,9 +98,6 @@ exports[`should enable sort order select when sort field provided 1`] = ` onChange={[Function]} scalingType="LIMIT" supportsClustering={false} - termFields={null} - topHitsSize={1} - topHitsSplitField="trackId" />
{ - const { - indexPattern, - geoFieldName, - filterByMapBounds, - scalingType, - topHitsSplitField, - topHitsSize, - } = this.state; + const { indexPattern, geoFieldName, filterByMapBounds, scalingType } = this.state; const sourceConfig = indexPattern && geoFieldName @@ -113,8 +103,6 @@ export class CreateSourceEditor extends Component { geoField: geoFieldName, filterByMapBounds, scalingType, - topHitsSplitField, - topHitsSize, } : null; this.props.onSourceConfigChange(sourceConfig); @@ -167,9 +155,6 @@ export class CreateSourceEditor extends Component { ) : null } - termFields={getTermsFields(this.state.indexPattern.fields)} - topHitsSplitField={this.state.topHitsSplitField} - topHitsSize={this.state.topHitsSize} /> ); diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx index c0606b5f4aec6a..26771c1bed0231 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx @@ -10,7 +10,6 @@ import React from 'react'; // @ts-ignore import { CreateSourceEditor } from './create_source_editor'; import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_registry'; -// @ts-ignore import { ESSearchSource, sourceTitle } from './es_search_source'; import { BlendedVectorLayer } from '../../layers/blended_vector_layer/blended_vector_layer'; import { VectorLayer } from '../../layers/vector_layer'; diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx index eae00710c4c25f..168448b6f72a02 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx @@ -60,6 +60,7 @@ import { ITooltipProperty } from '../../tooltips/tooltip_property'; import { DataRequest } from '../../util/data_request'; import { SortDirection, SortDirectionNumeric } from '../../../../../../../src/plugins/data/common'; import { isValidStringConfig } from '../../util/valid_string_config'; +import { TopHitsUpdateSourceEditor } from './top_hits'; export const sourceTitle = i18n.translate('xpack.maps.source.esSearchTitle', { defaultMessage: 'Documents', @@ -166,6 +167,22 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye } renderSourceSettingsEditor(sourceEditorArgs: SourceEditorArgs): ReactElement | null { + if (this._isTopHits()) { + return ( + + ); + } + const getGeoField = () => { return this._getGeoField(); }; @@ -180,8 +197,6 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye sortOrder={this._descriptor.sortOrder} scalingType={this._descriptor.scalingType} filterByMapBounds={this.isFilterByMapBounds()} - topHitsSplitField={this._descriptor.topHitsSplitField} - topHitsSize={this._descriptor.topHitsSize} /> ); } @@ -658,6 +673,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye getSyncMeta(): VectorSourceSyncMeta | null { return { + filterByMapBounds: this._descriptor.filterByMapBounds, sortField: this._descriptor.sortField, sortOrder: this._descriptor.sortOrder, scalingType: this._descriptor.scalingType, diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/index.ts b/x-pack/plugins/maps/public/classes/sources/es_search_source/index.ts index 73e7963024471d..75217c0a29c082 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/index.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/index.ts @@ -11,3 +11,4 @@ export { createDefaultLayerDescriptor, esDocumentsLayerWizardConfig, } from './es_documents_layer_wizard'; +export { esTopHitsLayerWizardConfig } from './top_hits'; diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.test.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.test.tsx index fe47208c32690e..b02eacc1334672 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.test.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.test.tsx @@ -26,8 +26,6 @@ const defaultProps = { scalingType: SCALING_TYPES.LIMIT, supportsClustering: true, termFields: [], - topHitsSplitField: null, - topHitsSize: 1, }; describe('scaling form', () => { @@ -48,12 +46,4 @@ describe('scaling form', () => { expect(component).toMatchSnapshot(); }); - - test('should render top hits form when scaling type is TOP_HITS', async () => { - const component = shallow( - - ); - - expect(component).toMatchSnapshot(); - }); }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.tsx index 6190c7ed8df3fc..b9ce43dbbdad42 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/scaling_form.tsx @@ -19,19 +19,9 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { SingleFieldSelect } from '../../../components/single_field_select'; import { getIndexPatternService } from '../../../kibana_services'; -// @ts-ignore -import { ValidatedRange } from '../../../components/validated_range'; -import { - DEFAULT_MAX_INNER_RESULT_WINDOW, - DEFAULT_MAX_RESULT_WINDOW, - LAYER_TYPE, - SCALING_TYPES, -} from '../../../../common/constants'; -// @ts-ignore +import { DEFAULT_MAX_RESULT_WINDOW, LAYER_TYPE, SCALING_TYPES } from '../../../../common/constants'; import { loadIndexSettings } from './load_index_settings'; -import { IFieldType } from '../../../../../../../src/plugins/data/public'; import { OnSourceChangeArgs } from '../../../connected_components/layer_panel/view'; interface Props { @@ -41,19 +31,14 @@ interface Props { scalingType: SCALING_TYPES; supportsClustering: boolean; clusteringDisabledReason?: string | null; - termFields: IFieldType[]; - topHitsSplitField: string | null; - topHitsSize: number; } interface State { - maxInnerResultWindow: number; maxResultWindow: number; } export class ScalingForm extends Component { state = { - maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW, maxResultWindow: DEFAULT_MAX_RESULT_WINDOW, }; _isMounted = false; @@ -70,11 +55,9 @@ export class ScalingForm extends Component { async loadIndexSettings() { try { const indexPattern = await getIndexPatternService().get(this.props.indexPatternId); - const { maxInnerResultWindow, maxResultWindow } = await loadIndexSettings( - indexPattern!.title - ); + const { maxResultWindow } = await loadIndexSettings(indexPattern!.title); if (this._isMounted) { - this.setState({ maxInnerResultWindow, maxResultWindow }); + this.setState({ maxResultWindow }); } } catch (err) { return; @@ -98,71 +81,6 @@ export class ScalingForm extends Component { this.props.onChange({ propName: 'filterByMapBounds', value: event.target.checked }); }; - _onTopHitsSplitFieldChange = (topHitsSplitField?: string) => { - if (!topHitsSplitField) { - return; - } - this.props.onChange({ propName: 'topHitsSplitField', value: topHitsSplitField }); - }; - - _onTopHitsSizeChange = (size: number) => { - this.props.onChange({ propName: 'topHitsSize', value: size }); - }; - - _renderTopHitsForm() { - let sizeSlider; - if (this.props.topHitsSplitField) { - sizeSlider = ( - - - - ); - } - - return ( - - - - - - {sizeSlider} - - ); - } - _renderClusteringRadio() { const clusteringRadio = ( { render() { let filterByBoundsSwitch; - if ( - this.props.scalingType === SCALING_TYPES.TOP_HITS || - this.props.scalingType === SCALING_TYPES.LIMIT - ) { + if (this.props.scalingType === SCALING_TYPES.LIMIT) { filterByBoundsSwitch = ( { ); } - let topHitsOptionsForm = null; - if (this.props.scalingType === SCALING_TYPES.TOP_HITS) { - topHitsOptionsForm = ( - - - {this._renderTopHitsForm()} - - ); - } - return ( @@ -267,21 +172,12 @@ export class ScalingForm extends Component { checked={this.props.scalingType === SCALING_TYPES.LIMIT} onChange={() => this._onScalingTypeChange(SCALING_TYPES.LIMIT)} /> - this._onScalingTypeChange(SCALING_TYPES.TOP_HITS)} - /> {this._renderClusteringRadio()} {this._renderMVTRadio()} {filterByBoundsSwitch} - {topHitsOptionsForm} ); } diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/create_source_editor.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/create_source_editor.tsx new file mode 100644 index 00000000000000..ec656be3efeaea --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/create_source_editor.tsx @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { Component } from 'react'; +import { EuiPanel } from '@elastic/eui'; + +import { SCALING_TYPES } from '../../../../../common/constants'; +import { GeoFieldSelect } from '../../../../components/geo_field_select'; +import { GeoIndexPatternSelect } from '../../../../components/geo_index_pattern_select'; +import { getGeoFields, getTermsFields, getSortFields } from '../../../../index_pattern_util'; +import { ESSearchSourceDescriptor } from '../../../../../common/descriptor_types'; +import { + IndexPattern, + IFieldType, + SortDirection, +} from '../../../../../../../../src/plugins/data/common'; +import { TopHitsForm } from './top_hits_form'; +import { OnSourceChangeArgs } from '../../../../connected_components/layer_panel/view'; + +interface Props { + onSourceConfigChange: (sourceConfig: Partial | null) => void; +} + +interface State { + indexPattern: IndexPattern | null; + geoFields: IFieldType[]; + geoFieldName: string | null; + sortField: string | null; + sortFields: IFieldType[]; + sortOrder: SortDirection; + termFields: IFieldType[]; + topHitsSplitField: string | null; + topHitsSize: number; +} + +export class CreateSourceEditor extends Component { + state: State = { + indexPattern: null, + geoFields: [], + geoFieldName: null, + sortField: null, + sortFields: [], + sortOrder: SortDirection.desc, + termFields: [], + topHitsSplitField: null, + topHitsSize: 1, + }; + + _onIndexPatternSelect = (indexPattern: IndexPattern) => { + const geoFields = getGeoFields(indexPattern.fields); + + this.setState( + { + indexPattern, + geoFields, + geoFieldName: geoFields.length ? geoFields[0].name : null, + sortField: indexPattern.timeFieldName ? indexPattern.timeFieldName : null, + sortFields: getSortFields(indexPattern.fields), + termFields: getTermsFields(indexPattern.fields), + topHitsSplitField: null, + }, + this._previewLayer + ); + }; + + _onGeoFieldSelect = (geoFieldName?: string) => { + this.setState({ geoFieldName: geoFieldName ? geoFieldName : null }, this._previewLayer); + }; + + _onTopHitsPropChange = ({ propName, value }: OnSourceChangeArgs) => { + this.setState( + // @ts-expect-error + { [propName]: value }, + this._previewLayer + ); + }; + + _previewLayer = () => { + const { + indexPattern, + geoFieldName, + sortField, + sortOrder, + topHitsSplitField, + topHitsSize, + } = this.state; + + const tooltipProperties: string[] = []; + if (topHitsSplitField) { + tooltipProperties.push(topHitsSplitField); + } + if (indexPattern && indexPattern.timeFieldName) { + tooltipProperties.push(indexPattern.timeFieldName); + } + + const sourceConfig = + indexPattern && geoFieldName && sortField && topHitsSplitField + ? { + indexPatternId: indexPattern.id, + geoField: geoFieldName, + scalingType: SCALING_TYPES.TOP_HITS, + sortField, + sortOrder, + tooltipProperties, + topHitsSplitField, + topHitsSize, + } + : null; + this.props.onSourceConfigChange(sourceConfig); + }; + + _renderGeoSelect() { + return this.state.indexPattern ? ( + + ) : null; + } + + _renderTopHitsPanel() { + if (!this.state.indexPattern || !this.state.indexPattern.id || !this.state.geoFieldName) { + return null; + } + + return ( + + ); + } + + render() { + return ( + + + + {this._renderGeoSelect()} + + {this._renderTopHitsPanel()} + + ); + } +} diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/index.ts b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/index.ts new file mode 100644 index 00000000000000..135ed7c991b3ab --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { TopHitsUpdateSourceEditor } from './update_source_editor'; +export { esTopHitsLayerWizardConfig } from './wizard'; diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/top_hits_form.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/top_hits_form.tsx new file mode 100644 index 00000000000000..e4f196e5e8a858 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/top_hits_form.tsx @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { ChangeEvent, Component, Fragment } from 'react'; +import { EuiFormRow, EuiSelect } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { SingleFieldSelect } from '../../../../components/single_field_select'; +import { getIndexPatternService } from '../../../../kibana_services'; +// @ts-expect-error +import { ValidatedRange } from '../../../../components/validated_range'; +import { DEFAULT_MAX_INNER_RESULT_WINDOW } from '../../../../../common/constants'; +import { loadIndexSettings } from '../load_index_settings'; +import { OnSourceChangeArgs } from '../../../../connected_components/layer_panel/view'; +import { IFieldType, SortDirection } from '../../../../../../../../src/plugins/data/public'; + +interface Props { + indexPatternId: string; + isColumnCompressed?: boolean; + onChange: (args: OnSourceChangeArgs) => void; + sortField: string; + sortFields: IFieldType[]; + sortOrder: SortDirection; + termFields: IFieldType[]; + topHitsSplitField: string | null; + topHitsSize: number; +} + +interface State { + maxInnerResultWindow: number; +} + +export class TopHitsForm extends Component { + state = { + maxInnerResultWindow: DEFAULT_MAX_INNER_RESULT_WINDOW, + }; + _isMounted = false; + + componentDidMount() { + this._isMounted = true; + this.loadIndexSettings(); + } + + componentWillUnmount() { + this._isMounted = false; + } + + _onTopHitsSplitFieldChange = (topHitsSplitField?: string) => { + if (!topHitsSplitField) { + return; + } + this.props.onChange({ propName: 'topHitsSplitField', value: topHitsSplitField }); + }; + + _onTopHitsSizeChange = (size: number) => { + this.props.onChange({ propName: 'topHitsSize', value: size }); + }; + + _onSortFieldChange = (sortField?: string) => { + this.props.onChange({ propName: 'sortField', value: sortField }); + }; + + _onSortOrderChange = (event: ChangeEvent) => { + this.props.onChange({ propName: 'sortOrder', value: event.target.value }); + }; + + async loadIndexSettings() { + try { + const indexPattern = await getIndexPatternService().get(this.props.indexPatternId); + const { maxInnerResultWindow } = await loadIndexSettings(indexPattern!.title); + if (this._isMounted) { + this.setState({ maxInnerResultWindow }); + } + } catch (err) { + return; + } + } + + render() { + let sizeSlider; + let sortField; + let sortOrder; + if (this.props.topHitsSplitField) { + sizeSlider = ( + + + + ); + + sortField = ( + + + + ); + + sortOrder = ( + + + + ); + } + + return ( + + + + + + {sizeSlider} + + {sortField} + + {sortOrder} + + ); + } +} diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/update_source_editor.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/update_source_editor.tsx new file mode 100644 index 00000000000000..90553d47e644a5 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/update_source_editor.tsx @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { Component, Fragment } from 'react'; +import { EuiFormRow, EuiTitle, EuiPanel, EuiSpacer, EuiSwitch, EuiSwitchEvent } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { FIELD_ORIGIN } from '../../../../../common/constants'; +import { TooltipSelector } from '../../../../components/tooltip_selector'; + +import { getIndexPatternService } from '../../../../kibana_services'; +import { getTermsFields, getSortFields, getSourceFields } from '../../../../index_pattern_util'; +import { SortDirection, IFieldType } from '../../../../../../../../src/plugins/data/public'; +import { ESDocField } from '../../../fields/es_doc_field'; +import { OnSourceChangeArgs } from '../../../../connected_components/layer_panel/view'; +import { TopHitsForm } from './top_hits_form'; +import { ESSearchSource } from '../es_search_source'; +import { IField } from '../../../fields/field'; + +interface Props { + filterByMapBounds: boolean; + indexPatternId: string; + onChange: (args: OnSourceChangeArgs) => void; + tooltipFields: IField[]; + topHitsSplitField: string; + topHitsSize: number; + sortField: string; + sortOrder: SortDirection; + source: ESSearchSource; +} + +interface State { + loadError?: string; + sourceFields: IField[]; + termFields: IFieldType[]; + sortFields: IFieldType[]; +} + +export class TopHitsUpdateSourceEditor extends Component { + private _isMounted = false; + + state: State = { + sourceFields: [], + termFields: [], + sortFields: [], + }; + + componentDidMount() { + this._isMounted = true; + this.loadFields(); + } + + componentWillUnmount() { + this._isMounted = false; + } + + async loadFields() { + let indexPattern; + try { + indexPattern = await getIndexPatternService().get(this.props.indexPatternId); + } catch (err) { + if (this._isMounted) { + this.setState({ + loadError: i18n.translate('xpack.maps.source.esSearch.loadErrorMessage', { + defaultMessage: `Unable to find Index pattern {id}`, + values: { + id: this.props.indexPatternId, + }, + }), + }); + } + return; + } + + if (!this._isMounted) { + return; + } + + const rawTooltipFields = getSourceFields(indexPattern.fields); + const sourceFields = rawTooltipFields.map((field) => { + return new ESDocField({ + fieldName: field.name, + source: this.props.source, + origin: FIELD_ORIGIN.SOURCE, + }); + }); + + this.setState({ + sourceFields, + termFields: getTermsFields(indexPattern.fields), + sortFields: getSortFields(indexPattern.fields), + }); + } + _onTooltipPropertiesChange = (propertyNames: string[]) => { + this.props.onChange({ propName: 'tooltipProperties', value: propertyNames }); + }; + + _onFilterByMapBoundsChange = (event: EuiSwitchEvent) => { + this.props.onChange({ propName: 'filterByMapBounds', value: event.target.checked }); + }; + + render() { + return ( + + + +
+ +
+
+ + + + +
+ + + + +
+ +
+
+ + + + + + + + +
+ +
+ ); + } +} diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/wizard.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/wizard.tsx new file mode 100644 index 00000000000000..e02ada305ecff3 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/top_hits/wizard.tsx @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { CreateSourceEditor } from './create_source_editor'; +import { LayerWizard, RenderWizardArguments } from '../../../layers/layer_wizard_registry'; +import { VectorLayer } from '../../../layers/vector_layer'; +import { LAYER_WIZARD_CATEGORY } from '../../../../../common/constants'; +import { TopHitsLayerIcon } from '../../../layers/icons/top_hits_layer_icon'; +import { ESSearchSourceDescriptor } from '../../../../../common/descriptor_types'; +import { ESSearchSource } from '../es_search_source'; + +export const esTopHitsLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH], + description: i18n.translate('xpack.maps.source.topHitsDescription', { + defaultMessage: + 'Display the most relevant documents per entity, e.g. the most recent GPS hits per vehicle.', + }), + icon: TopHitsLayerIcon, + renderWizard: ({ previewLayers, mapColors }: RenderWizardArguments) => { + const onSourceConfigChange = (sourceConfig: Partial | null) => { + if (!sourceConfig) { + previewLayers([]); + return; + } + + const sourceDescriptor = ESSearchSource.createDescriptor(sourceConfig); + const layerDescriptor = VectorLayer.createDescriptor({ sourceDescriptor }, mapColors); + previewLayers([layerDescriptor]); + }; + return ; + }, + title: i18n.translate('xpack.maps.source.topHitsTitle', { + defaultMessage: 'Top hits per entity', + }), +}; diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.js b/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.js index 1e870f423171f4..86326660110651 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.js +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.js @@ -8,6 +8,7 @@ import React, { Fragment, Component } from 'react'; import PropTypes from 'prop-types'; import { EuiFormRow, EuiSelect, EuiTitle, EuiPanel, EuiSpacer } from '@elastic/eui'; +import { FIELD_ORIGIN } from '../../../../common/constants'; import { SingleFieldSelect } from '../../../components/single_field_select'; import { TooltipSelector } from '../../../components/tooltip_selector'; @@ -15,7 +16,6 @@ import { getIndexPatternService } from '../../../kibana_services'; import { i18n } from '@kbn/i18n'; import { getGeoTileAggNotSupportedReason, - getTermsFields, getSourceFields, supportsGeoTileAgg, } from '../../../index_pattern_util'; @@ -33,14 +33,11 @@ export class UpdateSourceEditor extends Component { sortField: PropTypes.string, sortOrder: PropTypes.string.isRequired, scalingType: PropTypes.string.isRequired, - topHitsSplitField: PropTypes.string, - topHitsSize: PropTypes.number.isRequired, source: PropTypes.object, }; state = { sourceFields: null, - termFields: null, sortFields: null, supportsClustering: false, mvtDisabledReason: null, @@ -94,6 +91,7 @@ export class UpdateSourceEditor extends Component { return new ESDocField({ fieldName: field.name, source: this.props.source, + origin: FIELD_ORIGIN.SOURCE, }); }); @@ -102,7 +100,6 @@ export class UpdateSourceEditor extends Component { clusteringDisabledReason: getGeoTileAggNotSupportedReason(geoField), mvtDisabledReason: null, sourceFields: sourceFields, - termFields: getTermsFields(indexPattern.fields), //todo change term fields to use fields sortFields: indexPattern.fields.filter( (field) => field.sortable && !indexPatterns.isNestedField(field) ), //todo change sort fields to use fields @@ -212,9 +209,6 @@ export class UpdateSourceEditor extends Component { scalingType={this.props.scalingType} supportsClustering={this.state.supportsClustering} clusteringDisabledReason={this.state.clusteringDisabledReason} - termFields={this.state.termFields} - topHitsSplitField={this.props.topHitsSplitField} - topHitsSize={this.props.topHitsSize} />
); diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.test.js b/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.test.js index 00a7b2b0b34906..f54947bc91d192 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.test.js +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/update_source_editor.test.js @@ -26,8 +26,6 @@ const defaultProps = { tooltipFields: [], sortOrder: 'DESC', scalingType: SCALING_TYPES.LIMIT, - topHitsSplitField: 'trackId', - topHitsSize: 1, }; test('should render update source editor', async () => { diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/create_source_editor.js b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/create_source_editor.js index 436d05fdbc6e81..1278d84f103da9 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/create_source_editor.js +++ b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/create_source_editor.js @@ -8,7 +8,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import { EuiSelect, EuiFormRow, EuiPanel } from '@elastic/eui'; -import { getKibanaRegionList } from '../../../meta'; +import { getKibanaRegionList } from '../../../util'; import { i18n } from '@kbn/i18n'; export function CreateSourceEditor({ onSourceConfigChange }) { diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx index 907b80e6405a6e..9091e03fdf7f50 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx @@ -13,7 +13,7 @@ import { KibanaRegionmapSource, sourceTitle } from './kibana_regionmap_source'; import { VectorLayer } from '../../layers/vector_layer'; // @ts-ignore import { CreateSourceEditor } from './create_source_editor'; -import { getKibanaRegionList } from '../../../meta'; +import { getKibanaRegionList } from '../../../util'; import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const kibanaRegionMapLayerWizardConfig: LayerWizard = { diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_source.ts b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_source.ts index 0f778f194ce3f9..12e4b00c3c7b98 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_source.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; import { AbstractVectorSource, GeoJsonWithMeta } from '../vector_source'; -import { getKibanaRegionList } from '../../../meta'; +import { fetchGeoJson, getKibanaRegionList } from '../../../util'; import { getDataSourceLabel } from '../../../../common/i18n_getters'; import { FIELD_ORIGIN, FORMAT_TYPE, SOURCE_TYPES } from '../../../../common/constants'; import { KibanaRegionField } from '../../fields/kibana_region_field'; @@ -79,11 +79,12 @@ export class KibanaRegionmapSource extends AbstractVectorSource { async getGeoJsonWithMeta(): Promise { const vectorFileMeta = await this.getVectorFileMeta(); - const featureCollection = await AbstractVectorSource.getGeoJson({ - format: vectorFileMeta.format.type as FORMAT_TYPE, - featureCollectionPath: vectorFileMeta.meta.feature_collection_path, - fetchUrl: vectorFileMeta.url, - }); + const featureCollection = await fetchGeoJson( + vectorFileMeta.url, + vectorFileMeta.format.type as FORMAT_TYPE, + vectorFileMeta.meta.feature_collection_path + ); + return { data: featureCollection, meta: {}, diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/create_source_editor.js b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/create_source_editor.js index 8ec57d2b6f4fbb..4d6939a3b7d45c 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/create_source_editor.js +++ b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/create_source_editor.js @@ -9,7 +9,7 @@ import React, { useEffect } from 'react'; import PropTypes from 'prop-types'; import { EuiFieldText, EuiFormRow, EuiPanel } from '@elastic/eui'; -import { getKibanaTileMap } from '../../../meta'; +import { getKibanaTileMap } from '../../../util'; import { i18n } from '@kbn/i18n'; export function CreateSourceEditor({ onSourceConfigChange }) { diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx index 8d18cda4e70dda..26893086ba8f7e 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx @@ -13,7 +13,7 @@ import { CreateSourceEditor } from './create_source_editor'; // @ts-ignore import { KibanaTilemapSource, sourceTitle } from './kibana_tilemap_source'; import { TileLayer } from '../../layers/tile_layer/tile_layer'; -import { getKibanaTileMap } from '../../../meta'; +import { getKibanaTileMap } from '../../../util'; import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const kibanaBasemapLayerWizardConfig: LayerWizard = { diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_tilemap_source.js b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_tilemap_source.js index 0b88fe2e139056..94d082d8744e88 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_tilemap_source.js +++ b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_tilemap_source.js @@ -6,7 +6,7 @@ */ import { AbstractTMSSource } from '../tms_source'; -import { getKibanaTileMap } from '../../../meta'; +import { getKibanaTileMap } from '../../../util'; import { i18n } from '@kbn/i18n'; import { getDataSourceLabel } from '../../../../common/i18n_getters'; import _ from 'lodash'; diff --git a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx index 959d6994c73395..e86e459851c706 100644 --- a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx @@ -5,13 +5,9 @@ * 2.0. */ -// @ts-expect-error -import * as topojson from 'topojson-client'; -import _ from 'lodash'; -import { i18n } from '@kbn/i18n'; import { FeatureCollection, GeoJsonProperties } from 'geojson'; import { Filter, TimeRange } from 'src/plugins/data/public'; -import { FORMAT_TYPE, VECTOR_SHAPE_TYPE } from '../../../../common/constants'; +import { VECTOR_SHAPE_TYPE } from '../../../../common/constants'; import { TooltipProperty, ITooltipProperty } from '../../tooltips/tooltip_property'; import { AbstractSource, ISource } from '../source'; import { IField } from '../../fields/field'; @@ -85,48 +81,6 @@ export interface ITiledSingleLayerVectorSource extends IVectorSource { } export class AbstractVectorSource extends AbstractSource implements IVectorSource { - static async getGeoJson({ - format, - featureCollectionPath, - fetchUrl, - }: { - format: FORMAT_TYPE; - featureCollectionPath: string; - fetchUrl: string; - }) { - let fetchedJson; - try { - const response = await fetch(fetchUrl); - if (!response.ok) { - throw new Error('Request failed'); - } - fetchedJson = await response.json(); - } catch (e) { - throw new Error( - i18n.translate('xpack.maps.source.vetorSource.requestFailedErrorMessage', { - defaultMessage: `Unable to fetch vector shapes from url: {fetchUrl}`, - values: { fetchUrl }, - }) - ); - } - - if (format === FORMAT_TYPE.GEOJSON) { - return fetchedJson; - } - - if (format === FORMAT_TYPE.TOPOJSON) { - const features = _.get(fetchedJson, `objects.${featureCollectionPath}`); - return topojson.feature(fetchedJson, features); - } - - throw new Error( - i18n.translate('xpack.maps.source.vetorSource.formatErrorMessage', { - defaultMessage: `Unable to fetch vector shapes from url: {format}`, - values: { format }, - }) - ); - } - getFieldNames(): string[] { return []; } diff --git a/x-pack/plugins/maps/public/components/ems_file_select.tsx b/x-pack/plugins/maps/public/components/ems_file_select.tsx index 64ae57fc81dcf4..3d23854efb4fb1 100644 --- a/x-pack/plugins/maps/public/components/ems_file_select.tsx +++ b/x-pack/plugins/maps/public/components/ems_file_select.tsx @@ -10,7 +10,7 @@ import { EuiComboBox, EuiComboBoxOptionOption, EuiFormRow, EuiSelect } from '@el import { i18n } from '@kbn/i18n'; import { FileLayer } from '@elastic/ems-client'; -import { getEmsFileLayers } from '../meta'; +import { getEmsFileLayers } from '../util'; import { getEmsUnavailableMessage } from './ems_unavailable_message'; interface Props { diff --git a/x-pack/plugins/maps/public/components/geo_field_select.tsx b/x-pack/plugins/maps/public/components/geo_field_select.tsx new file mode 100644 index 00000000000000..0b04ec7146611d --- /dev/null +++ b/x-pack/plugins/maps/public/components/geo_field_select.tsx @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiFormRow } from '@elastic/eui'; +import { SingleFieldSelect } from './single_field_select'; +import { IFieldType } from '../../../../../src/plugins/data/common'; + +interface Props { + value: string; + geoFields: IFieldType[]; + onChange: (geoFieldName?: string) => void; +} + +export function GeoFieldSelect(props: Props) { + return ( + + + + ); +} diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/mb_map.tsx b/x-pack/plugins/maps/public/connected_components/mb_map/mb_map.tsx index 5e4c3c9b1981fa..66c9a2462736af 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/mb_map.tsx +++ b/x-pack/plugins/maps/public/connected_components/mb_map/mb_map.tsx @@ -33,7 +33,7 @@ import { RawValue, ZOOM_PRECISION, } from '../../../common/constants'; -import { getGlyphUrl, isRetina } from '../../meta'; +import { getGlyphUrl, isRetina } from '../../util'; import { syncLayerOrder } from './sort_layers'; // @ts-expect-error import { removeOrphanedSourcesAndLayers, addSpritesheetToMap } from './utils'; diff --git a/x-pack/plugins/maps/public/ems_autosuggest/ems_autosuggest.test.ts b/x-pack/plugins/maps/public/ems_autosuggest/ems_autosuggest.test.ts new file mode 100644 index 00000000000000..34a53be48a5cdf --- /dev/null +++ b/x-pack/plugins/maps/public/ems_autosuggest/ems_autosuggest.test.ts @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { suggestEMSTermJoinConfig } from './ems_autosuggest'; +import { FORMAT_TYPE } from '../../common'; +import { FeatureCollection } from 'geojson'; + +class MockFileLayer { + private readonly _url: string; + private readonly _id: string; + private readonly _fields: Array<{ id: string }>; + + constructor(url: string, fields: Array<{ id: string }>) { + this._url = url; + this._id = url; + this._fields = fields; + } + getDefaultFormatUrl() { + return this._url; + } + + getFields() { + return this._fields; + } + + getDefaultFormatType() { + return FORMAT_TYPE.GEOJSON; + } + + hasId(id: string) { + return id === this._id; + } +} + +jest.mock('../util', () => { + return { + async getEmsFileLayers() { + return [ + new MockFileLayer('world_countries', [{ id: 'iso2' }, { id: 'iso3' }]), + new MockFileLayer('zips', [{ id: 'zip' }]), + ]; + }, + async fetchGeoJson(url: string): Promise { + if (url === 'world_countries') { + return ({ + type: 'FeatureCollection', + features: [ + { properties: { iso2: 'CA', iso3: 'CAN' } }, + { properties: { iso2: 'US', iso3: 'USA' } }, + ], + } as unknown) as FeatureCollection; + } else if (url === 'zips') { + return ({ + type: 'FeatureCollection', + features: [{ properties: { zip: '40204' } }, { properties: { zip: '40205' } }], + } as unknown) as FeatureCollection; + } else { + throw new Error(`unrecognized mock url ${url}`); + } + }, + }; +}); + +describe('suggestEMSTermJoinConfig', () => { + test('no info provided', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({}); + expect(termJoinConfig).toBe(null); + }); + + describe('validate common column names', () => { + test('ecs region', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValuesColumnName: 'destination.geo.region_iso_code', + }); + expect(termJoinConfig).toEqual({ + layerId: 'administrative_regions_lvl2', + field: 'region_iso_code', + }); + }); + + test('ecs country', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValuesColumnName: 'country_iso_code', + }); + expect(termJoinConfig).toEqual({ + layerId: 'world_countries', + field: 'iso2', + }); + }); + + test('country', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValuesColumnName: 'Country_name', + }); + expect(termJoinConfig).toEqual({ + layerId: 'world_countries', + field: 'name', + }); + }); + + test('unknown name', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValuesColumnName: 'cntry', + }); + expect(termJoinConfig).toEqual(null); + }); + }); + + describe('validate well known formats', () => { + test('5-digit zip code', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValues: ['90201', 40204], + }); + expect(termJoinConfig).toEqual({ + layerId: 'usa_zip_codes', + field: 'zip', + }); + }); + + test('mismatch', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValues: ['90201', 'foobar'], + }); + expect(termJoinConfig).toEqual(null); + }); + }); + + describe('validate based on EMS data', () => { + test('Should validate with zip codes layer', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValues: ['40204', 40205], + emsLayerIds: ['world_countries', 'zips'], + }); + expect(termJoinConfig).toEqual({ + layerId: 'zips', + field: 'zip', + }); + }); + + test('Should not validate with faulty zip codes', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValues: ['40204', '00000'], + emsLayerIds: ['world_countries', 'zips'], + }); + expect(termJoinConfig).toEqual(null); + }); + + test('Should validate against countries', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValues: ['USA', 'USA', 'CAN'], + emsLayerIds: ['world_countries', 'zips'], + }); + expect(termJoinConfig).toEqual({ + layerId: 'world_countries', + field: 'iso3', + }); + }); + + test('Should not validate against missing countries', async () => { + const termJoinConfig = await suggestEMSTermJoinConfig({ + sampleValues: ['USA', 'BEL', 'CAN'], + emsLayerIds: ['world_countries', 'zips'], + }); + expect(termJoinConfig).toEqual(null); + }); + }); +}); diff --git a/x-pack/plugins/maps/public/ems_autosuggest/ems_autosuggest.ts b/x-pack/plugins/maps/public/ems_autosuggest/ems_autosuggest.ts new file mode 100644 index 00000000000000..1d5c1529a004ea --- /dev/null +++ b/x-pack/plugins/maps/public/ems_autosuggest/ems_autosuggest.ts @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FileLayer } from '@elastic/ems-client'; +import { getEmsFileLayers, fetchGeoJson } from '../util'; +import { FORMAT_TYPE, emsWorldLayerId, emsRegionLayerId, emsUsaZipLayerId } from '../../common'; + +export interface SampleValuesConfig { + emsLayerIds?: string[]; + sampleValues?: Array; + sampleValuesColumnName?: string; +} + +export interface EMSTermJoinConfig { + layerId: string; + field: string; +} + +const wellKnownColumnNames = [ + { + regex: /(geo\.){0,}country_iso_code$/i, // ECS postfix for country + emsConfig: { + layerId: emsWorldLayerId, + field: 'iso2', + }, + }, + { + regex: /(geo\.){0,}region_iso_code$/i, // ECS postfixn for region + emsConfig: { + layerId: emsRegionLayerId, + field: 'region_iso_code', + }, + }, + { + regex: /^country/i, // anything starting with country + emsConfig: { + layerId: emsWorldLayerId, + field: 'name', + }, + }, +]; + +const wellKnownColumnFormats = [ + { + regex: /(^\d{5}$)/i, // 5-digit zipcode + emsConfig: { + layerId: emsUsaZipLayerId, + field: 'zip', + }, + }, +]; + +interface UniqueMatch { + config: { layerId: string; field: string }; + count: number; +} + +export async function suggestEMSTermJoinConfig( + sampleValuesConfig: SampleValuesConfig +): Promise { + const matches: EMSTermJoinConfig[] = []; + + if (sampleValuesConfig.sampleValuesColumnName) { + matches.push(...suggestByName(sampleValuesConfig.sampleValuesColumnName)); + } + + if (sampleValuesConfig.sampleValues && sampleValuesConfig.sampleValues.length) { + if (sampleValuesConfig.emsLayerIds && sampleValuesConfig.emsLayerIds.length) { + matches.push( + ...(await suggestByEMSLayerIds( + sampleValuesConfig.emsLayerIds, + sampleValuesConfig.sampleValues + )) + ); + } else { + matches.push(...suggestByValues(sampleValuesConfig.sampleValues)); + } + } + + const uniqMatches: UniqueMatch[] = matches.reduce((accum: UniqueMatch[], match) => { + const found = accum.find((m) => { + return m.config.layerId === match.layerId && m.config.field === match.layerId; + }); + + if (found) { + found.count += 1; + } else { + accum.push({ + config: match, + count: 1, + }); + } + + return accum; + }, []); + + uniqMatches.sort((a, b) => { + return b.count - a.count; + }); + + return uniqMatches.length ? uniqMatches[0].config : null; +} + +function suggestByName(columnName: string): EMSTermJoinConfig[] { + const matches = wellKnownColumnNames.filter((wellknown) => { + return columnName.match(wellknown.regex); + }); + + return matches.map((m) => { + return m.emsConfig; + }); +} + +function suggestByValues(values: Array): EMSTermJoinConfig[] { + const matches = wellKnownColumnFormats.filter((wellknown) => { + for (let i = 0; i < values.length; i++) { + const value = values[i].toString(); + if (!value.match(wellknown.regex)) { + return false; + } + } + return true; + }); + + return matches.map((m) => { + return m.emsConfig; + }); +} + +function existsInEMS(emsJson: any, emsFieldId: string, sampleValue: string): boolean { + for (let i = 0; i < emsJson.features.length; i++) { + const emsFieldValue = emsJson.features[i].properties[emsFieldId].toString(); + if (emsFieldValue.toString() === sampleValue) { + return true; + } + } + return false; +} + +function matchesEmsField(emsJson: any, emsFieldId: string, sampleValues: Array) { + for (let j = 0; j < sampleValues.length; j++) { + const sampleValue = sampleValues[j].toString(); + if (!existsInEMS(emsJson, emsFieldId, sampleValue)) { + return false; + } + } + return true; +} + +async function getMatchesForEMSLayer( + emsLayerId: string, + sampleValues: Array +): Promise { + const fileLayers: FileLayer[] = await getEmsFileLayers(); + const emsFileLayer: FileLayer | undefined = fileLayers.find((fl: FileLayer) => + fl.hasId(emsLayerId) + ); + + if (!emsFileLayer) { + return []; + } + + const emsFields = emsFileLayer.getFields(); + const url = emsFileLayer.getDefaultFormatUrl(); + + try { + const emsJson = await fetchGeoJson( + url, + emsFileLayer.getDefaultFormatType() as FORMAT_TYPE, + 'data' + ); + const matches: EMSTermJoinConfig[] = []; + for (let f = 0; f < emsFields.length; f++) { + if (matchesEmsField(emsJson, emsFields[f].id, sampleValues)) { + matches.push({ + layerId: emsLayerId, + field: emsFields[f].id, + }); + } + } + return matches; + } catch (e) { + return []; + } +} + +async function suggestByEMSLayerIds( + emsLayerIds: string[], + values: Array +): Promise { + const matches = []; + for (const emsLayerId of emsLayerIds) { + const layerIdMathes = await getMatchesForEMSLayer(emsLayerId, values); + matches.push(...layerIdMathes); + } + return matches; +} diff --git a/x-pack/plugins/maps/public/ems_autosuggest/index.ts b/x-pack/plugins/maps/public/ems_autosuggest/index.ts new file mode 100644 index 00000000000000..86ed9e4fa70e12 --- /dev/null +++ b/x-pack/plugins/maps/public/ems_autosuggest/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './ems_autosuggest'; diff --git a/x-pack/plugins/maps/public/index_pattern_util.ts b/x-pack/plugins/maps/public/index_pattern_util.ts index f7894085b15ac0..3b1cb461c87793 100644 --- a/x-pack/plugins/maps/public/index_pattern_util.ts +++ b/x-pack/plugins/maps/public/index_pattern_util.ts @@ -56,6 +56,12 @@ export function getTermsFields(fields: IFieldType[]): IFieldType[] { }); } +export function getSortFields(fields: IFieldType[]): IFieldType[] { + return fields.filter((field) => { + return field.sortable && !indexPatterns.isNestedField(field); + }); +} + export function getAggregatableGeoFieldTypes(): string[] { const aggregatableFieldTypes = [ES_GEO_FIELD_TYPE.GEO_POINT]; if (getIsGoldPlus()) { diff --git a/x-pack/plugins/maps/public/lazy_load_bundle/index.ts b/x-pack/plugins/maps/public/lazy_load_bundle/index.ts index 0bf604a26544ba..3e5e2d54422d61 100644 --- a/x-pack/plugins/maps/public/lazy_load_bundle/index.ts +++ b/x-pack/plugins/maps/public/lazy_load_bundle/index.ts @@ -14,6 +14,7 @@ import { MapEmbeddableConfig, MapEmbeddableInput, MapEmbeddableOutput } from '.. import { SourceRegistryEntry } from '../classes/sources/source_registry'; import { LayerWizard } from '../classes/layers/layer_wizard_registry'; import type { CreateLayerDescriptorParams } from '../classes/sources/es_search_source'; +import type { EMSTermJoinConfig, SampleValuesConfig } from '../ems_autosuggest'; let loadModulesPromise: Promise; @@ -74,6 +75,7 @@ interface LazyLoadedMapModules { }) => LayerDescriptor | null; createBasemapLayerDescriptor: () => LayerDescriptor | null; createESSearchSourceLayerDescriptor: (params: CreateLayerDescriptorParams) => LayerDescriptor; + suggestEMSTermJoinConfig: (config: SampleValuesConfig) => Promise; } export async function lazyLoadMapModules(): Promise { @@ -94,6 +96,7 @@ export async function lazyLoadMapModules(): Promise { createRegionMapLayerDescriptor, createBasemapLayerDescriptor, createESSearchSourceLayerDescriptor, + suggestEMSTermJoinConfig, } = await import('./lazy'); resolve({ @@ -108,6 +111,7 @@ export async function lazyLoadMapModules(): Promise { createRegionMapLayerDescriptor, createBasemapLayerDescriptor, createESSearchSourceLayerDescriptor, + suggestEMSTermJoinConfig, }); }); return loadModulesPromise; diff --git a/x-pack/plugins/maps/public/lazy_load_bundle/lazy/index.ts b/x-pack/plugins/maps/public/lazy_load_bundle/lazy/index.ts index 85b58da0ab09a8..e7f5df49527b71 100644 --- a/x-pack/plugins/maps/public/lazy_load_bundle/lazy/index.ts +++ b/x-pack/plugins/maps/public/lazy_load_bundle/lazy/index.ts @@ -16,3 +16,4 @@ export { createTileMapLayerDescriptor } from '../../classes/layers/create_tile_m export { createRegionMapLayerDescriptor } from '../../classes/layers/create_region_map_layer_descriptor'; export { createBasemapLayerDescriptor } from '../../classes/layers/create_basemap_layer_descriptor'; export { createLayerDescriptor as createESSearchSourceLayerDescriptor } from '../../classes/sources/es_search_source'; +export { suggestEMSTermJoinConfig } from '../../ems_autosuggest'; diff --git a/x-pack/plugins/maps/public/plugin.ts b/x-pack/plugins/maps/public/plugin.ts index 7ddab6bf509ffc..ad8846bd48b608 100644 --- a/x-pack/plugins/maps/public/plugin.ts +++ b/x-pack/plugins/maps/public/plugin.ts @@ -47,8 +47,13 @@ import type { EmbeddableSetup, EmbeddableStart } from '../../../../src/plugins/e import { MapsXPackConfig, MapsConfigType } from '../config'; import { getAppTitle } from '../common/i18n_getters'; import { lazyLoadMapModules } from './lazy_load_bundle'; -import { MapsStartApi } from './api'; -import { createLayerDescriptors, registerLayerWizard, registerSource } from './api'; +import { + createLayerDescriptors, + registerLayerWizard, + registerSource, + MapsStartApi, + suggestEMSTermJoinConfig, +} from './api'; import type { SharePluginSetup, SharePluginStart } from '../../../../src/plugins/share/public'; import type { MapsEmsPluginSetup } from '../../../../src/plugins/maps_ems/public'; import type { DataPublicPluginStart } from '../../../../src/plugins/data/public'; @@ -177,6 +182,7 @@ export class MapsPlugin createLayerDescriptors, registerLayerWizard, registerSource, + suggestEMSTermJoinConfig, }; } } diff --git a/x-pack/plugins/maps/public/meta.test.js b/x-pack/plugins/maps/public/util.test.js similarity index 98% rename from x-pack/plugins/maps/public/meta.test.js rename to x-pack/plugins/maps/public/util.test.js index fc26bca48032f3..47c3d77180077d 100644 --- a/x-pack/plugins/maps/public/meta.test.js +++ b/x-pack/plugins/maps/public/util.test.js @@ -6,7 +6,7 @@ */ import { EMSClient } from '@elastic/ems-client'; -import { getEMSClient, getGlyphUrl } from './meta'; +import { getEMSClient, getGlyphUrl } from './util'; jest.mock('@elastic/ems-client'); diff --git a/x-pack/plugins/maps/public/meta.ts b/x-pack/plugins/maps/public/util.ts similarity index 73% rename from x-pack/plugins/maps/public/meta.ts rename to x-pack/plugins/maps/public/util.ts index 11dc0338462227..2745f9274f119f 100644 --- a/x-pack/plugins/maps/public/meta.ts +++ b/x-pack/plugins/maps/public/util.ts @@ -7,6 +7,10 @@ import { i18n } from '@kbn/i18n'; import { EMSClient, FileLayer, TMSService } from '@elastic/ems-client'; +import { FeatureCollection } from 'geojson'; +// @ts-expect-error +import * as topojson from 'topojson-client'; +import _ from 'lodash'; import fetch from 'node-fetch'; import { @@ -16,6 +20,7 @@ import { EMS_GLYPHS_PATH, EMS_APP_NAME, FONTS_API_PATH, + FORMAT_TYPE, } from '../common/constants'; import { getHttp, @@ -113,3 +118,41 @@ export function getGlyphUrl(): string { export function isRetina(): boolean { return window.devicePixelRatio === 2; } + +export async function fetchGeoJson( + fetchUrl: string, + format: FORMAT_TYPE, + featureCollectionPath: string +): Promise { + let fetchedJson; + try { + const response = await fetch(fetchUrl); + if (!response.ok) { + throw new Error('Request failed'); + } + fetchedJson = await response.json(); + } catch (e) { + throw new Error( + i18n.translate('xpack.maps.util.requestFailedErrorMessage', { + defaultMessage: `Unable to fetch vector shapes from url: {fetchUrl}`, + values: { fetchUrl }, + }) + ); + } + + if (format === FORMAT_TYPE.GEOJSON) { + return fetchedJson; + } + + if (format === FORMAT_TYPE.TOPOJSON) { + const features = _.get(fetchedJson, `objects.${featureCollectionPath}`); + return topojson.feature(fetchedJson, features); + } + + throw new Error( + i18n.translate('xpack.maps.util.formatErrorMessage', { + defaultMessage: `Unable to fetch vector shapes from url: {format}`, + values: { format }, + }) + ); +} diff --git a/x-pack/plugins/maps/server/sample_data/ecommerce_saved_objects.js b/x-pack/plugins/maps/server/sample_data/ecommerce_saved_objects.js index 268794b8a1bce2..6e68608c75cef5 100644 --- a/x-pack/plugins/maps/server/sample_data/ecommerce_saved_objects.js +++ b/x-pack/plugins/maps/server/sample_data/ecommerce_saved_objects.js @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import { emsWorldLayerId } from '../../common'; const layerList = [ { @@ -29,7 +30,7 @@ const layerList = [ alpha: 1, sourceDescriptor: { type: 'EMS_FILE', - id: 'world_countries', + id: emsWorldLayerId, tooltipProperties: ['name', 'iso2'], }, visible: true, diff --git a/x-pack/plugins/maps/server/sample_data/web_logs_saved_objects.js b/x-pack/plugins/maps/server/sample_data/web_logs_saved_objects.js index 31f353fab09ab9..86c6c14306faf4 100644 --- a/x-pack/plugins/maps/server/sample_data/web_logs_saved_objects.js +++ b/x-pack/plugins/maps/server/sample_data/web_logs_saved_objects.js @@ -6,6 +6,7 @@ */ import { i18n } from '@kbn/i18n'; +import { emsWorldLayerId } from '../../common'; const layerList = [ { @@ -29,7 +30,7 @@ const layerList = [ alpha: 0.5, sourceDescriptor: { type: 'EMS_FILE', - id: 'world_countries', + id: emsWorldLayerId, tooltipProperties: ['name', 'iso2'], }, visible: true, diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts index d5edd4678a9a22..b32d2a6542f4a4 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts @@ -41,6 +41,7 @@ describe('TelemetryEventsSender', () => { }, file: { size: 3, + created: 0, path: 'X', test: 'me', another: 'nope', @@ -66,6 +67,20 @@ describe('TelemetryEventsSender', () => { }, something_else: 'nope', }, + process: { + name: 'foo.exe', + nope: 'nope', + executable: null, // null fields are never allowlisted + }, + Target: { + process: { + name: 'bar.exe', + nope: 'nope', + thread: { + id: 1234, + }, + }, + }, }, ]; @@ -85,6 +100,7 @@ describe('TelemetryEventsSender', () => { }, file: { size: 3, + created: 0, path: 'X', Ext: { code_signature: { @@ -106,6 +122,17 @@ describe('TelemetryEventsSender', () => { name: 'windows', }, }, + process: { + name: 'foo.exe', + }, + Target: { + process: { + name: 'bar.exe', + thread: { + id: 1234, + }, + }, + }, }, ]); }); diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 114cf5d2d3425c..7d723c578e3d00 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -293,10 +293,46 @@ interface AllowlistFields { [key: string]: boolean | AllowlistFields; } +// Allow list process fields within events. This includes "process" and "Target.process".' +/* eslint-disable @typescript-eslint/naming-convention */ +const allowlistProcessFields: AllowlistFields = { + name: true, + executable: true, + command_line: true, + hash: true, + pid: true, + uptime: true, + Ext: { + architecture: true, + code_signature: true, + dll: true, + token: { + integrity_level_name: true, + }, + }, + parent: { + name: true, + executable: true, + command_line: true, + hash: true, + Ext: { + architecture: true, + code_signature: true, + dll: true, + token: { + integrity_level_name: true, + }, + }, + uptime: true, + pid: true, + ppid: true, + }, + thread: true, +}; + // Allow list for the data we include in the events. True means that it is deep-cloned // blindly. Object contents means that we only copy the fields that appear explicitly in // the sub-object. -/* eslint-disable @typescript-eslint/naming-convention */ const allowlistEventFields: AllowlistFields = { '@timestamp': true, agent: true, @@ -332,127 +368,9 @@ const allowlistEventFields: AllowlistFields = { host: { os: true, }, - process: { - name: true, - executable: true, - command_line: true, - hash: true, - pid: true, - uptime: true, - Ext: { - architecture: true, - code_signature: true, - dll: true, - token: { - integrity_level_name: true, - }, - }, - parent: { - name: true, - executable: true, - command_line: true, - hash: true, - Ext: { - architecture: true, - code_signature: true, - dll: true, - token: { - integrity_level_name: true, - }, - }, - uptime: true, - pid: true, - ppid: true, - }, - token: { - integrity_level_name: true, - }, - thread: true, - }, + process: allowlistProcessFields, Target: { - process: { - Ext: { - architecture: true, - code_signature: true, - dll: true, - token: { - integrity_level_name: true, - }, - }, - parent: { - process: { - Ext: { - architecture: true, - code_signature: true, - dll: true, - token: { - integrity_level_name: true, - }, - }, - }, - }, - thread: { - Ext: { - call_stack: true, - start_address: true, - start_address_allocation_offset: true, - start_address_bytes: true, - start_address_bytes_disasm: true, - start_address_bytes_disasm_hash: true, - start_address_details: { - allocation_base: true, - allocation_protection: true, - allocation_size: true, - allocation_type: true, - bytes_address: true, - bytes_allocation_offset: true, - bytes_compressed: true, - bytes_compressed_present: true, - mapped_pe: { - Ext: { - code_signature: { - status: true, - subject_name: true, - trusted: true, - }, - legal_copyright: true, - product_version: true, - }, - company: true, - description: true, - file_version: true, - imphash: true, - original_file_name: true, - product: true, - }, - mapped_pe_path: true, - memory_pe: { - Ext: { - code_signature: { - status: true, - subject_name: true, - trusted: true, - }, - legal_copyright: true, - product_version: true, - }, - company: true, - description: true, - file_version: true, - imphash: true, - original_file_name: true, - product: true, - }, - memory_pe_detected: true, - region_base: true, - region_protection: true, - region_size: true, - region_state: true, - strings: true, - }, - }, - }, - }, + process: allowlistProcessFields, }, }; @@ -462,7 +380,7 @@ export function copyAllowlistedFields( ): TelemetryEvent { return Object.entries(allowlist).reduce((newEvent, [allowKey, allowValue]) => { const eventValue = event[allowKey]; - if (eventValue) { + if (eventValue !== null && eventValue !== undefined) { if (allowValue === true) { return { ...newEvent, [allowKey]: eventValue }; } else if (typeof allowValue === 'object' && typeof eventValue === 'object') { diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx index 4ec510a5e69a79..830b9985f86fd6 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/policy_list/policy_table/policy_table.tsx @@ -382,7 +382,7 @@ export const PolicyTable: React.FunctionComponent = ({ > , ], diff --git a/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_table/repository_table.tsx b/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_table/repository_table.tsx index 45c62f7fc57c15..3e605ade5f3c33 100644 --- a/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_table/repository_table.tsx +++ b/x-pack/plugins/snapshot_restore/public/application/sections/home/repository_list/repository_table/repository_table.tsx @@ -261,7 +261,7 @@ export const RepositoryTable: React.FunctionComponent = ({ > , ], diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index d238d3077aadce..9602a324e5d51c 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -12762,7 +12762,6 @@ "xpack.maps.source.esSearch.topHitsSplitFieldLabel": "エンティティ", "xpack.maps.source.esSearch.topHitsSplitFieldSelectPlaceholder": "エンティティフィールドを選択", "xpack.maps.source.esSearch.useMVTVectorTiles": "ベクトルタイルを使用", - "xpack.maps.source.esSearch.useTopHitsLabel": "エンティティごとにトップヒットを表示。", "xpack.maps.source.esSearchDescription": "Elasticsearch の点、線、多角形", "xpack.maps.source.esSearchTitle": "ドキュメント", "xpack.maps.source.esSource.noGeoFieldErrorMessage": "インデックスパターン {indexPatternTitle} には現在ジオフィールド {geoField} が含まれていません", @@ -12806,8 +12805,6 @@ "xpack.maps.source.pewPewDescription": "ソースとデスティネーションの間の集約データパスです。", "xpack.maps.source.pewPewTitle": "ソースとデスティネーションの接続", "xpack.maps.source.urlLabel": "Url", - "xpack.maps.source.vetorSource.formatErrorMessage": "URL からベクターシェイプを取得できません:{format}", - "xpack.maps.source.vetorSource.requestFailedErrorMessage": "URL からベクターシェイプを取得できません:{fetchUrl}", "xpack.maps.source.wms.attributionLink": "属性テキストにはリンクが必要です", "xpack.maps.source.wms.attributionText": "属性 URL にはテキストが必要です", "xpack.maps.source.wms.getCapabilitiesButtonText": "負荷容量", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 58119a07398126..12a3c8925cfc62 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -12930,7 +12930,6 @@ "xpack.maps.source.esSearch.topHitsSplitFieldLabel": "实体", "xpack.maps.source.esSearch.topHitsSplitFieldSelectPlaceholder": "选择实体字段", "xpack.maps.source.esSearch.useMVTVectorTiles": "使用矢量磁贴", - "xpack.maps.source.esSearch.useTopHitsLabel": "显示每个实体最高命中结果。", "xpack.maps.source.esSearchDescription": "Elasticsearch 的点、线和多边形", "xpack.maps.source.esSearchTitle": "文档", "xpack.maps.source.esSource.noGeoFieldErrorMessage": "索引模式“{indexPatternTitle}”不再包含地理字段 {geoField}", @@ -12974,8 +12973,6 @@ "xpack.maps.source.pewPewDescription": "源和目标之间的聚合数据路径", "xpack.maps.source.pewPewTitle": "源-目标连接", "xpack.maps.source.urlLabel": "URL", - "xpack.maps.source.vetorSource.formatErrorMessage": "无法从以下 URL 获取矢量形状:{format}", - "xpack.maps.source.vetorSource.requestFailedErrorMessage": "无法从以下 URL 获取矢量形状:{fetchUrl}", "xpack.maps.source.wms.attributionLink": "属性文本必须附带链接", "xpack.maps.source.wms.attributionText": "属性 url 必须附带文本", "xpack.maps.source.wms.getCapabilitiesButtonText": "加载功能", diff --git a/x-pack/scripts/jest.js b/x-pack/scripts/jest.js index 4c83073a559a43..2ea950e075c8c2 100644 --- a/x-pack/scripts/jest.js +++ b/x-pack/scripts/jest.js @@ -5,4 +5,5 @@ * 2.0. */ +require('../../src/setup_node_env/ensure_node_preserve_symlinks'); require('@kbn/test').runJest(); diff --git a/x-pack/test/api_integration/apis/management/rollup/rollup.js b/x-pack/test/api_integration/apis/management/rollup/rollup.js index a556c8071ca80a..699592fd999202 100644 --- a/x-pack/test/api_integration/apis/management/rollup/rollup.js +++ b/x-pack/test/api_integration/apis/management/rollup/rollup.js @@ -24,8 +24,7 @@ export default function ({ getService }) { cleanUp, } = registerHelpers(getService); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/96002 - describe.skip('jobs', () => { + describe('jobs', () => { after(() => cleanUp()); describe('indices', () => { @@ -57,10 +56,16 @@ export default function ({ getService }) { expect(body.doesMatchIndices).to.be(true); expect(body.doesMatchRollupIndices).to.be(false); expect(body.dateFields).to.eql(['testCreatedField']); - expect(body.keywordFields).to.eql(['testTagField']); - - // Allowing the test to account for future addition of doc_count - expect(body.numericFields.indexOf('testTotalField')).to.be.greaterThan(-1); + // '_tier' is an expected metadata field from ES + // Order is not guaranteed, so we assert against individual field names + ['_tier', 'testTagField'].forEach((keywordField) => { + expect(body.keywordFields.includes(keywordField)).to.be(true); + }); + // '_doc_count' is an expected metadata field from ES + // Order is not guaranteed, so we assert against individual field names + ['_doc_count', 'testTotalField'].forEach((numericField) => { + expect(body.numericFields.includes(numericField)).to.be(true); + }); }); it("should not return any fields when the index pattern doesn't match any indices", async () => { diff --git a/x-pack/test/apm_api_integration/tests/correlations/errors_failed_transactions.ts b/x-pack/test/apm_api_integration/tests/correlations/errors_failed_transactions.ts new file mode 100644 index 00000000000000..80c2b982662480 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/errors_failed_transactions.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi'; +import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + const url = format({ + pathname: `/api/apm/correlations/errors/failed_transactions`, + query: { + start: range.start, + end: range.end, + fieldNames: 'user_agent.name,user_agent.os.name,url.original', + }, + }); + registry.when( + 'correlations errors failed transactions without data', + { config: 'trial', archives: [] }, + () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + } + ); + + registry.when( + 'correlations errors failed transactions with data and default args', + { config: 'trial', archives: ['apm_8.0.0'] }, + () => { + type ResponseBody = APIReturnType<'GET /api/apm/correlations/errors/failed_transactions'>; + let response: { + status: number; + body: NonNullable; + }; + + before(async () => { + response = await supertest.get(url); + }); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns significant terms', () => { + const { significantTerms } = response.body; + expect(significantTerms).to.have.length(2); + const sortedFieldNames = significantTerms.map(({ fieldName }) => fieldName).sort(); + expectSnapshot(sortedFieldNames).toMatchInline(` + Array [ + "user_agent.name", + "user_agent.name", + ] + `); + }); + + it('returns a distribution per term', () => { + const { significantTerms } = response.body; + expectSnapshot(significantTerms.map((term) => term.timeseries.length)).toMatchInline(` + Array [ + 31, + 31, + ] + `); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/errors_overall.ts b/x-pack/test/apm_api_integration/tests/correlations/errors_overall.ts new file mode 100644 index 00000000000000..206da2968b4c1a --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/errors_overall.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi'; +import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + const url = format({ + pathname: `/api/apm/correlations/errors/overall_timeseries`, + query: { + start: range.start, + end: range.end, + }, + }); + + registry.when( + 'correlations errors overall without data', + { config: 'trial', archives: [] }, + () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + } + ); + + registry.when( + 'correlations errors overall with data and default args', + { config: 'trial', archives: ['apm_8.0.0'] }, + () => { + type ResponseBody = APIReturnType<'GET /api/apm/correlations/errors/overall_timeseries'>; + let response: { + status: number; + body: NonNullable; + }; + + before(async () => { + response = await supertest.get(url); + }); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns overall distribution', () => { + expectSnapshot(response.body?.overall?.timeseries.length).toMatchInline(`31`); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency_overall.ts b/x-pack/test/apm_api_integration/tests/correlations/latency_overall.ts new file mode 100644 index 00000000000000..0d79333faa9ef1 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/latency_overall.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi'; +import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + const url = format({ + pathname: `/api/apm/correlations/latency/overall_distribution`, + query: { + start: range.start, + end: range.end, + }, + }); + + registry.when( + 'correlations latency overall without data', + { config: 'trial', archives: [] }, + () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + } + ); + + registry.when( + 'correlations latency overall with data and default args', + { config: 'trial', archives: ['apm_8.0.0'] }, + () => { + type ResponseBody = APIReturnType<'GET /api/apm/correlations/latency/overall_distribution'>; + let response: { + status: number; + body: NonNullable; + }; + + before(async () => { + response = await supertest.get(url); + }); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns overall distribution', () => { + expectSnapshot(response.body?.distributionInterval).toMatchInline(`238776`); + expectSnapshot(response.body?.maxLatency).toMatchInline(`3581640.00000003`); + expectSnapshot(response.body?.overallDistribution?.length).toMatchInline(`15`); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency_slow_transactions.ts b/x-pack/test/apm_api_integration/tests/correlations/latency_slow_transactions.ts new file mode 100644 index 00000000000000..d32beee0f31d53 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/correlations/latency_slow_transactions.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { format } from 'url'; +import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi'; +import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { registry } from '../../common/registry'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const archiveName = 'apm_8.0.0'; + const range = archives_metadata[archiveName]; + + const url = format({ + pathname: `/api/apm/correlations/latency/slow_transactions`, + query: { + start: range.start, + end: range.end, + durationPercentile: 95, + fieldNames: 'user_agent.name,user_agent.os.name,url.original', + maxLatency: 3581640.00000003, + distributionInterval: 238776, + }, + }); + registry.when( + 'correlations latency slow transactions without data', + { config: 'trial', archives: [] }, + () => { + it('handles the empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + expect(response.body.response).to.be(undefined); + }); + } + ); + + registry.when( + 'correlations latency slow transactions with data and default args', + { config: 'trial', archives: ['apm_8.0.0'] }, + () => { + type ResponseBody = APIReturnType<'GET /api/apm/correlations/latency/slow_transactions'>; + let response: { + status: number; + body: NonNullable; + }; + + before(async () => { + response = await supertest.get(url); + }); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); + + it('returns significant terms', () => { + const { significantTerms } = response.body; + expect(significantTerms).to.have.length(9); + const sortedFieldNames = significantTerms.map(({ fieldName }) => fieldName).sort(); + expectSnapshot(sortedFieldNames).toMatchInline(` + Array [ + "url.original", + "url.original", + "url.original", + "url.original", + "user_agent.name", + "user_agent.name", + "user_agent.name", + "user_agent.os.name", + "user_agent.os.name", + ] + `); + }); + + it('returns a distribution per term', () => { + const { significantTerms } = response.body; + expectSnapshot(significantTerms.map((term) => term.distribution.length)).toMatchInline(` + Array [ + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + 15, + ] + `); + }); + } + ); +} diff --git a/x-pack/test/apm_api_integration/tests/correlations/slow_transactions.ts b/x-pack/test/apm_api_integration/tests/correlations/slow_transactions.ts deleted file mode 100644 index c9686a8a9d5b0b..00000000000000 --- a/x-pack/test/apm_api_integration/tests/correlations/slow_transactions.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import { format } from 'url'; -import { APIReturnType } from '../../../../plugins/apm/public/services/rest/createCallApmApi'; -import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { registry } from '../../common/registry'; - -export default function ApiTest({ getService }: FtrProviderContext) { - const supertest = getService('supertest'); - const archiveName = 'apm_8.0.0'; - const range = archives_metadata[archiveName]; - - const url = format({ - pathname: `/api/apm/correlations/slow_transactions`, - query: { - start: range.start, - end: range.end, - durationPercentile: 95, - fieldNames: 'user_agent.name,user_agent.os.name,url.original', - }, - }); - - registry.when('without data', { config: 'trial', archives: [] }, () => { - it('handles the empty state', async () => { - const response = await supertest.get(url); - - expect(response.status).to.be(200); - expect(response.body.response).to.be(undefined); - }); - }); - - registry.when('with data and default args', { config: 'trial', archives: ['apm_8.0.0'] }, () => { - type ResponseBody = APIReturnType<'GET /api/apm/correlations/slow_transactions'>; - let response: { - status: number; - body: NonNullable; - }; - - before(async () => { - response = await supertest.get(url); - }); - - it('returns successfully', () => { - expect(response.status).to.eql(200); - }); - - it('returns significant terms', () => { - const significantTerms = response.body?.significantTerms as NonNullable< - typeof response.body.significantTerms - >; - expect(significantTerms).to.have.length(9); - const sortedFieldNames = significantTerms.map(({ fieldName }) => fieldName).sort(); - expectSnapshot(sortedFieldNames).toMatchInline(` - Array [ - "url.original", - "url.original", - "url.original", - "url.original", - "user_agent.name", - "user_agent.name", - "user_agent.name", - "user_agent.os.name", - "user_agent.os.name", - ] - `); - }); - - it('returns a distribution per term', () => { - expectSnapshot(response.body?.significantTerms?.map((term) => term.distribution.length)) - .toMatchInline(` - Array [ - 15, - 15, - 15, - 15, - 15, - 15, - 15, - 15, - 15, - ] - `); - }); - - it('returns overall distribution', () => { - expectSnapshot(response.body?.overall?.distribution.length).toMatchInline(`15`); - }); - }); -} diff --git a/x-pack/test/apm_api_integration/tests/index.ts b/x-pack/test/apm_api_integration/tests/index.ts index 9f0f1b15c05802..7c69d5b996cea7 100644 --- a/x-pack/test/apm_api_integration/tests/index.ts +++ b/x-pack/test/apm_api_integration/tests/index.ts @@ -24,8 +24,20 @@ export default function apmApiIntegrationTests(providerContext: FtrProviderConte loadTestFile(require.resolve('./alerts/chart_preview')); }); - describe('correlations/slow_transactions', function () { - loadTestFile(require.resolve('./correlations/slow_transactions')); + describe('correlations/latency_slow_transactions', function () { + loadTestFile(require.resolve('./correlations/latency_slow_transactions')); + }); + + describe('correlations/latency_overall', function () { + loadTestFile(require.resolve('./correlations/latency_overall')); + }); + + describe('correlations/errors_overall', function () { + loadTestFile(require.resolve('./correlations/errors_overall')); + }); + + describe('correlations/errors_failed_transactions', function () { + loadTestFile(require.resolve('./correlations/errors_failed_transactions')); }); describe('metrics_charts/metrics_charts', function () { diff --git a/x-pack/test/functional/apps/saved_objects_management/exports/_7.12_import_saved_objects.ndjson b/x-pack/test/functional/apps/saved_objects_management/exports/_7.12_import_saved_objects.ndjson new file mode 100644 index 00000000000000..5fe0c303668db2 --- /dev/null +++ b/x-pack/test/functional/apps/saved_objects_management/exports/_7.12_import_saved_objects.ndjson @@ -0,0 +1,34 @@ +{"attributes":{"fieldAttrs":"{\"machine.os\":{\"count\":1},\"spaces\":{\"count\":1},\"type\":{\"count\":1},\"bytes_scripted\":{\"count\":1}}","fields":"[{\"count\":1,\"script\":\"doc['bytes'].value*1024\",\"lang\":\"painless\",\"name\":\"bytes_scripted\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"logstash-*"},"coreMigrationVersion":"7.12.1","id":"56b34100-619d-11eb-aebf-c306684b328d","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1617218924557,0],"type":"index-pattern","updated_at":"2021-03-31T19:28:44.557Z","version":"WzksMV0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_scriptedfieldviz","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"logstash_scriptedfieldviz\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"isDisplayWarning\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":true,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"meter\",\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"range\",\"schema\":\"group\",\"params\":{\"field\":\"bytes_scripted\",\"ranges\":[{\"from\":0,\"to\":40000},{\"from\":40001,\"to\":20000000}]}}]}"},"coreMigrationVersion":"7.12.1","id":"0a274320-61cc-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218952314,184],"type":"visualization","updated_at":"2021-03-31T19:29:12.314Z","version":"WzY3LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_datatable","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"logstash_datatable\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":true,\"totalFunc\":\"sum\",\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"bucket\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"2015-07-24T08:58:14.175Z\",\"to\":\"2015-11-11T13:28:17.223Z\",\"mode\":\"absolute\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"response.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"0d8a8860-623a-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218938927,33],"type":"visualization","updated_at":"2021-03-31T19:28:58.927Z","version":"WzM5LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_area_chart","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_area_chart\",\"type\":\"area\",\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"zero\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"2010-01-28T19:25:55.242Z\",\"to\":\"2021-01-28T19:40:55.242Z\",\"mode\":\"absolute\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"machine.os.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"machine OS\"}}]}"},"coreMigrationVersion":"7.12.1","id":"36b91810-6239-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218930707,21],"type":"visualization","updated_at":"2021-03-31T19:28:50.707Z","version":"WzIzLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_horizontal","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_horizontal\",\"type\":\"horizontal_bar\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":200},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":75,\"filter\":true,\"truncate\":100},\"title\":{\"text\":\"no of documents\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"normal\",\"data\":{\"label\":\"no of documents\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":true,\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"no of documents\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"2015-07-24T08:58:14.175Z\",\"to\":\"2015-11-11T13:28:17.223Z\",\"mode\":\"absolute\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"agent.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"extension.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"e4aef350-623d-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218932758,19],"type":"visualization","updated_at":"2021-03-31T19:28:52.758Z","version":"WzI3LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_linechart","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_linechart\",\"type\":\"line\",\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"radiusRatio\":51,\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"zero\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"2015-09-18T06:38:43.311Z\",\"to\":\"2015-09-26T04:02:51.104Z\",\"mode\":\"absolute\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"radius\",\"params\":{\"field\":\"bytes\",\"customLabel\":\"bubbles\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"machine.os.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"f92e5630-623e-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218933787,95],"type":"visualization","updated_at":"2021-03-31T19:28:53.787Z","version":"WzI5LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_heatmap","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0% - 25%\":\"rgb(255,255,204)\",\"25% - 50%\":\"rgb(254,217,118)\",\"50% - 75%\":\"rgb(253,141,60)\",\"75% - 100%\":\"rgb(227,27,28)\"}}}","version":1,"visState":"{\"title\":\"logstash_heatmap\",\"type\":\"heatmap\",\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Yellow to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":false,\"percentageMode\":true,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"#555\"}}],\"row\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"2015-07-24T08:58:14.175Z\",\"to\":\"2015-11-11T13:28:17.223Z\",\"mode\":\"absolute\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"machine.os.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"response.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"9853d4d0-623d-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218934821,97],"type":"visualization","updated_at":"2021-03-31T19:28:54.821Z","version":"WzMxLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_goalchart","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 33\":\"rgb(0,104,55)\",\"33 - 67\":\"rgb(255,255,190)\",\"67 - 100\":\"rgb(165,0,38)\"}}}","version":1,"visState":"{\"title\":\"logstash_goalchart\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"isDisplayWarning\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":true,\"gaugeType\":\"Circle\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000},{\"from\":10001,\"to\":20000},{\"from\":20001,\"to\":30000}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"meter\",\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60},\"minAngle\":0,\"maxAngle\":6.283185307179586}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"group\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"2015-07-24T08:58:14.175Z\",\"to\":\"2015-11-11T13:28:17.223Z\",\"mode\":\"absolute\"},\"useNormalizedEsInterval\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}"},"coreMigrationVersion":"7.12.1","id":"6ecb33b0-623d-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218935846,99],"type":"visualization","updated_at":"2021-03-31T19:28:55.846Z","version":"WzMzLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_gauge","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 50\":\"rgb(0,104,55)\",\"50 - 75\":\"rgb(255,255,190)\",\"75 - 100\":\"rgb(165,0,38)\"}}}","version":1,"visState":"{\"title\":\"logstash_gauge\",\"type\":\"gauge\",\"params\":{\"type\":\"gauge\",\"addTooltip\":true,\"addLegend\":true,\"isDisplayWarning\":false,\"gauge\":{\"extendRange\":true,\"percentageMode\":false,\"gaugeType\":\"Arc\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Labels\",\"colorsRange\":[{\"from\":0,\"to\":50},{\"from\":50,\"to\":75},{\"from\":75,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":true,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\"},\"type\":\"meter\",\"style\":{\"bgWidth\":0.9,\"width\":0.9,\"mask\":false,\"bgMask\":false,\"maskBars\":50,\"bgFill\":\"#eee\",\"bgColor\":false,\"subText\":\"\",\"fontSize\":60,\"labelColor\":true},\"alignment\":\"horizontal\"}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"range\",\"schema\":\"group\",\"params\":{\"field\":\"bytes\",\"ranges\":[{\"from\":0,\"to\":10001},{\"from\":10002,\"to\":1000000}],\"json\":\"\"}}]}"},"coreMigrationVersion":"7.12.1","id":"b8e35c80-623c-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218936874,101],"type":"visualization","updated_at":"2021-03-31T19:28:56.874Z","version":"WzM1LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_coordinatemaps","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_coordinatemaps\",\"type\":\"tile_map\",\"params\":{\"colorSchema\":\"Yellow to Red\",\"mapType\":\"Scaled Circle Markers\",\"isDesaturated\":false,\"addTooltip\":true,\"heatClusterSize\":1.5,\"legendPosition\":\"bottomright\",\"mapZoom\":2,\"mapCenter\":[0,0],\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"selectedTmsLayer\":{\"origin\":\"elastic_maps_service\",\"id\":\"road_map\",\"minZoom\":0,\"maxZoom\":18,\"attribution\":\"

© OpenStreetMap contributors|OpenMapTiles|Elastic Maps Service

\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"geohash_grid\",\"schema\":\"segment\",\"params\":{\"field\":\"geo.coordinates\",\"autoPrecision\":true,\"isFilteredByCollar\":true,\"useGeocentroid\":true,\"mapZoom\":2,\"mapCenter\":[0,0],\"precision\":2,\"customLabel\":\"logstash src/dest\"}}]}"},"coreMigrationVersion":"7.12.1","id":"f1bc75d0-6239-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218937901,31],"type":"visualization","updated_at":"2021-03-31T19:28:57.901Z","version":"WzM3LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"logstash_inputcontrols","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_inputcontrols\",\"type\":\"input_control_vis\",\"params\":{\"controls\":[{\"id\":\"1611928563867\",\"fieldName\":\"machine.ram\",\"parent\":\"\",\"label\":\"Logstash RAM\",\"type\":\"range\",\"options\":{\"decimalPlaces\":0,\"step\":1024},\"indexPatternRefName\":\"control_0_index_pattern\"},{\"id\":\"1611928586274\",\"fieldName\":\"machine.os.raw\",\"parent\":\"\",\"label\":\"Logstash OS\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_1_index_pattern\"}],\"updateFiltersOnChange\":false,\"useTimeFilter\":false,\"pinFilters\":false},\"aggs\":[]}"},"coreMigrationVersion":"7.12.1","id":"d79fe3d0-6239-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"control_0_index_pattern","type":"index-pattern"},{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"control_1_index_pattern","type":"index-pattern"}],"sort":[1617218939955,36],"type":"visualization","updated_at":"2021-03-31T19:28:59.955Z","version":"WzQxLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"logstash_markdown","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_markdown\",\"type\":\"markdown\",\"params\":{\"fontSize\":12,\"openLinksInNewTab\":true,\"markdown\":\"Kibana is built with JS https://www.javascript.com/\"},\"aggs\":[]}"},"coreMigrationVersion":"7.12.1","id":"318375a0-6240-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[],"sort":[1617218940976,37],"type":"visualization","updated_at":"2021-03-31T19:29:00.976Z","version":"WzQzLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"logstash_vegaviz","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_vegaviz\",\"type\":\"vega\",\"params\":{\"spec\":\"{\\n/*\\n\\nWelcome to Vega visualizations. Here you can design your own dataviz from scratch using a declarative language called Vega, or its simpler form Vega-Lite. In Vega, you have the full control of what data is loaded, even from multiple sources, how that data is transformed, and what visual elements are used to show it. Use help icon to view Vega examples, tutorials, and other docs. Use the wrench icon to reformat this text, or to remove comments.\\n\\nThis example graph shows the document count in all indexes in the current time range. You might need to adjust the time filter in the upper right corner.\\n*/\\n\\n $schema: https://vega.github.io/schema/vega-lite/v2.json\\n title: Event counts from all indexes\\n\\n // Define the data source\\n data: {\\n url: {\\n/*\\nAn object instead of a string for the \\\"url\\\" param is treated as an Elasticsearch query. Anything inside this object is not part of the Vega language, but only understood by Kibana and Elasticsearch server. This query counts the number of documents per time interval, assuming you have a @timestamp field in your data.\\n\\nKibana has a special handling for the fields surrounded by \\\"%\\\". They are processed before the the query is sent to Elasticsearch. This way the query becomes context aware, and can use the time range and the dashboard filters.\\n*/\\n\\n // Apply dashboard context filters when set\\n %context%: true\\n // Filter the time picker (upper right corner) with this field\\n %timefield%: @timestamp\\n\\n/*\\nSee .search() documentation for : https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#api-search\\n*/\\n\\n // Which index to search\\n index: logstash-*\\n // Aggregate data by the time field into time buckets, counting the number of documents in each bucket.\\n body: {\\n aggs: {\\n time_buckets: {\\n date_histogram: {\\n // Use date histogram aggregation on @timestamp field\\n field: @timestamp\\n // The interval value will depend on the daterange picker (true), or use an integer to set an approximate bucket count\\n interval: {%autointerval%: true}\\n // Make sure we get an entire range, even if it has no data\\n extended_bounds: {\\n // Use the current time range's start and end\\n min: {%timefilter%: \\\"min\\\"}\\n max: {%timefilter%: \\\"max\\\"}\\n }\\n // Use this for linear (e.g. line, area) graphs. Without it, empty buckets will not show up\\n min_doc_count: 13\\n }\\n }\\n }\\n // Speed up the response by only including aggregation results\\n size: 0\\n }\\n }\\n/*\\nElasticsearch will return results in this format:\\n\\naggregations: {\\n time_buckets: {\\n buckets: [\\n {\\n key_as_string: 2015-11-30T22:00:00.000Z\\n key: 1448920800000\\n doc_count: 0\\n },\\n {\\n key_as_string: 2015-11-30T23:00:00.000Z\\n key: 1448924400000\\n doc_count: 0\\n }\\n ...\\n ]\\n }\\n}\\n\\nFor our graph, we only need the list of bucket values. Use the format.property to discard everything else.\\n*/\\n format: {property: \\\"aggregations.time_buckets.buckets\\\"}\\n }\\n\\n // \\\"mark\\\" is the graphics element used to show our data. Other mark values are: area, bar, circle, line, point, rect, rule, square, text, and tick. See https://vega.github.io/vega-lite/docs/mark.html\\n mark: line\\n\\n // \\\"encoding\\\" tells the \\\"mark\\\" what data to use and in what way. See https://vega.github.io/vega-lite/docs/encoding.html\\n encoding: {\\n x: {\\n // The \\\"key\\\" value is the timestamp in milliseconds. Use it for X axis.\\n field: key\\n type: temporal\\n axis: {title: false} // Customize X axis format\\n }\\n y: {\\n // The \\\"doc_count\\\" is the count per bucket. Use it for Y axis.\\n field: doc_count\\n type: quantitative\\n axis: {title: \\\"Document count\\\"}\\n }\\n }\\n}\\n\"},\"aggs\":[]}"},"coreMigrationVersion":"7.12.1","id":"e461eb20-6245-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[],"sort":[1617218942061,29],"type":"visualization","updated_at":"2021-03-31T19:29:02.061Z","version":"WzQ1LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_regionmap","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_regionmap\",\"type\":\"region_map\",\"params\":{\"addTooltip\":true,\"colorSchema\":\"Yellow to Red\",\"emsHotLink\":\"https://maps.elastic.co/v6.7?locale=en#file/world_countries\",\"isDisplayWarning\":true,\"legendPosition\":\"bottomright\",\"mapCenter\":[0,0],\"mapZoom\":2,\"outlineWeight\":1,\"selectedJoinField\":{\"type\":\"id\",\"name\":\"iso2\",\"description\":\"ISO 3166-1 alpha-2 code\"},\"showAllShapes\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"selectedTmsLayer\":{\"origin\":\"elastic_maps_service\",\"id\":\"road_map\",\"minZoom\":0,\"maxZoom\":18,\"attribution\":\"

© OpenStreetMap contributors|OpenMapTiles|Elastic Maps Service

\"}},\"selectedLayer\":{\"name\":\"World Countries\",\"origin\":\"elastic_maps_service\",\"id\":\"world_countries\",\"created_at\":\"2017-04-26T17:12:15.978370\",\"attribution\":\"Made with NaturalEarth | Elastic Maps Service\",\"fields\":[{\"type\":\"id\",\"name\":\"iso2\",\"description\":\"ISO 3166-1 alpha-2 code\"},{\"type\":\"id\",\"name\":\"iso3\",\"description\":\"ISO 3166-1 alpha-3 code\"},{\"type\":\"property\",\"name\":\"name\",\"description\":\"name\"}],\"format\":{\"type\":\"geojson\"},\"layerId\":\"elastic_maps_service.World Countries\",\"isEMS\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"geo.dest\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"25bdc750-6242-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218943039,44],"type":"visualization","updated_at":"2021-03-31T19:29:03.039Z","version":"WzQ3LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_verticalbarchart","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_verticalbarchart\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"defaultYExtents\":true},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":true,\"row\":true,\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"2015-09-18T06:38:43.311Z\",\"to\":\"2015-09-26T04:02:51.104Z\",\"mode\":\"absolute\"},\"useNormalizedEsInterval\":true,\"interval\":\"h\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"scaleMetricValues\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"response.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Response code\"}}]}"},"coreMigrationVersion":"7.12.1","id":"71dd7bc0-6248-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218944094,47],"type":"visualization","updated_at":"2021-03-31T19:29:04.094Z","version":"WzUxLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_metricviz","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_metricviz\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"range\",\"schema\":\"group\",\"params\":{\"field\":\"bytes_scripted\",\"ranges\":[{\"from\":0,\"to\":10000},{\"from\":10001,\"to\":300000}]}}]}"},"coreMigrationVersion":"7.12.1","id":"6aea48a0-6240-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218946147,51],"type":"visualization","updated_at":"2021-03-31T19:29:06.147Z","version":"WzU1LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_piechart","uiStateJSON":"{}","version":1,"visState":"{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"field\":\"machine.os.raw\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":5},\"schema\":\"segment\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":true,\"labels\":{\"last_level\":true,\"show\":false,\"truncate\":100,\"values\":true},\"legendPosition\":\"right\",\"type\":\"pie\"},\"title\":\"logstash_piechart\",\"type\":\"pie\"}"},"coreMigrationVersion":"7.12.1","id":"32b681f0-6241-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218947175,107],"type":"visualization","updated_at":"2021-03-31T19:29:07.175Z","version":"WzU3LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"logstash_tagcloud","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_tagcloud\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"log\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"geo.srcdest\",\"size\":23,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"ccca99e0-6244-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218948213,111],"type":"visualization","updated_at":"2021-03-31T19:29:08.213Z","version":"WzU5LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"logstash_timelion","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_timelion\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(q='machine.os.raw:win xp' , index=logstash-*)\",\"interval\":\"auto\"},\"aggs\":[]}"},"coreMigrationVersion":"7.12.1","id":"a4d7be80-6245-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[],"sort":[1617218949236,113],"type":"visualization","updated_at":"2021-03-31T19:29:09.236Z","version":"WzYxLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{}"},"title":"logstash_tsvb","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"logstash_tsvb\",\"type\":\"metrics\",\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"count\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"split_color_mode\":\"gradient\"}],\"time_field\":\"@timestamp\",\"index_pattern\":\"\",\"interval\":\"auto\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"default_index_pattern\":\"logstash-*\",\"annotations\":[{\"fields\":\"machine.os.raw\",\"template\":\"{{machine.os.raw}}\",\"index_pattern\":\"logstash-*\",\"query_string\":{\"query\":\"machine.os.raw :\\\"win xp\\\" \",\"language\":\"lucene\"},\"id\":\"aa43ceb0-6248-11eb-9a82-ef1c6e6c0265\",\"color\":\"#F00\",\"time_field\":\"@timestamp\",\"icon\":\"fa-tag\",\"ignore_global_filters\":1,\"ignore_panel_filters\":1}]},\"aggs\":[]}"},"coreMigrationVersion":"7.12.1","id":"c94d8440-6248-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[],"sort":[1617218951289,112],"type":"visualization","updated_at":"2021-03-31T19:29:11.289Z","version":"WzY1LDFd"} +{"attributes":{"columns":["bytes_scripted"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"machine.os.raw :\\\"win xp\\\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"logstash_scripted_saved_search","version":1},"coreMigrationVersion":"7.12.1","id":"db6226f0-61c0-11eb-aebf-c306684b328d","migrationVersion":{"search":"7.9.3"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218928794,16],"type":"search","updated_at":"2021-03-31T19:28:48.794Z","version":"WzE5LDFd"} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"1\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"2\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"4\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"5\",\"w\":24,\"x\":0,\"y\":30},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"6\",\"w\":24,\"x\":24,\"y\":30},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"7\",\"w\":24,\"x\":0,\"y\":45},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"8\",\"w\":24,\"x\":24,\"y\":45},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"9\",\"w\":24,\"x\":0,\"y\":60},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"10\",\"w\":24,\"x\":24,\"y\":60},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"11\",\"w\":24,\"x\":0,\"y\":75},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"12\",\"w\":24,\"x\":24,\"y\":75},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"13\",\"w\":24,\"x\":0,\"y\":90},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_12\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"14\",\"w\":24,\"x\":24,\"y\":90},\"panelIndex\":\"14\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_13\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"15\",\"w\":24,\"x\":0,\"y\":105},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_14\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"16\",\"w\":24,\"x\":24,\"y\":105},\"panelIndex\":\"16\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_15\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"17\",\"w\":24,\"x\":0,\"y\":120},\"panelIndex\":\"17\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_16\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"18\",\"w\":24,\"x\":24,\"y\":120},\"panelIndex\":\"18\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_17\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"19\",\"w\":24,\"x\":0,\"y\":135},\"panelIndex\":\"19\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_18\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"20\",\"w\":24,\"x\":24,\"y\":135},\"panelIndex\":\"20\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_19\"}]","timeRestore":false,"title":"logstash_dashboardwithtime","version":1},"coreMigrationVersion":"7.12.1","id":"154944b0-6249-11eb-aebf-c306684b328d","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"36b91810-6239-11eb-aebf-c306684b328d","name":"panel_0","type":"visualization"},{"id":"0a274320-61cc-11eb-aebf-c306684b328d","name":"panel_1","type":"visualization"},{"id":"e4aef350-623d-11eb-aebf-c306684b328d","name":"panel_2","type":"visualization"},{"id":"f92e5630-623e-11eb-aebf-c306684b328d","name":"panel_3","type":"visualization"},{"id":"9853d4d0-623d-11eb-aebf-c306684b328d","name":"panel_4","type":"visualization"},{"id":"6ecb33b0-623d-11eb-aebf-c306684b328d","name":"panel_5","type":"visualization"},{"id":"b8e35c80-623c-11eb-aebf-c306684b328d","name":"panel_6","type":"visualization"},{"id":"f1bc75d0-6239-11eb-aebf-c306684b328d","name":"panel_7","type":"visualization"},{"id":"0d8a8860-623a-11eb-aebf-c306684b328d","name":"panel_8","type":"visualization"},{"id":"d79fe3d0-6239-11eb-aebf-c306684b328d","name":"panel_9","type":"visualization"},{"id":"318375a0-6240-11eb-aebf-c306684b328d","name":"panel_10","type":"visualization"},{"id":"e461eb20-6245-11eb-aebf-c306684b328d","name":"panel_11","type":"visualization"},{"id":"25bdc750-6242-11eb-aebf-c306684b328d","name":"panel_12","type":"visualization"},{"id":"71dd7bc0-6248-11eb-aebf-c306684b328d","name":"panel_13","type":"visualization"},{"id":"6aea48a0-6240-11eb-aebf-c306684b328d","name":"panel_14","type":"visualization"},{"id":"32b681f0-6241-11eb-aebf-c306684b328d","name":"panel_15","type":"visualization"},{"id":"ccca99e0-6244-11eb-aebf-c306684b328d","name":"panel_16","type":"visualization"},{"id":"a4d7be80-6245-11eb-aebf-c306684b328d","name":"panel_17","type":"visualization"},{"id":"c94d8440-6248-11eb-aebf-c306684b328d","name":"panel_18","type":"visualization"},{"id":"db6226f0-61c0-11eb-aebf-c306684b328d","name":"panel_19","type":"search"}],"sort":[1617218953348,182],"type":"dashboard","updated_at":"2021-03-31T19:29:13.348Z","version":"WzY5LDFd"} +{"attributes":{"fieldAttrs":"{\"speaker\":{\"count\":1},\"text_entry\":{\"count\":6},\"type\":{\"count\":3}}","fields":"[]","runtimeFieldMap":"{}","title":"shakespeare"},"coreMigrationVersion":"7.12.1","id":"4e937b20-619d-11eb-aebf-c306684b328d","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1617218924067,3],"type":"index-pattern","updated_at":"2021-03-31T19:28:44.067Z","version":"WzcsMV0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"shakespeare_areachart","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"shakespeare_areachart\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"stacked\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"data\":{\"id\":\"2\",\"label\":\"Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true},\"aggs\":[{\"id\":\"2\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"play_name\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"2\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"play_name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"2\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"185283c0-619e-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"4e937b20-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218945128,49],"type":"visualization","updated_at":"2021-03-31T19:29:05.128Z","version":"WzUzLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"shakespeare_piechart","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"shakespeare_piechart\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":false,\"labels\":{\"show\":true,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"play_name\",\"size\":15,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"33736660-619e-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"4e937b20-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218950263,109],"type":"visualization","updated_at":"2021-03-31T19:29:10.263Z","version":"WzYzLDFd"} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"1\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"2\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"4\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"5\",\"w\":24,\"x\":0,\"y\":30},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"6\",\"w\":24,\"x\":24,\"y\":30},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"7\",\"w\":24,\"x\":0,\"y\":45},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"8\",\"w\":24,\"x\":24,\"y\":45},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"9\",\"w\":24,\"x\":0,\"y\":60},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"10\",\"w\":24,\"x\":24,\"y\":60},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"11\",\"w\":24,\"x\":0,\"y\":75},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"12\",\"w\":24,\"x\":24,\"y\":75},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"13\",\"w\":24,\"x\":0,\"y\":90},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_12\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"14\",\"w\":24,\"x\":24,\"y\":90},\"panelIndex\":\"14\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_13\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"15\",\"w\":24,\"x\":0,\"y\":105},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_14\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"16\",\"w\":24,\"x\":24,\"y\":105},\"panelIndex\":\"16\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_15\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"17\",\"w\":24,\"x\":0,\"y\":120},\"panelIndex\":\"17\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_16\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"18\",\"w\":24,\"x\":24,\"y\":120},\"panelIndex\":\"18\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_17\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"19\",\"w\":24,\"x\":0,\"y\":135},\"panelIndex\":\"19\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_18\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"20\",\"w\":24,\"x\":24,\"y\":135},\"panelIndex\":\"20\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_19\"}]","timeRestore":false,"title":"logstash_dashboard_withouttime","version":1},"coreMigrationVersion":"7.12.1","id":"5d3410c0-6249-11eb-aebf-c306684b328d","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"36b91810-6239-11eb-aebf-c306684b328d","name":"panel_0","type":"visualization"},{"id":"0a274320-61cc-11eb-aebf-c306684b328d","name":"panel_1","type":"visualization"},{"id":"e4aef350-623d-11eb-aebf-c306684b328d","name":"panel_2","type":"visualization"},{"id":"f92e5630-623e-11eb-aebf-c306684b328d","name":"panel_3","type":"visualization"},{"id":"9853d4d0-623d-11eb-aebf-c306684b328d","name":"panel_4","type":"visualization"},{"id":"6ecb33b0-623d-11eb-aebf-c306684b328d","name":"panel_5","type":"visualization"},{"id":"b8e35c80-623c-11eb-aebf-c306684b328d","name":"panel_6","type":"visualization"},{"id":"f1bc75d0-6239-11eb-aebf-c306684b328d","name":"panel_7","type":"visualization"},{"id":"0d8a8860-623a-11eb-aebf-c306684b328d","name":"panel_8","type":"visualization"},{"id":"d79fe3d0-6239-11eb-aebf-c306684b328d","name":"panel_9","type":"visualization"},{"id":"318375a0-6240-11eb-aebf-c306684b328d","name":"panel_10","type":"visualization"},{"id":"e461eb20-6245-11eb-aebf-c306684b328d","name":"panel_11","type":"visualization"},{"id":"25bdc750-6242-11eb-aebf-c306684b328d","name":"panel_12","type":"visualization"},{"id":"71dd7bc0-6248-11eb-aebf-c306684b328d","name":"panel_13","type":"visualization"},{"id":"6aea48a0-6240-11eb-aebf-c306684b328d","name":"panel_14","type":"visualization"},{"id":"32b681f0-6241-11eb-aebf-c306684b328d","name":"panel_15","type":"visualization"},{"id":"ccca99e0-6244-11eb-aebf-c306684b328d","name":"panel_16","type":"visualization"},{"id":"a4d7be80-6245-11eb-aebf-c306684b328d","name":"panel_17","type":"visualization"},{"id":"c94d8440-6248-11eb-aebf-c306684b328d","name":"panel_18","type":"visualization"},{"id":"db6226f0-61c0-11eb-aebf-c306684b328d","name":"panel_19","type":"search"}],"sort":[1617218954375,161],"type":"dashboard","updated_at":"2021-03-31T19:29:14.375Z","version":"WzcxLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"shakespeare_tag_cloud","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"shakespeare_tag_cloud\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"linear\",\"orientation\":\"multiple\",\"minFontSize\":59,\"maxFontSize\":100,\"showLabel\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"type.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"coreMigrationVersion":"7.12.1","id":"622ac7f0-619e-11eb-aebf-c306684b328d","migrationVersion":{"visualization":"7.12.0"},"references":[{"id":"4e937b20-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218929689,13],"type":"visualization","updated_at":"2021-03-31T19:28:49.689Z","version":"WzIyLDFd"} +{"attributes":{"buildNum":9007199254740991,"defaultIndex":"56b34100-619d-11eb-aebf-c306684b328d"},"coreMigrationVersion":"7.12.1","id":"7.12.1","migrationVersion":{"config":"7.12.0"},"references":[],"sort":[1617218966119,191],"type":"config","updated_at":"2021-03-31T19:29:26.119Z","version":"Wzc3LDFd"} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"meta\":{\"negate\":false,\"type\":\"phrase\",\"key\":\"text_entry\",\"value\":\"Christendom.\",\"params\":{\"query\":\"Christendom.\",\"type\":\"phrase\"},\"disabled\":false,\"alias\":null,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"text_entry\":{\"query\":\"Christendom.\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["_score","desc"]],"title":"shakespeare_saved_search","version":1},"coreMigrationVersion":"7.12.1","id":"712ebbe0-619d-11eb-aebf-c306684b328d","migrationVersion":{"search":"7.9.3"},"references":[{"id":"4e937b20-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"4e937b20-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"sort":[1617218925706,93],"type":"search","updated_at":"2021-03-31T19:28:45.706Z","version":"WzEzLDFd"} +{"attributes":{"columns":["play_name","speaker"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"speaker:\\\"GLOUCESTER\\\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["_score","desc"]],"title":"shakespeare_saved_lucene_search","version":1},"coreMigrationVersion":"7.12.1","id":"ddacc820-619d-11eb-aebf-c306684b328d","migrationVersion":{"search":"7.9.3"},"references":[{"id":"4e937b20-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218927635,25],"type":"search","updated_at":"2021-03-31T19:28:47.635Z","version":"WzE2LDFd"} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"text_entry :\\\"MORDAKE THE EARL OF FIFE, AND ELDEST SON\\\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["_score","desc"]],"title":"shakespeare_saved_kql_search","version":1},"coreMigrationVersion":"7.12.1","id":"f852d570-619d-11eb-aebf-c306684b328d","migrationVersion":{"search":"7.9.3"},"references":[{"id":"4e937b20-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1617218926603,23],"type":"search","updated_at":"2021-03-31T19:28:46.603Z","version":"WzE0LDFd"} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"1\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"2\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"4\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"5\",\"w\":24,\"x\":0,\"y\":30},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"6\",\"w\":24,\"x\":24,\"y\":30},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"}]","timeRestore":false,"title":"shakespeare_dashboard","version":1},"coreMigrationVersion":"7.12.1","id":"73398a90-619e-11eb-aebf-c306684b328d","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"185283c0-619e-11eb-aebf-c306684b328d","name":"panel_0","type":"visualization"},{"id":"33736660-619e-11eb-aebf-c306684b328d","name":"panel_1","type":"visualization"},{"id":"622ac7f0-619e-11eb-aebf-c306684b328d","name":"panel_2","type":"visualization"},{"id":"712ebbe0-619d-11eb-aebf-c306684b328d","name":"panel_3","type":"search"},{"id":"ddacc820-619d-11eb-aebf-c306684b328d","name":"panel_4","type":"search"},{"id":"f852d570-619d-11eb-aebf-c306684b328d","name":"panel_5","type":"search"}],"sort":[1617218931742,88],"type":"dashboard","updated_at":"2021-03-31T19:28:51.742Z","version":"WzI2LDFd"} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"meta\":{\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"geo.srcdest\",\"value\":\"IN:US\",\"params\":{\"query\":\"IN:US\",\"type\":\"phrase\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"geo.srcdest\":{\"query\":\"IN:US\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"1\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"2\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"4\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"5\",\"w\":24,\"x\":0,\"y\":30},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"6\",\"w\":24,\"x\":24,\"y\":30},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"7\",\"w\":24,\"x\":0,\"y\":45},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"8\",\"w\":24,\"x\":24,\"y\":45},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"9\",\"w\":24,\"x\":0,\"y\":60},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"10\",\"w\":24,\"x\":24,\"y\":60},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"11\",\"w\":24,\"x\":0,\"y\":75},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"12\",\"w\":24,\"x\":24,\"y\":75},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"13\",\"w\":24,\"x\":0,\"y\":90},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_12\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"14\",\"w\":24,\"x\":24,\"y\":90},\"panelIndex\":\"14\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_13\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"15\",\"w\":24,\"x\":0,\"y\":105},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_14\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"16\",\"w\":24,\"x\":24,\"y\":105},\"panelIndex\":\"16\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_15\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"17\",\"w\":24,\"x\":0,\"y\":120},\"panelIndex\":\"17\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_16\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"18\",\"w\":24,\"x\":24,\"y\":120},\"panelIndex\":\"18\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_17\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"19\",\"w\":24,\"x\":0,\"y\":135},\"panelIndex\":\"19\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_18\"},{\"version\":\"7.3.0\",\"gridData\":{\"h\":15,\"i\":\"20\",\"w\":24,\"x\":24,\"y\":135},\"panelIndex\":\"20\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_19\"}]","timeRestore":false,"title":"logstash_dashboardwithfilters","version":1},"coreMigrationVersion":"7.12.1","id":"79794f20-6249-11eb-aebf-c306684b328d","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"56b34100-619d-11eb-aebf-c306684b328d","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"36b91810-6239-11eb-aebf-c306684b328d","name":"panel_0","type":"visualization"},{"id":"0a274320-61cc-11eb-aebf-c306684b328d","name":"panel_1","type":"visualization"},{"id":"e4aef350-623d-11eb-aebf-c306684b328d","name":"panel_2","type":"visualization"},{"id":"f92e5630-623e-11eb-aebf-c306684b328d","name":"panel_3","type":"visualization"},{"id":"9853d4d0-623d-11eb-aebf-c306684b328d","name":"panel_4","type":"visualization"},{"id":"6ecb33b0-623d-11eb-aebf-c306684b328d","name":"panel_5","type":"visualization"},{"id":"b8e35c80-623c-11eb-aebf-c306684b328d","name":"panel_6","type":"visualization"},{"id":"f1bc75d0-6239-11eb-aebf-c306684b328d","name":"panel_7","type":"visualization"},{"id":"0d8a8860-623a-11eb-aebf-c306684b328d","name":"panel_8","type":"visualization"},{"id":"d79fe3d0-6239-11eb-aebf-c306684b328d","name":"panel_9","type":"visualization"},{"id":"318375a0-6240-11eb-aebf-c306684b328d","name":"panel_10","type":"visualization"},{"id":"e461eb20-6245-11eb-aebf-c306684b328d","name":"panel_11","type":"visualization"},{"id":"25bdc750-6242-11eb-aebf-c306684b328d","name":"panel_12","type":"visualization"},{"id":"71dd7bc0-6248-11eb-aebf-c306684b328d","name":"panel_13","type":"visualization"},{"id":"6aea48a0-6240-11eb-aebf-c306684b328d","name":"panel_14","type":"visualization"},{"id":"32b681f0-6241-11eb-aebf-c306684b328d","name":"panel_15","type":"visualization"},{"id":"ccca99e0-6244-11eb-aebf-c306684b328d","name":"panel_16","type":"visualization"},{"id":"a4d7be80-6245-11eb-aebf-c306684b328d","name":"panel_17","type":"visualization"},{"id":"c94d8440-6248-11eb-aebf-c306684b328d","name":"panel_18","type":"visualization"},{"id":"db6226f0-61c0-11eb-aebf-c306684b328d","name":"panel_19","type":"search"}],"sort":[1617218955401,140],"type":"dashboard","updated_at":"2021-03-31T19:29:15.401Z","version":"WzczLDFd"} +{"exportedCount":33,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/x-pack/test/functional/apps/saved_objects_management/import_saved_objects_between_versions.ts b/x-pack/test/functional/apps/saved_objects_management/import_saved_objects_between_versions.ts new file mode 100644 index 00000000000000..07fe0e910ea99d --- /dev/null +++ b/x-pack/test/functional/apps/saved_objects_management/import_saved_objects_between_versions.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* This test is importing saved objects from 7.12.0 to 8.0 and the backported version + * will import from 6.8.x to 8.0.0 + */ + +import expect from '@kbn/expect'; +import path from 'path'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const kibanaServer = getService('kibanaServer'); + const esArchiver = getService('esArchiver'); + const PageObjects = getPageObjects(['common', 'settings', 'header', 'savedObjects']); + const testSubjects = getService('testSubjects'); + const retry = getService('retry'); + + describe('Export import saved objects between versions', function () { + beforeEach(async function () { + await esArchiver.load('logstash_functional'); + await esArchiver.load('getting_started/shakespeare'); + await kibanaServer.uiSettings.replace({}); + await PageObjects.settings.navigateTo(); + await PageObjects.settings.clickKibanaSavedObjects(); + }); + + after(async () => { + await esArchiver.unload('logstash_functional'); + await esArchiver.unload('getting_started/shakespeare'); + await esArchiver.load('empty_kibana'); + }); + + it('should be able to import 7.12 saved objects into 8.0.0', async function () { + await retry.tryForTime(10000, async () => { + const existingSavedObjects = await testSubjects.getVisibleText('exportAllObjects'); + // Kibana always has 1 advanced setting as a saved object + await expect(existingSavedObjects).to.be('Export 1 object'); + }); + await PageObjects.savedObjects.importFile( + path.join(__dirname, 'exports', '_7.12_import_saved_objects.ndjson') + ); + await PageObjects.savedObjects.checkImportSucceeded(); + await PageObjects.savedObjects.clickImportDone(); + const importedSavedObjects = await testSubjects.getVisibleText('exportAllObjects'); + // verifying the count of saved objects after importing .ndjson + await expect(importedSavedObjects).to.be('Export 34 objects'); + }); + }); +} diff --git a/x-pack/test/functional/apps/saved_objects_management/index.ts b/x-pack/test/functional/apps/saved_objects_management/index.ts index 602f87c1af38eb..d474755af4676c 100644 --- a/x-pack/test/functional/apps/saved_objects_management/index.ts +++ b/x-pack/test/functional/apps/saved_objects_management/index.ts @@ -13,5 +13,6 @@ export default function savedObjectsManagementApp({ loadTestFile }: FtrProviderC loadTestFile(require.resolve('./spaces_integration')); loadTestFile(require.resolve('./feature_controls/saved_objects_management_security')); + loadTestFile(require.resolve('./import_saved_objects_between_versions')); }); } diff --git a/x-pack/test/functional/apps/security/users.js b/x-pack/test/functional/apps/security/users.js index 250a2d4ed71f9c..8730ee3aeeaf26 100644 --- a/x-pack/test/functional/apps/security/users.js +++ b/x-pack/test/functional/apps/security/users.js @@ -12,8 +12,7 @@ export default function ({ getService, getPageObjects }) { const config = getService('config'); const log = getService('log'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/96001 - describe.skip('users', function () { + describe('users', function () { before(async () => { log.debug('users'); await PageObjects.settings.navigateTo(); @@ -91,7 +90,7 @@ export default function ({ getService, getPageObjects }) { expect(roles.apm_system.deprecated).to.be(false); expect(roles.apm_user.reserved).to.be(true); - expect(roles.apm_user.deprecated).to.be(false); + expect(roles.apm_user.deprecated).to.be(true); expect(roles.beats_admin.reserved).to.be(true); expect(roles.beats_admin.deprecated).to.be(false); diff --git a/x-pack/test/functional/es_archives/getting_started/shakespeare/data.json.gz b/x-pack/test/functional/es_archives/getting_started/shakespeare/data.json.gz new file mode 100644 index 00000000000000..dcd31ef31085e0 Binary files /dev/null and b/x-pack/test/functional/es_archives/getting_started/shakespeare/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/getting_started/shakespeare/mappings.json b/x-pack/test/functional/es_archives/getting_started/shakespeare/mappings.json new file mode 100644 index 00000000000000..0e030c54d49122 --- /dev/null +++ b/x-pack/test/functional/es_archives/getting_started/shakespeare/mappings.json @@ -0,0 +1,28 @@ +{ + "type": "index", + "value": { + "index": "shakespeare", + "mappings": { + "properties": { + "line_id": { + "type": "integer" + }, + "play_name": { + "type": "keyword" + }, + "speaker": { + "type": "keyword" + }, + "speech_number": { + "type": "integer" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "5" + } + } + } +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index ea663207aafa5a..a7e5b0bbfe4dd3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1197,6 +1197,16 @@ resolved "https://registry.yarnpkg.com/@bazel/ibazel/-/ibazel-0.14.0.tgz#86fa0002bed2ce1123b7ad98d4dd4623a0d93244" integrity sha512-s0gyec6lArcRDwVfIP6xpY8iEaFpzrSpyErSppd3r2O49pOEg7n6HGS/qJ8ncvme56vrDk6crl/kQ6VAdEO+rg== +"@bazel/typescript@^3.2.3": + version "3.2.3" + resolved "https://registry.yarnpkg.com/@bazel/typescript/-/typescript-3.2.3.tgz#6e40bdb7c5294e588bac3b7d1269e58b98a1856c" + integrity sha512-Q1Yin/AYdh9yrkSJo3H6nVn6mMaohr5syjLd0Df0w7WI4zerdJTxrY5nhoWZwO/S1rPj8/MedDwZudCqPDeDMA== + dependencies: + protobufjs "6.8.8" + semver "5.6.0" + source-map-support "0.5.9" + tsutils "2.27.2" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -1375,7 +1385,7 @@ utility-types "^3.10.0" uuid "^3.3.2" -"@elastic/datemath@link:packages/elastic-datemath": +"@elastic/datemath@link:bazel-bin/packages/elastic-datemath/npm_module": version "0.0.0" uid "" @@ -3441,6 +3451,59 @@ dependencies: "@babel/runtime" "^7.0.0" +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha1-m4sMxmPWaafY9vXQiToU00jzD78= + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha1-NVy8mLr61ZePntCV85diHx0Ga3A= + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU= + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E= + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik= + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha1-bMKyDFya1q0NzP0hynZz2Nf79o0= + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q= + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= + "@reach/router@^1.3.3": version "1.3.4" resolved "https://registry.yarnpkg.com/@reach/router/-/router-1.3.4.tgz#d2574b19370a70c80480ed91f3da840136d10f8c" @@ -5181,6 +5244,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.159.tgz#61089719dc6fdd9c5cb46efc827f2571d1517065" integrity sha512-gF7A72f7WQN33DpqOWw9geApQPh4M3PxluMtaHxWHXEGSN12/WbcEk/eNSqWNQcQhF66VSZ06vCF94CrHwXJDg== +"@types/long@^4.0.0": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" + integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== + "@types/lru-cache@^5.1.0": version "5.1.0" resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" @@ -5330,7 +5398,7 @@ dependencies: "@types/node" "*" -"@types/node@*", "@types/node@14.14.14", "@types/node@8.10.54", "@types/node@>= 8", "@types/node@>=8.9.0", "@types/node@^12.0.2": +"@types/node@*", "@types/node@14.14.14", "@types/node@8.10.54", "@types/node@>= 8", "@types/node@>=8.9.0", "@types/node@^10.1.0", "@types/node@^12.0.2": version "14.14.14" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.14.tgz#f7fd5f3cc8521301119f63910f0fb965c7d761ae" integrity sha512-UHnOPWVWV1z+VV8k6L1HhG7UbGBgIdghqF3l9Ny9ApPghbjICXkUJSd/b9gOgQfjM1r+37cipdw/HJ3F6ICEnQ== @@ -22915,6 +22983,25 @@ proto-list@~1.2.1: resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" integrity sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk= +protobufjs@6.8.8: + version "6.8.8" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.8.8.tgz#c8b4f1282fd7a90e6f5b109ed11c84af82908e7c" + integrity sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.0" + "@types/node" "^10.1.0" + long "^4.0.0" + protocol-buffers-schema@^3.3.1: version "3.3.2" resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz#00434f608b4e8df54c59e070efeefc37fb4bb859" @@ -25437,6 +25524,11 @@ semver-greatest-satisfied-range@^1.1.0: resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== +semver@5.6.0: + version "5.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.6.0.tgz#7e74256fbaa49c75aa7c7a205cc22799cac80004" + integrity sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg== + semver@7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" @@ -25924,6 +26016,14 @@ source-map-resolve@^0.6.0: atob "^2.1.2" decode-uri-component "^0.2.0" +source-map-support@0.5.9: + version "0.5.9" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.9.tgz#41bc953b2534267ea2d605bccfa7bfa3111ced5f" + integrity sha512-gR6Rw4MvUlYy83vP0vxoVNzM6t8MUXqNuRsuBmBHQDu1Fh6X015FrLdgoDKcNdkwGubozq0P4N0Q37UyFVr1EA== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + source-map-support@^0.3.2: version "0.3.3" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.3.3.tgz#34900977d5ba3f07c7757ee72e73bb1a9b53754f" @@ -27859,6 +27959,13 @@ tslib@~2.1.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== +tsutils@2.27.2: + version "2.27.2" + resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-2.27.2.tgz#60ba88a23d6f785ec4b89c6e8179cac9b431f1c7" + integrity sha512-qf6rmT84TFMuxAKez2pIfR8UCai49iQsfB7YWVjV1bKpy/d0PWT5rEOSM6La9PiHZ0k1RRZQiwVdVJfQ3BPHgg== + dependencies: + tslib "^1.8.1" + tsutils@^3.17.1: version "3.17.1" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.17.1.tgz#ed719917f11ca0dee586272b2ac49e015a2dd759"