[7.x] [Security Solutions] Removes circular dependencies and introduces kbn-security-solution-es-utils package (#99828)

docs/developer/getting-started/monorepo-packages.asciidoc

@@ -82,6 +82,7 @@ yarn kbn watch-bazel
 - @kbn/logging
 - @kbn/securitysolution-constants
 - @kbn/securitysolution-utils
+- @kbn/securitysolution-es-utils
 - @kbn/securitysolution-io-ts-utils
 - @kbn/std
 - @kbn/tinymath

package.json

@@ -134,6 +134,7 @@
     "@kbn/monaco": "link:packages/kbn-monaco",
     "@kbn/securitysolution-constants": "link:bazel-bin/packages/kbn-securitysolution-constants/npm_module",
     "@kbn/securitysolution-utils": "link:bazel-bin/packages/kbn-securitysolution-utils/npm_module",
+    "@kbn/securitysolution-es-utils": "link:bazel-bin/packages/kbn-securitysolution-es-utils/npm_module",
     "@kbn/securitysolution-io-ts-utils": "link:bazel-bin/packages/kbn-securitysolution-io-ts-utils/npm_module",
     "@kbn/server-http-tools": "link:packages/kbn-server-http-tools",
     "@kbn/server-route-repository": "link:packages/kbn-server-route-repository",

packages/BUILD.bazel

@@ -26,6 +26,7 @@ filegroup(
       "//packages/kbn-securitysolution-constants:build",
       "//packages/kbn-securitysolution-io-ts-utils:build",
       "//packages/kbn-securitysolution-utils:build",
+      "//packages/kbn-securitysolution-es-utils:build",
       "//packages/kbn-std:build",
       "//packages/kbn-tinymath:build",
       "//packages/kbn-utility-types:build",

packages/kbn-securitysolution-es-utils/BUILD.bazel

@@ -0,0 +1,86 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
+
+PKG_BASE_NAME = "kbn-securitysolution-es-utils"
+
+PKG_REQUIRE_NAME = "@kbn/securitysolution-es-utils"
+
+SOURCE_FILES = glob(
+  [
+    "src/**/*.ts",
+  ],
+  exclude = [
+    "**/*.test.*",
+    "**/*.mock.*",
+  ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+  name = "srcs",
+  srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+  "package.json",
+  "README.md",
+]
+
+SRC_DEPS = [
+  "@npm//@elastic/elasticsearch",
+  "@npm//@hapi/hapi",
+  "@npm//tslib",
+]
+
+TYPES_DEPS = [
+  "@npm//@types/jest",
+  "@npm//@types/node",
+]
+
+DEPS = SRC_DEPS + TYPES_DEPS
+
+ts_config(
+  name = "tsconfig",
+  src = "tsconfig.json",
+  deps = [
+      "//:tsconfig.base.json",
+  ],
+)
+
+ts_project(
+  name = "tsc",
+  srcs = SRCS,
+  args = ["--pretty"],
+  declaration = True,
+  declaration_map = True,
+  incremental = True,
+  out_dir = "target",
+  root_dir = "src",
+  source_map = True,
+  tsconfig = ":tsconfig",
+  deps = DEPS,
+)
+
+js_library(
+  name = PKG_BASE_NAME,
+  package_name = PKG_REQUIRE_NAME,
+  srcs = NPM_MODULE_EXTRA_FILES,
+  visibility = ["//visibility:public"],
+  deps = [":tsc"] + DEPS,
+)
+
+pkg_npm(
+  name = "npm_module",
+  deps = [
+      ":%s" % PKG_BASE_NAME,
+  ],
+)
+
+filegroup(
+  name = "build",
+  srcs = [
+      ":npm_module",
+  ],
+  visibility = ["//visibility:public"],
+)

packages/kbn-securitysolution-es-utils/README.md

@@ -0,0 +1,6 @@
+# kbn-securitysolution-es-utils
+
+This is the shared security solution elastic search utilities among plugins. This was originally created
+to remove the dependencies between security_solution and other projects such as lists. This should only be
+used within server side code and not client side code since it is all elastic search utilities and packages.
+

packages/kbn-securitysolution-es-utils/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-securitysolution-es-utils'],
+};

packages/kbn-securitysolution-es-utils/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@kbn/securitysolution-es-utils",
+  "version": "1.0.0",
+  "description": "security solution elastic search utilities to use across plugins such lists, security_solution, cases, etc...",
+  "license": "SSPL-1.0 OR Elastic License 2.0",
+  "main": "./target/index.js",
+  "types": "./target/index.d.ts",
+  "private": true
+}

packages/kbn-securitysolution-es-utils/src/bad_request_error/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export class BadRequestError extends Error {}

packages/kbn-securitysolution-es-utils/src/create_boostrap_index/index.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+// See the reference(s) below on explanations about why -000001 was chosen and
+// why the is_write_index is true as well as the bootstrapping step which is needed.
+// Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/applying-policy-to-template.html
+export const createBootstrapIndex = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<unknown> => {
+  return (
+    await esClient.transport.request({
+      path: `/${index}-000001`,
+      method: 'PUT',
+      body: {
+        aliases: {
+          [index]: {
+            is_write_index: true,
+          },
+        },
+      },
+    })
+  ).body;
+};

packages/kbn-securitysolution-es-utils/src/delete_all_index/index.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const deleteAllIndex = async (
+  esClient: ElasticsearchClient,
+  pattern: string,
+  maxAttempts = 5
+): Promise<boolean> => {
+  for (let attempt = 1; ; attempt++) {
+    if (attempt > maxAttempts) {
+      throw new Error(
+        `Failed to delete indexes with pattern [${pattern}] after ${maxAttempts} attempts`
+      );
+    }
+
+    // resolve pattern to concrete index names
+    const { body: resp } = await esClient.indices.getAlias(
+      {
+        index: pattern,
+      },
+      { ignore: [404] }
+    );
+
+    // @ts-expect-error status doesn't exist on response
+    if (resp.status === 404) {
+      return true;
+    }
+
+    const indices = Object.keys(resp) as string[];
+
+    // if no indexes exits then we're done with this pattern
+    if (!indices.length) {
+      return true;
+    }
+
+    // delete the concrete indexes we found and try again until this pattern resolves to no indexes
+    await esClient.indices.delete({
+      index: indices,
+      ignore_unavailable: true,
+    });
+  }
+};

packages/kbn-securitysolution-es-utils/src/delete_policy/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const deletePolicy = async (
+  esClient: ElasticsearchClient,
+  policy: string
+): Promise<unknown> => {
+  return (
+    await esClient.transport.request({
+      path: `/_ilm/policy/${policy}`,
+      method: 'DELETE',
+    })
+  ).body;
+};

packages/kbn-securitysolution-es-utils/src/delete_template/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const deleteTemplate = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<unknown> => {
+  return (
+    await esClient.indices.deleteTemplate({
+      name,
+    })
+  ).body;
+};

packages/kbn-securitysolution-es-utils/src/elasticsearch_client/index.ts

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// Copied from src/core/server/elasticsearch/client/types.ts
+// as these types aren't part of any package yet. Once they are, remove this completely
+
+import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
+import type {
+  ApiResponse,
+  TransportRequestOptions,
+  TransportRequestParams,
+  TransportRequestPromise,
+} from '@elastic/elasticsearch/lib/Transport';
+
+/**
+ * Client used to query the elasticsearch cluster.
+ * @deprecated At some point use the one from src/core/server/elasticsearch/client/types.ts when it is made into a package. If it never is, then keep using this one.
+ * @public
+ */
+export type ElasticsearchClient = Omit<
+  KibanaClient,
+  'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close'
+> & {
+  transport: {
+    request(
+      params: TransportRequestParams,
+      options?: TransportRequestOptions
+    ): TransportRequestPromise<ApiResponse>;
+  };
+};

packages/kbn-securitysolution-es-utils/src/get_index_exists/index.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const getIndexExists = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<boolean> => {
+  try {
+    const { body: response } = await esClient.search({
+      index,
+      size: 0,
+      allow_no_indices: true,
+      body: {
+        terminate_after: 1,
+      },
+    });
+    return response._shards.total > 0;
+  } catch (err) {
+    if (err.body != null && err.body.status === 404) {
+      return false;
+    } else {
+      throw err.body ? err.body : err;
+    }
+  }
+};

packages/kbn-securitysolution-es-utils/src/get_policy_exists/index.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const getPolicyExists = async (
+  esClient: ElasticsearchClient,
+  policy: string
+): Promise<boolean> => {
+  try {
+    await esClient.transport.request({
+      path: `/_ilm/policy/${policy}`,
+      method: 'GET',
+    });
+    // Return true that there exists a policy which is not 404 or some error
+    // Since there is not a policy exists API, this is how we create one by calling
+    // into the API to get it if it exists or rely on it to throw a 404
+    return true;
+  } catch (err) {
+    if (err.statusCode === 404) {
+      return false;
+    } else {
+      throw err;
+    }
+  }
+};

packages/kbn-securitysolution-es-utils/src/get_template_exists/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { ElasticsearchClient } from '../elasticsearch_client';
+
+export const getTemplateExists = async (
+  esClient: ElasticsearchClient,
+  template: string
+): Promise<boolean> => {
+  return (
+    await esClient.indices.existsTemplate({
+      name: template,
+    })
+  ).body;
+};