diff --git a/docs/build.gradle b/docs/build.gradle index 5b98a62d99640..7f3d3efbc3ce9 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -42,6 +42,7 @@ integTestCluster { // enable regexes in painless so our tests don't complain about example snippets that use them setting 'script.painless.regex.enabled', 'true' + setting 'path.repo', new File(buildDir, "cluster/shared/repo") Closure configFile = { extraConfigFile it, "src/test/cluster/config/$it" } @@ -1136,3 +1137,13 @@ buildRestTests.setups['seats'] = ''' {"theatre": "Graye", "cost": 33, "row": 2, "number": 6, "sold": false} {"index":{"_id": "4"}} {"theatre": "Skyline", "cost": 20, "row": 5, "number": 2, "sold": false}''' +// Used by snapshot lifecycle management docs +buildRestTests.setups['setup-repository'] = ''' + - do: + snapshot.create_repository: + repository: my_repository + body: + type: fs + settings: + location: buildDir/cluster/shared/repo +''' diff --git a/docs/reference/ilm/apis/slm-api.asciidoc b/docs/reference/ilm/apis/slm-api.asciidoc new file mode 100644 index 0000000000000..7a2924bd4a21d --- /dev/null +++ b/docs/reference/ilm/apis/slm-api.asciidoc @@ -0,0 +1,346 @@ +[role="xpack"] +[testenv="basic"] +[[snapshot-lifecycle-management-api]] +== Snapshot Lifecycle Management API + +The Snapshot Lifecycle Management APIs are used to manage policies for the time +and frequency of automatic snapshots. Snapshot Lifecycle Management is related +to <>, however, instead +of managing a lifecycle of actions that are performed on a single index, SLM +allows configuring policies spanning multiple indices. + +SLM policy management is split into three different CRUD APIs, a way to put or update +policies, a way to retrieve policies, and a way to delete unwanted policies, as +well as a separate API for immediately invoking a snapshot based on a policy. + +Since SLM falls under the same category as ILM, it is stopped and started by +using the <> ILM APIs. + +=== Put Snapshot Lifecycle Policy API + +Creates or updates a snapshot policy. If the policy already exists, the version +is incremented. Only the latest version of a policy is stored. + +When a policy is created it is immediately scheduled based on the schedule of +the policy, when a policy is updated its schedule changes are immediately +applied. + +==== Path Parameters + +`policy_id` (required):: + (string) Identifier (id) for the policy. + +==== Request Parameters + +include::{docdir}/rest-api/timeoutparms.asciidoc[] + +==== Authorization + +You must have the `manage_slm` cluster privilege to use this API. You must also +have the `manage` index privilege on all indices being managed by `policy`. All +operations executed by {slm} for a policy are executed as the user that put the +latest version of a policy. For more information, see +{stack-ov}/security-privileges.html[Security Privileges]. + +==== Example + +The following creates a snapshot lifecycle policy with an id of +`daily-snapshots`: + +[source,js] +-------------------------------------------------- +PUT /_slm/policy/daily-snapshots +{ + "schedule": "0 30 1 * * ?", <1> + "name": "", <2> + "repository": "my_repository", <3> + "config": { <4> + "indices": ["data-*", "important"], <5> + "ignore_unavailable": false, + "include_global_state": false + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:setup-repository] +<1> When the snapshot should be taken, in this case, 1:30am daily +<2> The name each snapshot should be given +<3> Which repository to take the snapshot in +<4> Any extra snapshot configuration +<5> Which indices the snapshot should contain + +The top-level keys that the policy supports are described below: + +|================== +| Key | Description + +| `schedule` | A periodic or absolute time schedule. Supports all values + supported by the cron scheduler: + {xpack-ref}/trigger-schedule.html#schedule-cron[Cron scheduler configuration] + +| `name` | A name automatically given to each snapshot performed by this policy. + Supports the same <> supported in index + names. A UUID is automatically appended to the end of the name to prevent + conflicting snapshot names. + +| `repository` | The snapshot repository that will contain snapshots created by + this policy. The repository must exist prior to the policy's creation and can + be created with the <>. + +| `config` | Configuration for each snapshot that will be created by this + policy. Any configuration is included with <> issued by this policy. +|================== + +To update an existing policy, simply use the put snapshot lifecycle policy API +with the same policy id as an existing policy. + +=== Get Snapshot Lifecycle Policy API + +Once a policy is in place, you can retrieve one or more of the policies using +the get snapshot lifecycle policy API. This also includes information about the +latest successful and failed invocation that the automatic snapshots have taken. + +==== Path Parameters + +`policy_ids` (optional):: + (string) Comma-separated ids of policies to retrieve. + +==== Examples + +To retrieve a policy, perform a `GET` with the policy's id + +[source,js] +-------------------------------------------------- +GET /_slm/policy/daily-snapshots?human +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +The output looks similar to the following: + +[source,js] +-------------------------------------------------- +{ + "daily-snapshots" : { + "version": 1, <1> + "modified_date": "2019-04-23T01:30:00.000Z", <2> + "modified_date_millis": 1556048137314, + "policy" : { + "schedule": "0 30 1 * * ?", + "name": "", + "repository": "my_repository", + "config": { + "indices": ["data-*", "important"], + "ignore_unavailable": false, + "include_global_state": false + } + }, + "next_execution": "2019-04-24T01:30:00.000Z", <3> + "next_execution_millis": 1556048160000 + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"modified_date": "2019-04-23T01:30:00.000Z"/"modified_date": $body.daily-snapshots.modified_date/ s/"modified_date_millis": 1556048137314/"modified_date_millis": $body.daily-snapshots.modified_date_millis/ s/"next_execution": "2019-04-24T01:30:00.000Z"/"next_execution": $body.daily-snapshots.next_execution/ s/"next_execution_millis": 1556048160000/"next_execution_millis": $body.daily-snapshots.next_execution_millis/] +<1> The version of the snapshot policy, only the latest verison is stored and incremented when the policy is updated +<2> The last time this policy was modified +<3> The next time this policy will be executed + +Or, to retrieve all policies: + +[source,js] +-------------------------------------------------- +GET /_slm/policy +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +=== Execute Snapshot Lifecycle Policy API + +Sometimes it can be useful to immediately execute a snapshot based on policy, +perhaps before an upgrade or before performing other maintenance on indices. The +execute snapshot policy API allows you to perform a snapshot immediately without +waiting for a policy's scheduled invocation. + +==== Path Parameters + +`policy_id` (required):: + (string) Id of the policy to execute + +==== Example + +To take an immediate snapshot using a policy, use the following + +[source,js] +-------------------------------------------------- +PUT /_slm/policy/daily-snapshots/_execute +-------------------------------------------------- +// CONSOLE +// TEST[skip:we can't easily handle snapshots from docs tests] + +This API will immediately return with the generated snapshot name + +[source,js] +-------------------------------------------------- +{ + "snapshot_name": "daily-snap-2019.04.24-gwrqoo2xtea3q57vvg0uea" +} +-------------------------------------------------- +// TESTRESPONSE[skip:we can't handle snapshots from docs tests] + +The snapshot will be taken in the background, you can use the +<> to monitor the status of the snapshot. + +Once a snapshot has been kicked off, you can see the latest successful or failed +snapshot using the get snapshot lifecycle policy API: + +[source,js] +-------------------------------------------------- +GET /_slm/policy/daily-snapshots?human +-------------------------------------------------- +// CONSOLE +// TEST[skip:we already tested get policy above, the last_failure may not be present though] + +Which, in this case shows an error because the index did not exist: + +[source,js] +-------------------------------------------------- +{ + "daily-snapshots" : { + "version": 1, + "modified_date": "2019-04-23T01:30:00.000Z", + "modified_date_millis": 1556048137314, + "policy" : { + "schedule": "0 30 1 * * ?", + "name": "", + "repository": "my_repository", + "config": { + "indices": ["data-*", "important"], + "ignore_unavailable": false, + "include_global_state": false + } + }, + "last_failure": { <1> + "snapshot_name": "daily-snap-2019.04.02-lohisb5ith2n8hxacaq3mw", + "time_string": "2019-04-02T01:30:00.000Z", + "time": 1556042030000, + "details": "{\"type\":\"index_not_found_exception\",\"reason\":\"no such index [important]\",\"resource.type\":\"index_or_alias\",\"resource.id\":\"important\",\"index_uuid\":\"_na_\",\"index\":\"important\",\"stack_trace\":\"[important] IndexNotFoundException[no such index [important]]\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.indexNotFoundException(IndexNameExpressionResolver.java:762)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.innerResolve(IndexNameExpressionResolver.java:714)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.resolve(IndexNameExpressionResolver.java:670)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:163)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:142)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:102)\\n\\tat org.elasticsearch.snapshots.SnapshotsService$1.execute(SnapshotsService.java:280)\\n\\tat org.elasticsearch.cluster.ClusterStateUpdateTask.execute(ClusterStateUpdateTask.java:47)\\n\\tat org.elasticsearch.cluster.service.MasterService.executeTasks(MasterService.java:687)\\n\\tat org.elasticsearch.cluster.service.MasterService.calculateTaskOutputs(MasterService.java:310)\\n\\tat org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:210)\\n\\tat org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:142)\\n\\tat org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150)\\n\\tat org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188)\\n\\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:688)\\n\\tat org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252)\\n\\tat org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\\n\\tat java.base/java.lang.Thread.run(Thread.java:834)\\n\"}" + } , + "next_execution": "2019-04-24T01:30:00.000Z", + "next_execution_millis": 1556048160000 + } +} +-------------------------------------------------- +// TESTRESPONSE[skip:the presence of last_failure is asynchronous and will be present for users, but is untestable] +<1> The last unsuccessfully initiated snapshot by this policy, along with the details of its failure + +In this case, it failed due to the "important" index not existing and +`ignore_unavailable` setting being set to `false`. + +Updating the policy to change the `ignore_unavailable` setting is done using the +same put snapshot lifecycle policy API: + +[source,js] +-------------------------------------------------- +PUT /_slm/policy/daily-snapshots +{ + "schedule": "0 30 1 * * ?", + "name": "", + "repository": "my_repository", + "config": { + "indices": ["data-*", "important"], + "ignore_unavailable": true, + "include_global_state": false + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +Another snapshot can immediately be executed to ensure the new policy works: + +[source,js] +-------------------------------------------------- +PUT /_slm/policy/daily-snapshots/_execute +-------------------------------------------------- +// CONSOLE +// TEST[skip:we can't handle snapshots in docs tests] + +[source,js] +-------------------------------------------------- +{ + "snapshot_name": "daily-snap-2019.04.24-tmtnyjtrsxkhbrrdcgg18a" +} +-------------------------------------------------- +// TESTRESPONSE[skip:we can't handle snapshots in docs tests] + +Now retriving the policy shows that the policy has successfully been executed: + + +[source,js] +-------------------------------------------------- +GET /_slm/policy/daily-snapshots?human +-------------------------------------------------- +// CONSOLE +// TEST[skip:we already tested this above and the output may not be available yet] + +Which now includes the successful snapshot information: + +[source,js] +-------------------------------------------------- +{ + "daily-snapshots" : { + "version": 2, <1> + "modified_date": "2019-04-23T01:30:00.000Z", + "modified_date_millis": 1556048137314, + "policy" : { + "schedule": "0 30 1 * * ?", + "name": "", + "repository": "my_repository", + "config": { + "indices": ["data-*", "important"], + "ignore_unavailable": true, + "include_global_state": false + } + }, + "last_success": { <2> + "snapshot_name": "daily-snap-2019.04.24-tmtnyjtrsxkhbrrdcgg18a", + "time_string": "2019-04-24T16:43:49.316Z", + "time": 1556124229316 + } , + "last_failure": { + "snapshot_name": "daily-snap-2019.04.02-lohisb5ith2n8hxacaq3mw", + "time_string": "2019-04-02T01:30:00.000Z", + "time": 1556042030000, + "details": "{\"type\":\"index_not_found_exception\",\"reason\":\"no such index [important]\",\"resource.type\":\"index_or_alias\",\"resource.id\":\"important\",\"index_uuid\":\"_na_\",\"index\":\"important\",\"stack_trace\":\"[important] IndexNotFoundException[no such index [important]]\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.indexNotFoundException(IndexNameExpressionResolver.java:762)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.innerResolve(IndexNameExpressionResolver.java:714)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.resolve(IndexNameExpressionResolver.java:670)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:163)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:142)\\n\\tat org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndexNames(IndexNameExpressionResolver.java:102)\\n\\tat org.elasticsearch.snapshots.SnapshotsService$1.execute(SnapshotsService.java:280)\\n\\tat org.elasticsearch.cluster.ClusterStateUpdateTask.execute(ClusterStateUpdateTask.java:47)\\n\\tat org.elasticsearch.cluster.service.MasterService.executeTasks(MasterService.java:687)\\n\\tat org.elasticsearch.cluster.service.MasterService.calculateTaskOutputs(MasterService.java:310)\\n\\tat org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:210)\\n\\tat org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:142)\\n\\tat org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150)\\n\\tat org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188)\\n\\tat org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:688)\\n\\tat org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252)\\n\\tat org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\\n\\tat java.base/java.lang.Thread.run(Thread.java:834)\\n\"}" + } , + "next_execution": "2019-04-24T01:30:00.000Z", + "next_execution_millis": 1556048160000 + } +} +-------------------------------------------------- +// TESTRESPONSE[skip:the presence of last_failure and last_success is asynchronous and will be present for users, but is untestable] +<1> The policy's version has been incremented because it was updated +<2> The last successfully initiated snapshot information + +It is a good idea to test policies using the execute API to ensure they work. + +=== Delete Snapshot Lifecycle Policy API + +A policy can be deleted by issuing a delete request with the policy id. Note +that this prevents any future snapshots from being taken, but does not cancel +any currently ongoing snapshots or remove any previously taken snapshots. + +==== Path Parameters + +`policy_id` (optional):: + (string) Id of the policy to remove. + +==== Example + +[source,js] +-------------------------------------------------- +DELETE /_slm/policy/daily-snapshots +-------------------------------------------------- +// CONSOLE +// TEST[continued] diff --git a/docs/reference/ilm/index.asciidoc b/docs/reference/ilm/index.asciidoc index b906f9ade4447..e9972a713dfb3 100644 --- a/docs/reference/ilm/index.asciidoc +++ b/docs/reference/ilm/index.asciidoc @@ -47,6 +47,14 @@ to a single shard. hardware. . Delete the index once the required 30 day retention period is reached. +*Snapshot Lifecycle Management* + +ILM itself does allow managing indices, however, managing snapshots for a set of +indices is outside of the scope of an index-level policy. Instead, there are +separate APIs for managing snapshot lifecycles. Please see the +<> +documentation for information about configuring snapshots. + [IMPORTANT] =========================== {ilm} does not support mixed-version cluster usage. Although it diff --git a/docs/reference/ilm/start-stop-ilm.asciidoc b/docs/reference/ilm/start-stop-ilm.asciidoc index e5366f028a9c7..d65a808ef4f51 100644 --- a/docs/reference/ilm/start-stop-ilm.asciidoc +++ b/docs/reference/ilm/start-stop-ilm.asciidoc @@ -10,6 +10,10 @@ maybe there are scheduled maintenance windows when cluster topology changes are desired that may impact running ILM actions. For this reason, ILM has two ways to disable operations. +When stopping ILM, snapshot lifecycle management operations are also stopped, +this means that no scheduled snapshots are created (currently ongoing snapshots +are unaffected). + Normally, ILM will be running by default. To see the current operating status of ILM, use the <> to see the current state of ILM. diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index 1009c42e8b4ca..4c6e2a421c2e2 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -12,6 +12,7 @@ directly to configure and access {xpack} features. * <> * <>, <> * <> +* <> * <> * <> * <> @@ -27,6 +28,7 @@ include::{es-repo-dir}/data-frames/apis/index.asciidoc[] include::{es-repo-dir}/graph/explore.asciidoc[] include::{es-repo-dir}/indices/apis/freeze.asciidoc[] include::{es-repo-dir}/ilm/apis/ilm-api.asciidoc[] +include::{es-repo-dir}/ilm/apis/slm-api.asciidoc[] include::{es-repo-dir}/licensing/index.asciidoc[] include::{es-repo-dir}/migration/migration.asciidoc[] include::{es-repo-dir}/ml/apis/ml-api.asciidoc[] diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.delete_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.delete_lifecycle.json new file mode 100644 index 0000000000000..c60c99783d239 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.delete_lifecycle.json @@ -0,0 +1,19 @@ +{ + "slm.delete_lifecycle": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/slm-api.html", + "methods": [ "DELETE" ], + "url": { + "path": "/_slm/policy/{policy_id}", + "paths": ["/_slm/policy/{policy_id}"], + "parts": { + "policy": { + "type" : "string", + "description" : "The id of the snapshot lifecycle policy to remove" + } + }, + "params": { + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.execute_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.execute_lifecycle.json new file mode 100644 index 0000000000000..c8e2d80ecb1ea --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.execute_lifecycle.json @@ -0,0 +1,19 @@ +{ + "slm.execute_lifecycle": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/slm-api.html", + "methods": [ "PUT" ], + "url": { + "path": "/_slm/policy/{policy_id}/_execute", + "paths": ["/_slm/policy/{policy_id}/_execute"], + "parts": { + "policy_id": { + "type" : "string", + "description" : "The id of the snapshot lifecycle policy to be executed" + } + }, + "params": { + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.get_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.get_lifecycle.json new file mode 100644 index 0000000000000..18016b50f63d9 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.get_lifecycle.json @@ -0,0 +1,19 @@ +{ + "slm.get_lifecycle": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/slm-api.html", + "methods": [ "GET" ], + "url": { + "path": "/_slm/policy/{policy_id}", + "paths": ["/_slm/policy/{policy_id}", "/_slm/policy"], + "parts": { + "policy_id": { + "type" : "string", + "description" : "Comma-separated list of snapshot lifecycle policies to retrieve" + } + }, + "params": { + } + }, + "body": null + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.put_lifecycle.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.put_lifecycle.json new file mode 100644 index 0000000000000..7d5f1da875fba --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/slm.put_lifecycle.json @@ -0,0 +1,21 @@ +{ + "slm.put_lifecycle": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/current/slm-api.html", + "methods": [ "PUT" ], + "url": { + "path": "/_slm/policy/{policy_id}", + "paths": ["/_slm/policy/{policy_id}"], + "parts": { + "policy_id": { + "type" : "string", + "description" : "The id of the snapshot lifecycle policy" + } + }, + "params": { + } + }, + "body": { + "description": "The snapshot lifecycle policy definition to register" + } + } +}