diff --git a/control-plane/agents/src/bin/core/tests/volume/capacity_limit.rs b/control-plane/agents/src/bin/core/tests/volume/capacity_limit.rs new file mode 100644 index 000000000..07618bf90 --- /dev/null +++ b/control-plane/agents/src/bin/core/tests/volume/capacity_limit.rs @@ -0,0 +1,88 @@ +#![cfg(test)] + +use deployer_cluster::ClusterBuilder; +use grpc::operations::volume::traits::VolumeOperations; +use std::time::Duration; +use stor_port::{ + transport_api::ReplyErrorKind, + types::v0::transport::{CreateVolume, DestroyVolume}, +}; +use uuid::Uuid; + +const SIZE: u64 = 5242880; +const EXCESS: u64 = 1000000; + +#[tokio::test] +async fn volume_create_with_capacity_limit() { + let cache_period = Duration::from_millis(250); + let reconcile_period = Duration::from_millis(3000); + let cluster = ClusterBuilder::builder() + .with_rest(true) + .with_io_engines(1) + .with_pool(0, "malloc:///p1?size_mb=300") + .with_cache_period(&humantime::Duration::from(cache_period).to_string()) + .with_reconcile_period(reconcile_period, reconcile_period) + .build() + .await + .unwrap(); + + let volume_client = cluster.grpc_client().volume(); + + // test with no capacity limit + grpc_create_volume_with_limit(&volume_client, SIZE, None, None).await; + + // test exceeding the capacity limit + grpc_create_volume_with_limit( + &volume_client, + SIZE, + Some(SIZE - EXCESS), // capacity smaller than volume + Some(ReplyErrorKind::CapacityLimitExceeded {}), + ) + .await; + + // test at the capacity limit + grpc_create_volume_with_limit(&volume_client, SIZE, Some(SIZE), None).await; + + // test below the capacity limit + grpc_create_volume_with_limit(&volume_client, SIZE, Some(SIZE + EXCESS), None).await; +} + +async fn grpc_create_volume_with_limit( + volume_client: &dyn VolumeOperations, + size: u64, + capacity: Option, + expected_error: Option, +) { + let vol_uuid = Uuid::new_v4(); + + let volume = volume_client + .create( + &CreateVolume { + uuid: vol_uuid.try_into().unwrap(), + size, + replicas: 1, + cluster_capacity_limit: capacity, + ..Default::default() + }, + None, + ) + .await; + match volume { + Ok(_) => { + volume_client + .destroy( + &DestroyVolume { + uuid: vol_uuid.try_into().unwrap(), + }, + None, + ) + .await + .unwrap(); + assert!(expected_error.is_none()); + } + Err(e) => { + assert!(expected_error.is_some()); // expected an error + assert_eq!(expected_error.unwrap(), e.kind); // wrong error + } + } +} diff --git a/control-plane/agents/src/bin/core/tests/volume/mod.rs b/control-plane/agents/src/bin/core/tests/volume/mod.rs index 4cef3983a..0714f5fbd 100644 --- a/control-plane/agents/src/bin/core/tests/volume/mod.rs +++ b/control-plane/agents/src/bin/core/tests/volume/mod.rs @@ -2,6 +2,7 @@ mod affinity_group; mod capacity; +mod capacity_limit; mod garbage_collection; mod helpers; mod hotspare; diff --git a/control-plane/agents/src/bin/core/volume/operations.rs b/control-plane/agents/src/bin/core/volume/operations.rs index 9fc7ebd12..331ff9e84 100644 --- a/control-plane/agents/src/bin/core/volume/operations.rs +++ b/control-plane/agents/src/bin/core/volume/operations.rs @@ -721,7 +721,7 @@ impl ResourceLifecycleExt for OperationGuardArc { ) -> Result { let specs = registry.specs(); let mut volume = specs - .get_or_create_volume(&CreateVolumeSource::None(request)) + .get_or_create_volume(&CreateVolumeSource::None(request))? .operation_guard_wait() .await?; let volume_clone = volume.start_create(registry, request).await?; @@ -815,7 +815,7 @@ impl ResourceLifecycleExt> for OperationGuardArc ResourceMutex { + ) -> Result, SvcError> { let mut specs = self.write(); if let Some(volume) = specs.volumes.get(&request.source().uuid) { - volume.clone() + Ok(volume.clone()) } else { + // if request has a capacity limit, add up the volumes and reject + // if the capacity limit would be exceeded + match request.source().cluster_capacity_limit { + None => {} // no limit, no check needed + Some(limit) => { + let mut total: u64 = specs.volumes.values().map(|v| v.lock().size).sum(); + total += request.source().size; + if total > limit { + return Err(SvcError::CapacityLimitExceeded { + cluster_capacity_limit: limit, + excess: total - limit, + }); + } + } + } match request { CreateVolumeSource::None(_) => { - specs.volumes.insert(VolumeSpec::from(request.source())) + Ok(specs.volumes.insert(VolumeSpec::from(request.source()))) } CreateVolumeSource::Snapshot(create_from_snap) => { let mut spec = VolumeSpec::from(request.source()); spec.set_content_source(Some(create_from_snap.to_snapshot_source())); - specs.volumes.insert(spec) + Ok(specs.volumes.insert(spec)) } } } diff --git a/control-plane/agents/src/common/errors.rs b/control-plane/agents/src/common/errors.rs index 8ea2f3505..e3c3410d6 100644 --- a/control-plane/agents/src/common/errors.rs +++ b/control-plane/agents/src/common/errors.rs @@ -354,6 +354,15 @@ pub enum SvcError { DrainNotAllowedWhenHAisDisabled {}, #[snafu(display("Target switchover is not allowed without HA"))] SwitchoverNotAllowedWhenHAisDisabled {}, + #[snafu(display( + "The volume would exceed the capacity limit {} by {}", + cluster_capacity_limit, + excess + ))] + CapacityLimitExceeded { + cluster_capacity_limit: u64, + excess: u64, + }, } impl SvcError { @@ -966,6 +975,12 @@ impl From for ReplyError { source, extra, }, + SvcError::CapacityLimitExceeded { .. } => ReplyError { + kind: ReplyErrorKind::CapacityLimitExceeded, + resource: ResourceKind::Volume, + source, + extra, + }, } } } diff --git a/control-plane/grpc/proto/v1/misc/common.proto b/control-plane/grpc/proto/v1/misc/common.proto index a4dd4bda8..76115c2e9 100644 --- a/control-plane/grpc/proto/v1/misc/common.proto +++ b/control-plane/grpc/proto/v1/misc/common.proto @@ -67,6 +67,7 @@ enum ReplyErrorKind { ReplicaCreateNumber = 27; VolumeNoReplicas = 28; InUse = 29; + CapacityLimitExceeded = 30; } // ResourceKind for the resource which has undergone this error diff --git a/control-plane/grpc/proto/v1/volume/volume.proto b/control-plane/grpc/proto/v1/volume/volume.proto index 633b90c20..7f6e36535 100644 --- a/control-plane/grpc/proto/v1/volume/volume.proto +++ b/control-plane/grpc/proto/v1/volume/volume.proto @@ -263,6 +263,8 @@ message CreateVolumeRequest { bool thin = 8; // Affinity Group related information. optional AffinityGroup affinity_group = 9; + // maximum total volume size + optional uint64 cluster_capacity_limit = 10; } // Publish a volume on a node diff --git a/control-plane/grpc/src/misc/traits.rs b/control-plane/grpc/src/misc/traits.rs index e43d135cb..64acbd462 100644 --- a/control-plane/grpc/src/misc/traits.rs +++ b/control-plane/grpc/src/misc/traits.rs @@ -98,6 +98,7 @@ impl From for common::ReplyErrorKind { ReplyErrorKind::ReplicaCreateNumber => Self::ReplicaCreateNumber, ReplyErrorKind::VolumeNoReplicas => Self::VolumeNoReplicas, ReplyErrorKind::InUse => Self::InUse, + ReplyErrorKind::CapacityLimitExceeded => Self::CapacityLimitExceeded, } } } @@ -135,6 +136,7 @@ impl From for ReplyErrorKind { common::ReplyErrorKind::ReplicaCreateNumber => Self::ReplicaCreateNumber, common::ReplyErrorKind::VolumeNoReplicas => Self::VolumeNoReplicas, common::ReplyErrorKind::InUse => Self::InUse, + common::ReplyErrorKind::CapacityLimitExceeded => Self::CapacityLimitExceeded, } } } diff --git a/control-plane/grpc/src/operations/volume/traits.rs b/control-plane/grpc/src/operations/volume/traits.rs index 9b431e390..99f425e7d 100644 --- a/control-plane/grpc/src/operations/volume/traits.rs +++ b/control-plane/grpc/src/operations/volume/traits.rs @@ -890,6 +890,8 @@ pub trait CreateVolumeInfo: Send + Sync + std::fmt::Debug { fn thin(&self) -> bool; /// Affinity Group related information. fn affinity_group(&self) -> Option; + /// Capacity Limit, + fn cluster_capacity_limit(&self) -> Option; } impl CreateVolumeInfo for CreateVolume { @@ -924,6 +926,10 @@ impl CreateVolumeInfo for CreateVolume { fn affinity_group(&self) -> Option { self.affinity_group.clone() } + + fn cluster_capacity_limit(&self) -> Option { + self.cluster_capacity_limit + } } /// Intermediate structure that validates the conversion to CreateVolumeRequest type. @@ -972,6 +978,10 @@ impl CreateVolumeInfo for ValidatedCreateVolumeRequest { fn affinity_group(&self) -> Option { self.inner.affinity_group.clone().map(|ag| ag.into()) } + + fn cluster_capacity_limit(&self) -> Option { + self.inner.cluster_capacity_limit + } } impl ValidateRequestTypes for CreateVolumeRequest { @@ -1008,6 +1018,7 @@ impl From<&dyn CreateVolumeInfo> for CreateVolume { labels: data.labels(), thin: data.thin(), affinity_group: data.affinity_group(), + cluster_capacity_limit: data.cluster_capacity_limit(), } } } @@ -1025,6 +1036,7 @@ impl From<&dyn CreateVolumeInfo> for CreateVolumeRequest { .map(|labels| crate::common::StringMapValue { value: labels }), thin: data.thin(), affinity_group: data.affinity_group().map(|ag| ag.into()), + cluster_capacity_limit: data.cluster_capacity_limit(), } } } diff --git a/control-plane/rest/openapi-specs/v0_api_spec.yaml b/control-plane/rest/openapi-specs/v0_api_spec.yaml index fada405f7..661e7c090 100644 --- a/control-plane/rest/openapi-specs/v0_api_spec.yaml +++ b/control-plane/rest/openapi-specs/v0_api_spec.yaml @@ -2911,6 +2911,7 @@ components: - FailedPersist - Deleting - InUse + - CapacityLimitExceeded required: - details - kind @@ -3827,4 +3828,4 @@ components: content: application/json: schema: - $ref: '#/components/schemas/RestJsonError' \ No newline at end of file + $ref: '#/components/schemas/RestJsonError' diff --git a/control-plane/rest/src/versions/v0.rs b/control-plane/rest/src/versions/v0.rs index 34029fa6c..82e73a3a5 100644 --- a/control-plane/rest/src/versions/v0.rs +++ b/control-plane/rest/src/versions/v0.rs @@ -231,6 +231,7 @@ impl CreateVolumeBody { labels: self.labels.clone(), thin: self.thin, affinity_group: self.affinity_group.clone(), + cluster_capacity_limit: None, } } /// Convert into rpc request type. diff --git a/control-plane/stor-port/src/transport_api/mod.rs b/control-plane/stor-port/src/transport_api/mod.rs index 25f04a95a..c1ed772ac 100644 --- a/control-plane/stor-port/src/transport_api/mod.rs +++ b/control-plane/stor-port/src/transport_api/mod.rs @@ -442,6 +442,7 @@ pub enum ReplyErrorKind { ReplicaCreateNumber, VolumeNoReplicas, InUse, + CapacityLimitExceeded, } impl From for ReplyErrorKind { diff --git a/control-plane/stor-port/src/types/mod.rs b/control-plane/stor-port/src/types/mod.rs index 59c0b4212..51ec81df6 100644 --- a/control-plane/stor-port/src/types/mod.rs +++ b/control-plane/stor-port/src/types/mod.rs @@ -134,6 +134,10 @@ impl From for RestError { let error = RestJsonError::new(details, message, Kind::FailedPrecondition); (StatusCode::PRECONDITION_FAILED, error) } + ReplyErrorKind::CapacityLimitExceeded => { + let error = RestJsonError::new(details, message, Kind::CapacityLimitExceeded); + (StatusCode::INSUFFICIENT_STORAGE, error) + } }; RestError::new(status, error) diff --git a/control-plane/stor-port/src/types/v0/transport/volume.rs b/control-plane/stor-port/src/types/v0/transport/volume.rs index a4b589ace..9df52f64e 100644 --- a/control-plane/stor-port/src/types/v0/transport/volume.rs +++ b/control-plane/stor-port/src/types/v0/transport/volume.rs @@ -455,6 +455,8 @@ pub struct CreateVolume { pub thin: bool, /// Affinity Group related information. pub affinity_group: Option, + /// Maximum total system volume size. + pub cluster_capacity_limit: Option, } /// Resize volume request. diff --git a/utils/dependencies b/utils/dependencies index 2d25330c5..7b8da24e1 160000 --- a/utils/dependencies +++ b/utils/dependencies @@ -1 +1 @@ -Subproject commit 2d25330c5005b87d9491cddf7e0c2a4533392eb0 +Subproject commit 7b8da24e1d086911ccfa5fbb44a2cf536456d719