Merge #1590

1590: feat(nexus/resize): add support for resizing a nexus r=dsharma-dc a=dsharma-dc



Co-authored-by: Diwakar Sharma <diwakar.sharma@datacore.com>

io-engine-tests/src/nexus.rs

@@ -17,6 +17,7 @@ use super::{
             RebuildHistoryRecord,
             RebuildHistoryRequest,
             RemoveChildNexusRequest,
+            ResizeNexusRequest,
             ShutdownNexusRequest,
         },
         snapshot::SnapshotInfo,
@@ -250,6 +251,19 @@ impl NexusBuilder {
             .map(|r| r.into_inner().nexus.unwrap())
     }
 
+    pub async fn resize(&self, req_size: u64) -> Result<Nexus, Status> {
+        self.rpc()
+            .lock()
+            .await
+            .nexus
+            .resize_nexus(ResizeNexusRequest {
+                uuid: self.uuid(),
+                requested_size: req_size,
+            })
+            .await
+            .map(|r| r.into_inner().nexus.unwrap())
+    }
+
     pub async fn add_child(
         &self,
         bdev: &str,

io-engine-tests/src/replica.rs

@@ -11,6 +11,7 @@ use io_engine_api::v1::replica::{
     DestroyReplicaRequest,
     ListReplicaOptions,
     Replica,
+    ResizeReplicaRequest,
     ShareReplicaRequest,
 };
 
@@ -184,6 +185,22 @@ impl ReplicaBuilder {
         Ok(r)
     }
 
+    pub async fn resize(&mut self, req_size: u64) -> Result<Replica, Status> {
+        let r = self
+            .rpc()
+            .lock()
+            .await
+            .replica
+            .resize_replica(ResizeReplicaRequest {
+                uuid: self.uuid(),
+                requested_size: req_size,
+            })
+            .await
+            .map(|r| r.into_inner())?;
+        self.size = Some(r.size);
+        Ok(r)
+    }
+
     pub async fn get_replica(&self) -> Result<Replica, Status> {
         let uuid = self.uuid();
         list_replicas(self.rpc())

io-engine/src/bdev/nexus/nexus_bdev.rs

@@ -18,6 +18,7 @@ use std::{
 
 use crossbeam::atomic::AtomicCell;
 use futures::channel::oneshot;
+use nix::errno::Errno;
 use serde::Serialize;
 use snafu::ResultExt;
 use uuid::Uuid;
@@ -69,6 +70,7 @@ use crate::{
 use crate::core::{BlockDeviceIoStats, CoreError, IoCompletionStatus};
 use events_api::event::EventAction;
 use spdk_rs::{
+    libspdk::spdk_bdev_notify_blockcnt_change,
     BdevIo,
     BdevOps,
     ChannelTraverseStatus,
@@ -98,6 +100,7 @@ pub enum NexusOperation {
     ReplicaRemove,
     ReplicaOnline,
     ReplicaFault,
+    NexusResize,
     NexusSnapshot,
 }
 
@@ -681,7 +684,10 @@ impl<'n> Nexus<'n> {
     }
 
     /// Configure nexus's block device to match parameters of the child devices.
-    async fn setup_nexus_bdev(mut self: Pin<&mut Self>) -> Result<(), Error> {
+    async fn setup_nexus_bdev(
+        mut self: Pin<&mut Self>,
+        resizing: bool,
+    ) -> Result<(), Error> {
         let name = self.name.clone();
 
         if self.children().is_empty() {
@@ -725,7 +731,23 @@ impl<'n> Nexus<'n> {
             }
 
             match partition::calc_data_partition(self.req_size(), nb, bs) {
-                Some((start, end)) => {
+                Some((start, end, req_blocks)) => {
+                    // During expansion - if the requested number of blocks
+                    // aren't available on any child device,
+                    // then the operation needs to fail in its entirety. Hence
+                    // make sure that the `end` block number
+                    // returned is greater than the current end block number.
+                    // XXX: A shrink operation has to be taken care of later, if
+                    // needed.
+                    if resizing && (end <= (start + self.num_blocks())) {
+                        return Err(Error::ChildTooSmall {
+                            child: child.uri().to_owned(),
+                            name,
+                            num_blocks: nb,
+                            block_size: bs,
+                            req_blocks,
+                        });
+                    }
                     if start_blk == 0 {
                         start_blk = start;
                         end_blk = end;
@@ -746,6 +768,7 @@ impl<'n> Nexus<'n> {
                         name,
                         num_blocks: nb,
                         block_size: bs,
+                        req_blocks: self.req_size() / bs,
                     })
                 }
             }
@@ -754,15 +777,33 @@ impl<'n> Nexus<'n> {
         unsafe {
             self.as_mut().set_data_ent_offset(start_blk);
             self.as_mut().set_block_len(blk_size as u32);
-            self.as_mut().set_num_blocks(end_blk - start_blk);
+            let nbdev = self.as_mut().bdev_mut().unsafe_inner_mut_ptr();
+            if !resizing {
+                self.as_mut().set_num_blocks(end_blk - start_blk);
+            } else {
+                let rc = spdk_bdev_notify_blockcnt_change(
+                    nbdev,
+                    end_blk - start_blk,
+                );
+                if rc != 0 {
+                    error!(
+                        "{self:?}: failed to notify block cnt change on nexus"
+                    );
+                    return Err(Error::NexusResize {
+                        source: Errno::from_i32(rc),
+                        name,
+                    });
+                }
+            }
         }
 
         info!(
-            "{self:?}: nexus device initialized: \
+            "{self:?}: nexus device {action}: \
             requested={req_blk} blocks ({req} bytes) \
             start block={start_blk}, end block={end_blk}, \
             block size={blk_size}, \
             smallest devices size={min_dev_size} blocks",
+            action = if resizing { "resized" } else { "initialized" },
             req_blk = self.req_size() / blk_size,
             req = self.req_size(),
         );
@@ -781,7 +822,7 @@ impl<'n> Nexus<'n> {
 
         info!("{:?}: registering nexus bdev...", nex);
 
-        nex.as_mut().setup_nexus_bdev().await?;
+        nex.as_mut().setup_nexus_bdev(false).await?;
 
         // Register the bdev with SPDK and set the callbacks for io channel
         // creation.
@@ -893,6 +934,34 @@ impl<'n> Nexus<'n> {
         }
     }
 
+    /// Resize the nexus as part of volume resize workflow. The underlying
+    /// replicas are already resized before nexus resize is called.
+    pub async fn resize(
+        mut self: Pin<&mut Self>,
+        resize_to: u64,
+    ) -> Result<(), Error> {
+        // XXX: This check is likely relevant for resize as well to
+        // avoid unforeseen complications.
+        self.check_nexus_operation(NexusOperation::NexusResize)?;
+
+        let current_size = self.req_size();
+        if current_size == resize_to {
+            return Ok(());
+        }
+        info!(
+            "Resizing nexus {} from {current_size} to {resize_to}",
+            self.uuid()
+        );
+        unsafe { self.as_mut().set_req_size(resize_to) };
+        let ret = self.as_mut().setup_nexus_bdev(true).await;
+        if ret.is_err() {
+            // Reset the req_size back to original in case of failure.
+            unsafe { self.as_mut().set_req_size(current_size) };
+        }
+
+        ret
+    }
+
     /// Returns a mutable reference to Nexus I/O.
     fn io_subsystem_mut(self: Pin<&mut Self>) -> &mut NexusIoSubsystem<'n> {
         unsafe { self.get_unchecked_mut().io_subsystem.as_mut().unwrap() }
@@ -1256,6 +1325,11 @@ impl<'n> Nexus<'n> {
     pub(crate) unsafe fn set_data_ent_offset(self: Pin<&mut Self>, val: u64) {
         self.get_unchecked_mut().data_ent_offset = val;
     }
+
+    /// Sets the requested nexus size.
+    pub(crate) unsafe fn set_req_size(self: Pin<&mut Self>, val: u64) {
+        self.get_unchecked_mut().req_size = val;
+    }
 }
 
 impl Drop for Nexus<'_> {

io-engine/src/bdev/nexus/nexus_bdev_error.rs

@@ -67,17 +67,20 @@ pub enum Error {
     ))]
     NexusIncomplete { name: String, reason: String },
     #[snafu(display(
-        "Child {} of nexus {} is too small: size = {} x {}",
+        "Child {} of nexus {} is too small: size = {} x {}, required = {} x {}",
         child,
         name,
         num_blocks,
+        block_size,
+        req_blocks,
         block_size
     ))]
     ChildTooSmall {
         child: String,
         name: String,
         num_blocks: u64,
         block_size: u64,
+        req_blocks: u64,
     },
     #[snafu(display("Children of nexus {} have mixed block sizes", name))]
     MixedBlockSizes { name: String },
@@ -183,6 +186,8 @@ pub enum Error {
     NexusCreate { name: String, reason: String },
     #[snafu(display("Failed to destroy nexus {}", name))]
     NexusDestroy { name: String },
+    #[snafu(display("Failed to resize nexus {}", name))]
+    NexusResize { source: Errno, name: String },
     #[snafu(display(
         "Child {} of nexus {} is not degraded but {}",
         child,
@@ -281,6 +286,12 @@ impl From<Error> for tonic::Status {
             Error::RebuildJobNotFound {
                 ..
             } => Status::not_found(e.to_string()),
+            Error::NexusIncomplete {
+                ..
+            } => Status::failed_precondition(e.verbose()),
+            Error::NexusResize {
+                ..
+            } => Status::failed_precondition(e.to_string()),
             Error::NexusNotFound {
                 ..
             } => Status::not_found(e.to_string()),

io-engine/src/bin/io-engine-client/v1/nexus_cli.rs

@@ -196,6 +196,16 @@ pub fn subcommands() -> Command {
             .help("uuid of nexus"),
     );
 
+    let resize = Command::new("resize")
+        .about("Resize nexus")
+        .arg(Arg::new("uuid").required(true).index(1).help("Nexus uuid"))
+        .arg(
+            Arg::new("size")
+                .required(true)
+                .index(2)
+                .help("Requested new size of the nexus"),
+        );
+
     Command::new("nexus")
         .subcommand_required(true)
         .arg_required_else_help(true)
@@ -210,6 +220,7 @@ pub fn subcommands() -> Command {
         .subcommand(ana_state)
         .subcommand(list)
         .subcommand(children)
+        .subcommand(resize)
         .subcommand(nexus_child_cli::subcommands())
 }
 
@@ -220,6 +231,7 @@ pub async fn handler(ctx: Context, matches: &ArgMatches) -> crate::Result<()> {
         ("shutdown", args) => nexus_shutdown(ctx, args).await,
         ("list", args) => nexus_list(ctx, args).await,
         ("children", args) => nexus_children_2(ctx, args).await,
+        ("resize", args) => nexus_resize(ctx, args).await,
         ("publish", args) => nexus_publish(ctx, args).await,
         ("unpublish", args) => nexus_unpublish(ctx, args).await,
         ("ana_state", args) => nexus_nvme_ana_state(ctx, args).await,
@@ -564,6 +576,54 @@ async fn nexus_children_2(
     Ok(())
 }
 
+async fn nexus_resize(
+    mut ctx: Context,
+    matches: &ArgMatches,
+) -> crate::Result<()> {
+    let uuid = matches
+        .get_one::<String>("uuid")
+        .ok_or_else(|| ClientError::MissingValue {
+            field: "uuid".to_string(),
+        })?
+        .to_owned();
+
+    let requested_size =
+        parse_size(matches.get_one::<String>("size").ok_or_else(|| {
+            ClientError::MissingValue {
+                field: "size".to_string(),
+            }
+        })?)
+        .map_err(|s| Status::invalid_argument(format!("Bad size '{s}'")))
+        .context(GrpcStatus)?;
+
+    let response = ctx
+        .v1
+        .nexus
+        .resize_nexus(v1::nexus::ResizeNexusRequest {
+            uuid: uuid.clone(),
+            requested_size: requested_size.get_bytes() as u64,
+        })
+        .await
+        .context(GrpcStatus)?;
+
+    match ctx.output {
+        OutputFormat::Json => {
+            println!(
+                "{}",
+                serde_json::to_string_pretty(&response.get_ref())
+                    .unwrap()
+                    .to_colored_json_auto()
+                    .unwrap()
+            );
+        }
+        OutputFormat::Default => {
+            println!("Resized nexus {uuid} to {requested_size}");
+        }
+    };
+
+    Ok(())
+}
+
 async fn nexus_publish(
     mut ctx: Context,
     matches: &ArgMatches,

io-engine/src/core/partition.rs

@@ -50,7 +50,7 @@ pub fn calc_data_partition(
     req_size: u64,
     num_blocks: u64,
     block_size: u64,
-) -> Option<(u64, u64)> {
+) -> Option<(u64, u64, u64)> {
     // Number of blocks occupied by GPT tables.
     let gpt_blocks = bytes_to_alinged_blocks(GPT_TABLE_SIZE, block_size);
 
@@ -78,7 +78,7 @@ pub fn calc_data_partition(
     // Last data block.
     let data_end = min(data_start + req_blocks - 1, lba_end);
 
-    Some((data_start, data_end))
+    Some((data_start, data_end, req_blocks))
 }
 
 /// Converts an offset in bytes into offset in number of aligned blocks for the