Skip to content

Commit

Permalink
Auto merge of #122079 - tbu-:pr_copy_file_range_probe, r=the8472
Browse files Browse the repository at this point in the history
Less syscalls for the `copy_file_range` probe

If it's obvious from the actual syscall results themselves that the syscall is supported or unsupported, don't do an extra syscall with an invalid file descriptor.

CC #122052
  • Loading branch information
bors committed May 26, 2024
2 parents 785eb65 + 7d67ee5 commit b092569
Showing 1 changed file with 48 additions and 19 deletions.
67 changes: 48 additions & 19 deletions library/std/src/sys/pal/unix/kernel_copy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
// We store the availability in a global to avoid unnecessary syscalls
static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);

let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => false,
UNAVAILABLE => return CopyResult::Fallback(0),
_ => true,
};

syscall! {
fn copy_file_range(
fd_in: libc::c_int,
Expand All @@ -571,25 +577,22 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
) -> libc::ssize_t
}

match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => {
// EPERM can indicate seccomp filters or an immutable file.
// To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported
// and some other error (ENOSYS or EPERM) if it's not available
let result = unsafe {
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
};

if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) {
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
} else {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
return CopyResult::Fallback(0);
}
fn probe_copy_file_range_support() -> u8 {
// In some cases, we cannot determine availability from the first
// `copy_file_range` call. In this case, we probe with an invalid file
// descriptor so that the results are easily interpretable.
match unsafe {
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
.map_err(|e| e.raw_os_error())
} {
Err(Some(EPERM | ENOSYS)) => UNAVAILABLE,
Err(Some(EBADF)) => AVAILABLE,
Ok(_) => panic!("unexpected copy_file_range probe success"),
// Treat other errors as the syscall
// being unavailable.
Err(_) => UNAVAILABLE,
}
UNAVAILABLE => return CopyResult::Fallback(0),
_ => {}
};
}

let mut written = 0u64;
while written < max_len {
Expand All @@ -604,6 +607,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
};

if !have_probed && copy_result.is_ok() {
have_probed = true;
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
}

match copy_result {
Ok(0) if written == 0 => {
// fallback to work around several kernel bugs where copy_file_range will fail to
Expand All @@ -619,7 +627,28 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
return match err.raw_os_error() {
// when file offset + max_length > u64::MAX
Some(EOVERFLOW) => CopyResult::Fallback(written),
Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => {
Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF))
if written == 0 =>
{
if !have_probed {
let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) {
// EPERM can indicate seccomp filters or an
// immutable file. To distinguish these
// cases we probe with invalid file
// descriptors which should result in EBADF
// if the syscall is supported and EPERM or
// ENOSYS if it's not available.
//
// For EOPNOTSUPP, see below. In the case of
// ENOSYS, we try to cover for faulty FUSE
// drivers.
probe_copy_file_range_support()
} else {
AVAILABLE
};
HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed);
}

// Try fallback io::copy if either:
// - Kernel version is < 4.5 (ENOSYS¹)
// - Files are mounted on different fs (EXDEV)
Expand Down

0 comments on commit b092569

Please sign in to comment.