From a62891789596e85bfb5206a16664e2990235fde2 Mon Sep 17 00:00:00 2001 From: Chris Spencer Date: Mon, 31 Jul 2023 14:57:18 +0100 Subject: [PATCH] media_codec: Add support for asynchronous notification callbacks --- ndk/CHANGELOG.md | 1 + ndk/src/media/media_codec.rs | 311 +++++++++++++++++++++++++++++++---- 2 files changed, 279 insertions(+), 33 deletions(-) diff --git a/ndk/CHANGELOG.md b/ndk/CHANGELOG.md index 5103efb2..e44cd9cd 100644 --- a/ndk/CHANGELOG.md +++ b/ndk/CHANGELOG.md @@ -15,6 +15,7 @@ - hardware_buffer_format: Add `YCbCr_P010` and `R8_UNORM` variants. (#405) - **Breaking:** hardware_buffer_format: Add catch-all variant. (#407) - Add panic guards to callbacks. (#412) +- **Breaking:** media_codec: Add support for asynchronous notification callbacks. (#410) # 0.7.0 (2022-07-24) diff --git a/ndk/src/media/media_codec.rs b/ndk/src/media/media_codec.rs index b695579d..1f5d163f 100644 --- a/ndk/src/media/media_codec.rs +++ b/ndk/src/media/media_codec.rs @@ -5,11 +5,14 @@ use crate::media_error::{MediaError, Result}; use crate::native_window::NativeWindow; +use crate::utils::abort_on_panic; use std::{ convert::TryInto, - ffi::{CStr, CString}, + ffi::{c_char, c_void, CStr, CString}, + fmt, fmt::Display, mem::MaybeUninit, + pin::Pin, ptr::{self, NonNull}, slice, time::Duration, @@ -214,11 +217,73 @@ impl Drop for MediaFormat { /// A native [`AMediaCodec *`] /// /// [`AMediaCodec *`]: https://developer.android.com/ndk/reference/group/media#amediacodec -#[derive(Debug)] pub struct MediaCodec { inner: NonNull, + async_notify_callback: Option>>, } +impl fmt::Debug for MediaCodec { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("MediaCodec") + .field("inner", &self.inner) + .field( + "async_notify_callback", + match &self.async_notify_callback { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .finish() + } +} + +pub struct AsyncNotifyCallback { + pub on_input_available: Option, + pub on_output_available: Option, + pub on_format_changed: Option, + pub on_error: Option, +} + +impl fmt::Debug for AsyncNotifyCallback { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("AsyncNotifyCallback") + .field( + "on_input_available", + match &self.on_input_available { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_output_available", + match &self.on_output_available { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_format_changed", + match &self.on_format_changed { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_error", + match &self.on_error { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .finish() + } +} + +pub type InputAvailableCallback = Box; +pub type OutputAvailableCallback = Box; +pub type FormatChangedCallback = Box; +pub type ErrorCallback = Box, ActionCode, &CStr)>; + impl MediaCodec { fn as_ptr(&self) -> *mut ffi::AMediaCodec { self.inner.as_ptr() @@ -228,6 +293,7 @@ impl MediaCodec { let c_string = CString::new(name).unwrap(); Some(Self { inner: NonNull::new(unsafe { ffi::AMediaCodec_createCodecByName(c_string.as_ptr()) })?, + async_notify_callback: None, }) } @@ -237,6 +303,7 @@ impl MediaCodec { inner: NonNull::new(unsafe { ffi::AMediaCodec_createDecoderByType(c_string.as_ptr()) })?, + async_notify_callback: None, }) } @@ -246,9 +313,121 @@ impl MediaCodec { inner: NonNull::new(unsafe { ffi::AMediaCodec_createEncoderByType(c_string.as_ptr()) })?, + async_notify_callback: None, }) } + #[cfg(feature = "api-level-28")] + pub fn set_async_notify_callback( + &mut self, + callback: Option, + ) -> Result<()> { + unsafe extern "C" fn ffi_on_input_available( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + index: i32, + ) { + abort_on_panic(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_input_available.as_mut() { + f(index as usize); + } + }) + } + + unsafe extern "C" fn ffi_on_output_available( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + index: i32, + buffer_info: *mut ffi::AMediaCodecBufferInfo, + ) { + abort_on_panic(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_output_available.as_mut() { + let buffer_info = BufferInfo { + inner: *buffer_info, + }; + f(index as usize, &buffer_info); + } + }) + } + + unsafe extern "C" fn ffi_on_format_changed( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + format: *mut ffi::AMediaFormat, + ) { + abort_on_panic(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_format_changed.as_mut() { + let format = MediaFormat { + inner: NonNull::new_unchecked(format), + }; + f(&format); + } + }) + } + + unsafe extern "C" fn ffi_on_error( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + error: ffi::media_status_t, + action_code: i32, + detail: *const c_char, + ) { + abort_on_panic(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_error.as_mut() { + f( + MediaError::from_status(error), + ActionCode(action_code), + CStr::from_ptr(detail), + ); + } + }) + } + + let (callback, ffi_callback, user_data) = if let Some(callback) = callback { + // On Android 12 and earlier, due to faulty null checks, if a callback is not set, but at least one other + // callback *is* set, then it will segfault in NdkMediaCodec.cpp when trying to invoke the unset callback. + // To work around this we just enable all callbacks and do nothing if the corresponding callback is not set + // in AsyncNotifyCallback + let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback { + onAsyncInputAvailable: Some(ffi_on_input_available), + onAsyncOutputAvailable: Some(ffi_on_output_available), + onAsyncFormatChanged: Some(ffi_on_format_changed), + onAsyncError: Some(ffi_on_error), + }; + + let mut boxed = Box::pin(callback); + let ptr: *mut AsyncNotifyCallback = &mut *boxed; + + (Some(boxed), ffi_callback, ptr as *mut c_void) + } else { + let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback { + onAsyncInputAvailable: None, + onAsyncOutputAvailable: None, + onAsyncFormatChanged: None, + onAsyncError: None, + }; + + (None, ffi_callback, ptr::null_mut()) + }; + + let status = unsafe { + ffi::AMediaCodec_setAsyncNotifyCallback(self.as_ptr(), ffi_callback, user_data) + }; + let result = MediaError::from_status(status); + + if result.is_ok() { + self.async_notify_callback = callback; + } else { + self.async_notify_callback = None; + } + + result + } + pub fn configure( &self, format: &MediaFormat, @@ -342,7 +521,9 @@ impl MediaCodec { Ok(DequeuedOutputBufferInfoResult::Buffer(OutputBuffer { codec: self, index: result as usize, - info: unsafe { info.assume_init() }, + info: BufferInfo { + inner: unsafe { info.assume_init() }, + }, })) } else { Err(MediaError::from_status(ffi::media_status_t(result as _)).unwrap_err()) @@ -354,6 +535,28 @@ impl MediaCodec { MediaError::from_status(status) } + pub fn input_buffer(&self, index: usize) -> Option<&mut [MaybeUninit]> { + unsafe { + let mut out_size = 0; + let buffer_ptr = ffi::AMediaCodec_getInputBuffer(self.as_ptr(), index, &mut out_size); + if buffer_ptr.is_null() { + return None; + } + Some(slice::from_raw_parts_mut(buffer_ptr.cast(), out_size)) + } + } + + pub fn output_buffer(&self, index: usize) -> Option<&[u8]> { + unsafe { + let mut out_size = 0; + let buffer_ptr = ffi::AMediaCodec_getOutputBuffer(self.as_ptr(), index, &mut out_size); + if buffer_ptr.is_null() { + return None; + } + Some(slice::from_raw_parts(buffer_ptr, out_size)) + } + } + #[cfg(feature = "api-level-28")] pub fn input_format(&self) -> MediaFormat { let inner = NonNull::new(unsafe { ffi::AMediaCodec_getInputFormat(self.as_ptr()) }) @@ -386,11 +589,22 @@ impl MediaCodec { size: usize, time: u64, flags: u32, + ) -> Result<()> { + self.queue_input_buffer_by_index(buffer.index, offset, size, time, flags) + } + + pub fn queue_input_buffer_by_index( + &self, + buffer_index: usize, + offset: usize, + size: usize, + time: u64, + flags: u32, ) -> Result<()> { let status = unsafe { ffi::AMediaCodec_queueInputBuffer( self.as_ptr(), - buffer.index, + buffer_index, offset as ffi::off_t, size, time, @@ -401,8 +615,12 @@ impl MediaCodec { } pub fn release_output_buffer(&self, buffer: OutputBuffer, render: bool) -> Result<()> { + self.release_output_buffer_by_index(buffer.index, render) + } + + pub fn release_output_buffer_by_index(&self, buffer_index: usize, render: bool) -> Result<()> { let status = - unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer.index, render) }; + unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer_index, render) }; MediaError::from_status(status) } @@ -410,9 +628,17 @@ impl MediaCodec { &self, buffer: OutputBuffer, timestamp_ns: i64, + ) -> Result<()> { + self.release_output_buffer_at_time_by_index(buffer.index, timestamp_ns) + } + + pub fn release_output_buffer_at_time_by_index( + &self, + buffer_index: usize, + timestamp_ns: i64, ) -> Result<()> { let status = unsafe { - ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer.index, timestamp_ns) + ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer_index, timestamp_ns) }; MediaError::from_status(status) } @@ -468,17 +694,12 @@ pub struct InputBuffer<'a> { impl InputBuffer<'_> { pub fn buffer_mut(&mut self) -> &mut [MaybeUninit] { - unsafe { - let mut out_size = 0; - let buffer_ptr = - ffi::AMediaCodec_getInputBuffer(self.codec.as_ptr(), self.index, &mut out_size); - assert!( - !buffer_ptr.is_null(), + self.codec.input_buffer(self.index).unwrap_or_else(|| { + panic!( "AMediaCodec_getInputBuffer returned NULL for index {}", self.index - ); - slice::from_raw_parts_mut(buffer_ptr.cast(), out_size) - } + ) + }) } } @@ -492,25 +713,17 @@ pub enum DequeuedInputBufferResult<'a> { pub struct OutputBuffer<'a> { codec: &'a MediaCodec, index: usize, - info: ffi::AMediaCodecBufferInfo, + info: BufferInfo, } impl OutputBuffer<'_> { pub fn buffer(&self) -> &[u8] { - unsafe { - let mut _out_size = 0; - let buffer_ptr = - ffi::AMediaCodec_getOutputBuffer(self.codec.as_ptr(), self.index, &mut _out_size); - assert!( - !buffer_ptr.is_null(), + self.codec.output_buffer(self.index).unwrap_or_else(|| { + panic!( "AMediaCodec_getOutputBuffer returned NULL for index {}", self.index - ); - slice::from_raw_parts( - buffer_ptr.add(self.info.offset as usize), - self.info.size as usize, ) - } + }) } #[cfg(feature = "api-level-28")] @@ -522,12 +735,8 @@ impl OutputBuffer<'_> { MediaFormat { inner } } - pub fn flags(&self) -> u32 { - self.info.flags - } - - pub fn presentation_time_us(&self) -> i64 { - self.info.presentationTimeUs + pub fn info(&self) -> &BufferInfo { + &self.info } } @@ -538,3 +747,39 @@ pub enum DequeuedOutputBufferInfoResult<'a> { OutputFormatChanged, OutputBuffersChanged, } + +#[derive(Copy, Clone, Debug)] +pub struct BufferInfo { + inner: ffi::AMediaCodecBufferInfo, +} + +impl BufferInfo { + pub fn offset(&self) -> i32 { + self.inner.offset + } + + pub fn size(&self) -> i32 { + self.inner.size + } + + pub fn presentation_time_us(&self) -> i64 { + self.inner.presentationTimeUs + } + + pub fn flags(&self) -> u32 { + self.inner.flags + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ActionCode(pub i32); + +impl ActionCode { + pub fn is_recoverable(&self) -> bool { + unsafe { ffi::AMediaCodecActionCode_isRecoverable(self.0) } + } + + pub fn is_transient(&self) -> bool { + unsafe { ffi::AMediaCodecActionCode_isTransient(self.0) } + } +}