From 6e1607ef127f14ac464c1c5802035bfb3a4bb005 Mon Sep 17 00:00:00 2001 From: Chris Spencer Date: Mon, 31 Jul 2023 14:57:18 +0100 Subject: [PATCH] media_codec: Add support for asynchronous notification callbacks (#410) --- ndk/CHANGELOG.md | 1 + ndk/src/media/media_codec.rs | 277 ++++++++++++++++++++++++++++++++++- 2 files changed, 273 insertions(+), 5 deletions(-) diff --git a/ndk/CHANGELOG.md b/ndk/CHANGELOG.md index 742d5ea8..3c7c00af 100644 --- a/ndk/CHANGELOG.md +++ b/ndk/CHANGELOG.md @@ -14,6 +14,7 @@ - native_window: Add `lock()` to blit raw pixel data. (#404) - hardware_buffer_format: Add `YCbCr_P010` and `R8_UNORM` variants. (#405) - **Breaking:** hardware_buffer_format: Add catch-all variant. (#407) +- media_codec: Add support for asynchronous notification callbacks. (#410) # 0.7.0 (2022-07-24) diff --git a/ndk/src/media/media_codec.rs b/ndk/src/media/media_codec.rs index b695579d..39efec5a 100644 --- a/ndk/src/media/media_codec.rs +++ b/ndk/src/media/media_codec.rs @@ -7,9 +7,12 @@ use crate::media_error::{MediaError, Result}; use crate::native_window::NativeWindow; use std::{ convert::TryInto, - ffi::{CStr, CString}, + ffi::{c_char, c_void, CStr, CString}, + fmt, fmt::Display, mem::MaybeUninit, + panic, + process::abort, ptr::{self, NonNull}, slice, time::Duration, @@ -214,9 +217,66 @@ impl Drop for MediaFormat { /// A native [`AMediaCodec *`] /// /// [`AMediaCodec *`]: https://developer.android.com/ndk/reference/group/media#amediacodec -#[derive(Debug)] pub struct MediaCodec { inner: NonNull, + async_notify_callback: Option>, +} + +impl fmt::Debug for MediaCodec { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("MediaCodec") + .field("inner", &self.inner) + .field( + "async_notify_callback", + match &self.async_notify_callback { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .finish() + } +} + +pub struct AsyncNotifyCallback { + pub on_input_available: Option>, + pub on_output_available: Option>, + pub on_format_changed: Option>, + pub on_error: Option, ActionCode, &CStr)>>, +} + +impl fmt::Debug for AsyncNotifyCallback { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("AsyncNotifyCallback") + .field( + "on_input_available", + match &self.on_input_available { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_output_available", + match &self.on_output_available { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_format_changed", + match &self.on_format_changed { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .field( + "on_error", + match &self.on_error { + Some(_) => &"Some(_)", + None => &"None", + }, + ) + .finish() + } } impl MediaCodec { @@ -228,6 +288,7 @@ impl MediaCodec { let c_string = CString::new(name).unwrap(); Some(Self { inner: NonNull::new(unsafe { ffi::AMediaCodec_createCodecByName(c_string.as_ptr()) })?, + async_notify_callback: None, }) } @@ -237,6 +298,7 @@ impl MediaCodec { inner: NonNull::new(unsafe { ffi::AMediaCodec_createDecoderByType(c_string.as_ptr()) })?, + async_notify_callback: None, }) } @@ -246,9 +308,133 @@ impl MediaCodec { inner: NonNull::new(unsafe { ffi::AMediaCodec_createEncoderByType(c_string.as_ptr()) })?, + async_notify_callback: None, }) } + #[cfg(feature = "api-level-28")] + pub fn set_async_notify_callback( + &mut self, + callback: Option, + ) -> Result<()> { + unsafe extern "C" fn ffi_on_input_available( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + index: i32, + ) { + match panic::catch_unwind(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_input_available.as_mut() { + f(index as usize); + } + }) { + Ok(result) => result, + Err(_) => abort(), + } + } + + unsafe extern "C" fn ffi_on_output_available( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + index: i32, + buffer_info: *mut ffi::AMediaCodecBufferInfo, + ) { + match panic::catch_unwind(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_output_available.as_mut() { + let buffer_info = BufferInfo { + inner: *buffer_info, + }; + f(index as usize, &buffer_info); + } + }) { + Ok(result) => result, + Err(_) => abort(), + } + } + + unsafe extern "C" fn ffi_on_format_changed( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + format: *mut ffi::AMediaFormat, + ) { + match panic::catch_unwind(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_format_changed.as_mut() { + let format = MediaFormat { + inner: NonNull::new_unchecked(format), + }; + f(&format); + } + }) { + Ok(result) => result, + Err(_) => abort(), + } + } + + unsafe extern "C" fn ffi_on_error( + _codec: *mut ffi::AMediaCodec, + user_data: *mut c_void, + error: ffi::media_status_t, + action_code: i32, + detail: *const c_char, + ) { + match panic::catch_unwind(|| { + let callback = &mut *(user_data as *mut AsyncNotifyCallback); + if let Some(f) = callback.on_error.as_mut() { + f( + MediaError::from_status(error), + ActionCode(action_code), + CStr::from_ptr(detail), + ); + } + }) { + Ok(result) => result, + Err(_) => abort(), + } + } + + let (callback, ffi_callback, user_data) = if let Some(callback) = callback { + // On Android 12 and earlier, due to faulty null checks, if a callback is not set, but at least one other + // callback *is* set, then it will segfault in NdkMediaCodec.cpp when trying to invoke the unset callback. + // To work around this we just enable all callbacks and do nothing if the corresponding callback is not set + // in AsyncNotifyCallback + let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback { + onAsyncInputAvailable: Some(ffi_on_input_available), + onAsyncOutputAvailable: Some(ffi_on_output_available), + onAsyncFormatChanged: Some(ffi_on_format_changed), + onAsyncError: Some(ffi_on_error), + }; + + let mut boxed = Box::new(callback); + let ptr: *mut AsyncNotifyCallback = &mut *boxed; + + (Some(boxed), ffi_callback, ptr as *mut c_void) + } else { + let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback { + onAsyncInputAvailable: None, + onAsyncOutputAvailable: None, + onAsyncFormatChanged: None, + onAsyncError: None, + }; + + (None, ffi_callback, ptr::null_mut()) + }; + + let status = unsafe { + ffi::AMediaCodec_setAsyncNotifyCallback(self.as_ptr(), ffi_callback, user_data) + }; + let result = MediaError::from_status(status); + + if result.is_ok() { + self.async_notify_callback = callback; + } else { + self.async_notify_callback = None; + } + + result + } + pub fn configure( &self, format: &MediaFormat, @@ -354,6 +540,28 @@ impl MediaCodec { MediaError::from_status(status) } + pub fn input_buffer(&self, index: usize) -> Option<&mut [u8]> { + unsafe { + let mut out_size = 0; + let buffer_ptr = ffi::AMediaCodec_getInputBuffer(self.as_ptr(), index, &mut out_size); + if buffer_ptr.is_null() { + return None; + } + Some(slice::from_raw_parts_mut(buffer_ptr, out_size)) + } + } + + pub fn output_buffer(&self, index: usize) -> Option<&[u8]> { + unsafe { + let mut out_size = 0; + let buffer_ptr = ffi::AMediaCodec_getOutputBuffer(self.as_ptr(), index, &mut out_size); + if buffer_ptr.is_null() { + return None; + } + Some(slice::from_raw_parts(buffer_ptr, out_size)) + } + } + #[cfg(feature = "api-level-28")] pub fn input_format(&self) -> MediaFormat { let inner = NonNull::new(unsafe { ffi::AMediaCodec_getInputFormat(self.as_ptr()) }) @@ -386,11 +594,22 @@ impl MediaCodec { size: usize, time: u64, flags: u32, + ) -> Result<()> { + self.queue_input_buffer_by_index(buffer.index, offset, size, time, flags) + } + + pub fn queue_input_buffer_by_index( + &self, + buffer_index: usize, + offset: usize, + size: usize, + time: u64, + flags: u32, ) -> Result<()> { let status = unsafe { ffi::AMediaCodec_queueInputBuffer( self.as_ptr(), - buffer.index, + buffer_index, offset as ffi::off_t, size, time, @@ -401,8 +620,12 @@ impl MediaCodec { } pub fn release_output_buffer(&self, buffer: OutputBuffer, render: bool) -> Result<()> { + self.release_output_buffer_by_index(buffer.index, render) + } + + pub fn release_output_buffer_by_index(&self, buffer_index: usize, render: bool) -> Result<()> { let status = - unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer.index, render) }; + unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer_index, render) }; MediaError::from_status(status) } @@ -410,9 +633,17 @@ impl MediaCodec { &self, buffer: OutputBuffer, timestamp_ns: i64, + ) -> Result<()> { + self.release_output_buffer_at_time_by_index(buffer.index, timestamp_ns) + } + + pub fn release_output_buffer_at_time_by_index( + &self, + buffer_index: usize, + timestamp_ns: i64, ) -> Result<()> { let status = unsafe { - ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer.index, timestamp_ns) + ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer_index, timestamp_ns) }; MediaError::from_status(status) } @@ -538,3 +769,39 @@ pub enum DequeuedOutputBufferInfoResult<'a> { OutputFormatChanged, OutputBuffersChanged, } + +#[derive(Copy, Clone, Debug)] +pub struct BufferInfo { + inner: ffi::AMediaCodecBufferInfo, +} + +impl BufferInfo { + pub fn offset(&self) -> i32 { + self.inner.offset + } + + pub fn size(&self) -> i32 { + self.inner.size + } + + pub fn presentation_time_us(&self) -> i64 { + self.inner.presentationTimeUs + } + + pub fn flags(&self) -> u32 { + self.inner.flags + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ActionCode(pub i32); + +impl ActionCode { + pub fn is_recoverable(&self) -> bool { + unsafe { ffi::AMediaCodecActionCode_isRecoverable(self.0) } + } + + pub fn is_transient(&self) -> bool { + unsafe { ffi::AMediaCodecActionCode_isTransient(self.0) } + } +}