Skip to content

Commit

Permalink
media_codec: Add support for asynchronous notification callbacks (#410)
Browse files Browse the repository at this point in the history
  • Loading branch information
@spencercw
spencercw committed Jul 31, 2023
1 parent 563089a commit e51eac8
Show file tree
Hide file tree
Showing 2 changed files with 278 additions and 5 deletions.
1 change: 1 addition & 0 deletions ndk/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
- native_window: Add `lock()` to blit raw pixel data. (#404)
- hardware_buffer_format: Add `YCbCr_P010` and `R8_UNORM` variants. (#405)
- **Breaking:** hardware_buffer_format: Add catch-all variant. (#407)
- media_codec: Add support for asynchronous notification callbacks. (#410)

# 0.7.0 (2022-07-24)

Expand Down
282 changes: 277 additions & 5 deletions ndk/src/media/media_codec.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@ use crate::media_error::{MediaError, Result};
use crate::native_window::NativeWindow;
use std::{
convert::TryInto,
ffi::{CStr, CString},
ffi::{c_char, c_void, CStr, CString},
fmt,
fmt::Display,
mem::MaybeUninit,
panic,
process::abort,
ptr::{self, NonNull},
slice,
time::Duration,
Expand Down Expand Up @@ -214,11 +217,73 @@ impl Drop for MediaFormat {
/// A native [`AMediaCodec *`]
///
/// [`AMediaCodec *`]: https://developer.android.com/ndk/reference/group/media#amediacodec
#[derive(Debug)]
pub struct MediaCodec {
inner: NonNull<ffi::AMediaCodec>,
async_notify_callback: Option<Box<AsyncNotifyCallback>>,
}

impl fmt::Debug for MediaCodec {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("MediaCodec")
.field("inner", &self.inner)
.field(
"async_notify_callback",
match &self.async_notify_callback {
Some(_) => &"Some(_)",
None => &"None",
},
)
.finish()
}
}

pub struct AsyncNotifyCallback {
pub on_input_available: Option<InputAvailableCallback>,
pub on_output_available: Option<OutputAvailableCallback>,
pub on_format_changed: Option<FormatChangedCallback>,
pub on_error: Option<ErrorCallback>,
}

impl fmt::Debug for AsyncNotifyCallback {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("AsyncNotifyCallback")
.field(
"on_input_available",
match &self.on_input_available {
Some(_) => &"Some(_)",
None => &"None",
},
)
.field(
"on_output_available",
match &self.on_output_available {
Some(_) => &"Some(_)",
None => &"None",
},
)
.field(
"on_format_changed",
match &self.on_format_changed {
Some(_) => &"Some(_)",
None => &"None",
},
)
.field(
"on_error",
match &self.on_error {
Some(_) => &"Some(_)",
None => &"None",
},
)
.finish()
}
}

pub type InputAvailableCallback = Box<dyn FnMut(usize)>;
pub type OutputAvailableCallback = Box<dyn FnMut(usize, &BufferInfo)>;
pub type FormatChangedCallback = Box<dyn FnMut(&MediaFormat)>;
pub type ErrorCallback = Box<dyn FnMut(Result<()>, ActionCode, &CStr)>;

impl MediaCodec {
fn as_ptr(&self) -> *mut ffi::AMediaCodec {
self.inner.as_ptr()
Expand All @@ -228,6 +293,7 @@ impl MediaCodec {
let c_string = CString::new(name).unwrap();
Some(Self {
inner: NonNull::new(unsafe { ffi::AMediaCodec_createCodecByName(c_string.as_ptr()) })?,
async_notify_callback: None,
})
}

Expand All @@ -237,6 +303,7 @@ impl MediaCodec {
inner: NonNull::new(unsafe {
ffi::AMediaCodec_createDecoderByType(c_string.as_ptr())
})?,
async_notify_callback: None,
})
}

Expand All @@ -246,9 +313,133 @@ impl MediaCodec {
inner: NonNull::new(unsafe {
ffi::AMediaCodec_createEncoderByType(c_string.as_ptr())
})?,
async_notify_callback: None,
})
}

#[cfg(feature = "api-level-28")]
pub fn set_async_notify_callback(
&mut self,
callback: Option<AsyncNotifyCallback>,
) -> Result<()> {
unsafe extern "C" fn ffi_on_input_available(
_codec: *mut ffi::AMediaCodec,
user_data: *mut c_void,
index: i32,
) {
match panic::catch_unwind(|| {
let callback = &mut *(user_data as *mut AsyncNotifyCallback);
if let Some(f) = callback.on_input_available.as_mut() {
f(index as usize);
}
}) {
Ok(result) => result,
Err(_) => abort(),
}
}

unsafe extern "C" fn ffi_on_output_available(
_codec: *mut ffi::AMediaCodec,
user_data: *mut c_void,
index: i32,
buffer_info: *mut ffi::AMediaCodecBufferInfo,
) {
match panic::catch_unwind(|| {
let callback = &mut *(user_data as *mut AsyncNotifyCallback);
if let Some(f) = callback.on_output_available.as_mut() {
let buffer_info = BufferInfo {
inner: *buffer_info,
};
f(index as usize, &buffer_info);
}
}) {
Ok(result) => result,
Err(_) => abort(),
}
}

unsafe extern "C" fn ffi_on_format_changed(
_codec: *mut ffi::AMediaCodec,
user_data: *mut c_void,
format: *mut ffi::AMediaFormat,
) {
match panic::catch_unwind(|| {
let callback = &mut *(user_data as *mut AsyncNotifyCallback);
if let Some(f) = callback.on_format_changed.as_mut() {
let format = MediaFormat {
inner: NonNull::new_unchecked(format),
};
f(&format);
}
}) {
Ok(result) => result,
Err(_) => abort(),
}
}

unsafe extern "C" fn ffi_on_error(
_codec: *mut ffi::AMediaCodec,
user_data: *mut c_void,
error: ffi::media_status_t,
action_code: i32,
detail: *const c_char,
) {
match panic::catch_unwind(|| {
let callback = &mut *(user_data as *mut AsyncNotifyCallback);
if let Some(f) = callback.on_error.as_mut() {
f(
MediaError::from_status(error),
ActionCode(action_code),
CStr::from_ptr(detail),
);
}
}) {
Ok(result) => result,
Err(_) => abort(),
}
}

let (callback, ffi_callback, user_data) = if let Some(callback) = callback {
// On Android 12 and earlier, due to faulty null checks, if a callback is not set, but at least one other
// callback *is* set, then it will segfault in NdkMediaCodec.cpp when trying to invoke the unset callback.
// To work around this we just enable all callbacks and do nothing if the corresponding callback is not set
// in AsyncNotifyCallback
let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback {
onAsyncInputAvailable: Some(ffi_on_input_available),
onAsyncOutputAvailable: Some(ffi_on_output_available),
onAsyncFormatChanged: Some(ffi_on_format_changed),
onAsyncError: Some(ffi_on_error),
};

let mut boxed = Box::new(callback);
let ptr: *mut AsyncNotifyCallback = &mut *boxed;

(Some(boxed), ffi_callback, ptr as *mut c_void)
} else {
let ffi_callback = ffi::AMediaCodecOnAsyncNotifyCallback {
onAsyncInputAvailable: None,
onAsyncOutputAvailable: None,
onAsyncFormatChanged: None,
onAsyncError: None,
};

(None, ffi_callback, ptr::null_mut())
};

let status = unsafe {
ffi::AMediaCodec_setAsyncNotifyCallback(self.as_ptr(), ffi_callback, user_data)
};
let result = MediaError::from_status(status);

if result.is_ok() {
self.async_notify_callback = callback;
} else {
self.async_notify_callback = None;
}

result
}

pub fn configure(
&self,
format: &MediaFormat,
Expand Down Expand Up @@ -354,6 +545,28 @@ impl MediaCodec {
MediaError::from_status(status)
}

pub fn input_buffer(&self, index: usize) -> Option<&mut [u8]> {
unsafe {
let mut out_size = 0;
let buffer_ptr = ffi::AMediaCodec_getInputBuffer(self.as_ptr(), index, &mut out_size);
if buffer_ptr.is_null() {
return None;
}
Some(slice::from_raw_parts_mut(buffer_ptr, out_size))
}
}

pub fn output_buffer(&self, index: usize) -> Option<&[u8]> {
unsafe {
let mut out_size = 0;
let buffer_ptr = ffi::AMediaCodec_getOutputBuffer(self.as_ptr(), index, &mut out_size);
if buffer_ptr.is_null() {
return None;
}
Some(slice::from_raw_parts(buffer_ptr, out_size))
}
}

#[cfg(feature = "api-level-28")]
pub fn input_format(&self) -> MediaFormat {
let inner = NonNull::new(unsafe { ffi::AMediaCodec_getInputFormat(self.as_ptr()) })
Expand Down Expand Up @@ -386,11 +599,22 @@ impl MediaCodec {
size: usize,
time: u64,
flags: u32,
) -> Result<()> {
self.queue_input_buffer_by_index(buffer.index, offset, size, time, flags)
}

pub fn queue_input_buffer_by_index(
&self,
buffer_index: usize,
offset: usize,
size: usize,
time: u64,
flags: u32,
) -> Result<()> {
let status = unsafe {
ffi::AMediaCodec_queueInputBuffer(
self.as_ptr(),
buffer.index,
buffer_index,
offset as ffi::off_t,
size,
time,
Expand All @@ -401,18 +625,30 @@ impl MediaCodec {
}

pub fn release_output_buffer(&self, buffer: OutputBuffer, render: bool) -> Result<()> {
self.release_output_buffer_by_index(buffer.index, render)
}

pub fn release_output_buffer_by_index(&self, buffer_index: usize, render: bool) -> Result<()> {
let status =
unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer.index, render) };
unsafe { ffi::AMediaCodec_releaseOutputBuffer(self.as_ptr(), buffer_index, render) };
MediaError::from_status(status)
}

pub fn release_output_buffer_at_time(
&self,
buffer: OutputBuffer,
timestamp_ns: i64,
) -> Result<()> {
self.release_output_buffer_at_time_by_index(buffer.index, timestamp_ns)
}

pub fn release_output_buffer_at_time_by_index(
&self,
buffer_index: usize,
timestamp_ns: i64,
) -> Result<()> {
let status = unsafe {
ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer.index, timestamp_ns)
ffi::AMediaCodec_releaseOutputBufferAtTime(self.as_ptr(), buffer_index, timestamp_ns)
};
MediaError::from_status(status)
}
Expand Down Expand Up @@ -538,3 +774,39 @@ pub enum DequeuedOutputBufferInfoResult<'a> {
OutputFormatChanged,
OutputBuffersChanged,
}

#[derive(Copy, Clone, Debug)]
pub struct BufferInfo {
inner: ffi::AMediaCodecBufferInfo,
}

impl BufferInfo {
pub fn offset(&self) -> i32 {
self.inner.offset
}

pub fn size(&self) -> i32 {
self.inner.size
}

pub fn presentation_time_us(&self) -> i64 {
self.inner.presentationTimeUs
}

pub fn flags(&self) -> u32 {
self.inner.flags
}
}

#[derive(Copy, Clone, Debug)]
pub struct ActionCode(pub i32);

impl ActionCode {
pub fn is_recoverable(&self) -> bool {
unsafe { ffi::AMediaCodecActionCode_isRecoverable(self.0) }
}

pub fn is_transient(&self) -> bool {
unsafe { ffi::AMediaCodecActionCode_isTransient(self.0) }
}
}

0 comments on commit e51eac8

Please sign in to comment.