Add WMF video playing with multi VideoPlayer sync (audio control isn'…

…t implemented)

Thanks to the DisplaySweet team https://github.com/DisplaySweet.

modules/wmf/SCsub

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+Import("env")
+env.Append(LIBS=["mfplat", "mf", "wmcodecdspuuid", "mfuuid", "mfreadwrite"])
+Import("env_modules")
+
+env_wmf = env_modules.Clone()
+
+env_wmf.add_source_files(env.modules_sources, "*.cpp")
+Export("env")

modules/wmf/config.py

@@ -0,0 +1,16 @@
+def can_build(env, platform):
+    return platform == "windows"
+
+
+def configure(env):
+    pass
+
+
+def get_doc_classes():
+    return [
+        "WindowsMediaFoundation",
+    ]
+
+
+def get_doc_path():
+    return "doc_classes"

modules/wmf/register_types.cpp

@@ -0,0 +1,27 @@
+#include "register_types.h"
+#include "resource_importer_wmf_video.h"
+#include "video_stream_wmf.h"
+
+#include "mfapi.h"
+#include <stdio.h>
+
+void initialize_wmf_module(ModuleInitializationLevel p_level) {
+	if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) {
+		return;
+	}
+	MFStartup(MF_VERSION, MFSTARTUP_FULL);
+
+#ifdef TOOLS_ENABLED
+	Ref<ResourceImporterWMFVideo> wmfv_import;
+	wmfv_import.instantiate();
+	ResourceFormatImporter::get_singleton()->add_importer(wmfv_import);
+#endif
+	ClassDB::register_class<VideoStreamWMF>();
+}
+
+void uninitialize_wmf_module(ModuleInitializationLevel p_level) {
+	if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) {
+		return;
+	}
+	MFShutdown();
+}

modules/wmf/register_types.h

@@ -0,0 +1,4 @@
+#include "modules/register_module_types.h"
+
+void initialize_wmf_module(ModuleInitializationLevel p_level);
+void uninitialize_wmf_module(ModuleInitializationLevel p_level);

modules/wmf/resource_importer_wmf_video.cpp

@@ -0,0 +1,54 @@
+#include "resource_importer_wmf_video.h"
+#include "video_stream_wmf.h"
+
+#include "core/io/file_access.h"
+#include "core/io/resource_saver.h"
+
+String ResourceImporterWMFVideo::get_importer_name() const {
+	return "WindowsMediaFoundation";
+}
+
+String ResourceImporterWMFVideo::get_visible_name() const {
+	return "WindowsMediaFoundation";
+}
+
+void ResourceImporterWMFVideo::get_recognized_extensions(List<String> *p_extensions) const {
+	p_extensions->push_back("mp4");
+	p_extensions->push_back("avi");
+	p_extensions->push_back("mkv");
+	p_extensions->push_back("webm");
+}
+
+String ResourceImporterWMFVideo::get_save_extension() const {
+	return "wmfvstr";
+}
+
+String ResourceImporterWMFVideo::get_resource_type() const {
+	return "VideoStreamWMF";
+}
+
+bool ResourceImporterWMFVideo::get_option_visibility(const String &p_path, const String &p_option, const HashMap<StringName, Variant> &p_options) const {
+	return true;
+}
+
+int ResourceImporterWMFVideo::get_preset_count() const {
+	return 0;
+}
+
+String ResourceImporterWMFVideo::get_preset_name(int p_idx) const {
+	return String();
+}
+
+void ResourceImporterWMFVideo::get_import_options(const String &p_path, List<ImportOption> *r_options, int p_preset) const {
+	r_options->push_back(ImportOption(PropertyInfo(Variant::BOOL, "loop"), false));
+}
+
+Error ResourceImporterWMFVideo::import(const String &p_source_file, const String &p_save_path, const HashMap<StringName, Variant> &p_options, List<String> *r_platform_variants, List<String> *r_gen_files, Variant *r_metadata) {
+	VideoStreamWMF *stream = memnew(VideoStreamWMF);
+	stream->set_file(p_source_file);
+	Ref<VideoStreamWMF> wmfv_stream = Ref<VideoStreamWMF>(stream);
+	return ResourceSaver::save(wmfv_stream, p_save_path + ".wmfvstr");
+}
+
+ResourceImporterWMFVideo::ResourceImporterWMFVideo() {
+}

modules/wmf/resource_importer_wmf_video.h

@@ -0,0 +1,26 @@
+#ifndef RESOURCEIMPORTERWMFVIDEO_H
+#define RESOURCEIMPORTERWMFVIDEO_H
+
+#include "core/io/resource_importer.h"
+
+class ResourceImporterWMFVideo : public ResourceImporter {
+	GDCLASS(ResourceImporterWMFVideo, ResourceImporter)
+
+public:
+	virtual String get_importer_name() const;
+	virtual String get_visible_name() const;
+	virtual void get_recognized_extensions(List<String> *p_extensions) const;
+	virtual String get_save_extension() const;
+	virtual String get_resource_type() const;
+
+	virtual int get_preset_count() const;
+	virtual String get_preset_name(int p_idx) const;
+	virtual void get_import_options(const String &p_path, List<ImportOption> *r_options, int p_preset = 0) const;
+	virtual bool get_option_visibility(const String &p_path, const String &p_option, const HashMap<StringName, Variant> &p_options) const;
+
+	virtual Error import(const String &p_source_file, const String &p_save_path, const HashMap<StringName, Variant> &p_options, List<String> *r_platform_variants, List<String> *r_gen_files = NULL, Variant *r_metadata = NULL);
+
+	ResourceImporterWMFVideo();
+};
+
+#endif

modules/wmf/sample_grabber_callback.cpp

@@ -0,0 +1,204 @@
+#include "sample_grabber_callback.h"
+#include "core/string/print_string.h"
+#include "video_stream_wmf.h"
+#include <Shlwapi.h>
+#include <mfapi.h>
+#include <minwindef.h>
+#include <cassert>
+#include <cstdio>
+#include <new>
+
+#define CHECK_HR(func)                                                         \
+	if (SUCCEEDED(hr)) {                                                       \
+		hr = (func);                                                           \
+		if (FAILED(hr)) {                                                      \
+			print_line(vformat("%s failed, return:", __FUNCTION__, itos(hr))); \
+		}                                                                      \
+	}
+
+SampleGrabberCallback::SampleGrabberCallback(VideoStreamPlaybackWMF *playback, Mutex &mtx) :
+		m_cRef(1), playback(playback), mtx(mtx) {
+}
+
+HRESULT SampleGrabberCallback::CreateInstance(SampleGrabberCallback **ppCB, VideoStreamPlaybackWMF *playback, Mutex &mtx) {
+	*ppCB = new (std::nothrow) SampleGrabberCallback(playback, mtx);
+
+	if (ppCB == nullptr) {
+		return E_OUTOFMEMORY;
+	}
+	return S_OK;
+}
+
+SampleGrabberCallback::~SampleGrabberCallback() {
+}
+
+STDMETHODIMP SampleGrabberCallback::QueryInterface(REFIID riid, void **ppv) {
+	static const QITAB qit[] = {
+#pragma clang diagnostic ignored "-Wc++11-narrowing"
+		QITABENT(SampleGrabberCallback, IMFSampleGrabberSinkCallback),
+#pragma clang diagnostic ignored "-Wc++11-narrowing"
+		QITABENT(SampleGrabberCallback, IMFClockStateSink),
+		{ 0 }
+	};
+	return QISearch(this, qit, riid, ppv);
+}
+
+STDMETHODIMP_(ULONG)
+SampleGrabberCallback::AddRef() {
+	return InterlockedIncrement(&m_cRef);
+}
+
+STDMETHODIMP_(ULONG)
+SampleGrabberCallback::Release() {
+	ULONG cRef = InterlockedDecrement(&m_cRef);
+	if (cRef == 0) {
+		delete this;
+	}
+	return cRef;
+}
+
+// IMFClockStateSink methods.
+
+// In these example, the IMFClockStateSink methods do not perform any actions.
+// You can use these methods to track the state of the sample grabber sink.
+
+STDMETHODIMP SampleGrabberCallback::OnClockStart(MFTIME hnsSystemTime, LONGLONG llClockStartOffset) {
+	return S_OK;
+}
+
+STDMETHODIMP SampleGrabberCallback::OnClockStop(MFTIME hnsSystemTime) {
+	return S_OK;
+}
+
+STDMETHODIMP SampleGrabberCallback::OnClockPause(MFTIME hnsSystemTime) {
+	return S_OK;
+}
+
+STDMETHODIMP SampleGrabberCallback::OnClockRestart(MFTIME hnsSystemTime) {
+	return S_OK;
+}
+
+STDMETHODIMP SampleGrabberCallback::OnClockSetRate(MFTIME hnsSystemTime, float flRate) {
+	return S_OK;
+}
+
+// IMFSampleGrabberSink methods.
+
+STDMETHODIMP SampleGrabberCallback::OnSetPresentationClock(IMFPresentationClock *pClock) {
+	return S_OK;
+}
+
+HRESULT SampleGrabberCallback::CreateMediaSample(DWORD cbData, IMFSample **ppSample) {
+	assert(ppSample);
+
+	HRESULT hr = S_OK;
+
+	IMFSample *pSample = nullptr;
+	IMFMediaBuffer *pBuffer = nullptr;
+
+	CHECK_HR(hr = MFCreateSample(&pSample));
+	CHECK_HR(hr = MFCreateMemoryBuffer(cbData, &pBuffer));
+	CHECK_HR(hr = pSample->AddBuffer(pBuffer));
+
+	*ppSample = pSample;
+	(*ppSample)->AddRef();
+
+	return hr;
+}
+
+STDMETHODIMP SampleGrabberCallback::OnProcessSample(REFGUID guidMajorMediaType,
+		DWORD dwSampleFlags,
+		LONGLONG llSampleTime,
+		LONGLONG llSampleDuration,
+		const BYTE *pSampleBuffer,
+		DWORD dwSampleSize) {
+	HRESULT hr = S_OK;
+	//assert(frame_data->size() == width * height * 3);
+
+	const int rgb24FrameSize = width * height * 3;
+	if (m_pSample == nullptr)
+		CreateMediaSample(dwSampleSize, &m_pSample);
+	if (m_pOutSample == nullptr)
+		CreateMediaSample(rgb24FrameSize, &m_pOutSample);
+
+	IMFMediaBuffer *pMediaBuffer = nullptr;
+	m_pSample->SetSampleTime(llSampleTime);
+	m_pSample->SetSampleDuration(llSampleDuration);
+	m_pSample->GetBufferByIndex(0, &pMediaBuffer);
+
+	BYTE *pData = nullptr;
+	pMediaBuffer->Lock(&pData, NULL, NULL);
+	{
+		memcpy(pData, pSampleBuffer, dwSampleSize);
+		hr = pMediaBuffer->SetCurrentLength(dwSampleSize);
+	}
+	pMediaBuffer->Unlock();
+
+	DWORD ProcessStatus;
+	CHECK_HR(m_pColorTransform->ProcessInput(0, m_pSample, 0));
+	if (FAILED(hr))
+		print_line("Failed to process video frames");
+
+	MFT_OUTPUT_DATA_BUFFER RGBOutputDataBuffer;
+	RGBOutputDataBuffer.dwStreamID = 0;
+	RGBOutputDataBuffer.dwStatus = 0;
+	RGBOutputDataBuffer.pEvents = NULL;
+	RGBOutputDataBuffer.pSample = m_pOutSample;
+	CHECK_HR(m_pColorTransform->ProcessOutput(0, 1, &RGBOutputDataBuffer, &ProcessStatus));
+	if (FAILED(hr))
+		print_line("Failed to process video frames");
+
+	IMFMediaBuffer *pOutputBuffer;
+	RGBOutputDataBuffer.pSample->GetBufferByIndex(0, &pOutputBuffer);
+
+	BYTE *outData;
+	DWORD outDataLen;
+	pOutputBuffer->Lock(&outData, NULL, &outDataLen);
+
+	//mtx.lock();
+	{
+		FrameData *frame = playback->get_next_writable_frame();
+		frame->sample_time = llSampleTime / 10000;
+		//print_line(itos(llSampleTime));
+
+		uint8_t *dst = frame->data.ptrw();
+
+		char *rgb_buffer = (char *)dst;
+		// convert 4 pixels at once
+		for (int i = 0; i < outDataLen; i += 12) {
+			rgb_buffer[i + 0] = outData[i + 2];
+			rgb_buffer[i + 1] = outData[i + 1];
+			rgb_buffer[i + 2] = outData[i + 0];
+
+			rgb_buffer[i + 3] = outData[i + 5];
+			rgb_buffer[i + 4] = outData[i + 4];
+			rgb_buffer[i + 5] = outData[i + 3];
+
+			rgb_buffer[i + 6] = outData[i + 8];
+			rgb_buffer[i + 7] = outData[i + 7];
+			rgb_buffer[i + 8] = outData[i + 6];
+
+			rgb_buffer[i + 9] = outData[i + 11];
+			rgb_buffer[i + 10] = outData[i + 10];
+			rgb_buffer[i + 11] = outData[i + 9];
+		}
+		//memcpy(rgb_buffer, outData, outDataLen);
+	}
+	//mtx.unlock();
+
+	pOutputBuffer->Unlock();
+
+	playback->write_frame_done();
+
+	return S_OK;
+}
+
+STDMETHODIMP SampleGrabberCallback::OnShutdown() {
+	print_line(__FUNCTION__);
+	return S_OK;
+}
+
+void SampleGrabberCallback::set_frame_size(int w, int h) {
+	width = w;
+	height = h;
+}