diff --git a/src/esp/bindings/SimBindings.cpp b/src/esp/bindings/SimBindings.cpp index 4cd623243c..df21e32b79 100644 --- a/src/esp/bindings/SimBindings.cpp +++ b/src/esp/bindings/SimBindings.cpp @@ -438,12 +438,16 @@ void initSimBindings(py::module& m) { .def( "render", [](AbstractReplayRenderer& self, - std::vector images) { - self.render(images); + std::vector colorImageViews, + std::vector depthImageViews) { + self.render(colorImageViews, depthImageViews); }, - R"(Render color sensors into the specified image vector (one per environment). - The images are required to be pre-allocated. - Blocks the thread during the GPU-to-CPU memory transfer operation.)") + R"(Render sensors into the specified image vectors (one per environment). + Blocks the thread during the GPU-to-CPU memory transfer operation. + Empty lists can be supplied to skip the copying render targets. + The images are required to be pre-allocated.)", + py::arg("color_images") = std::vector{}, + py::arg("depth_images") = std::vector{}) .def( "set_sensor_transforms_from_keyframe", &AbstractReplayRenderer::setSensorTransformsFromKeyframe, diff --git a/src/esp/sim/AbstractReplayRenderer.cpp b/src/esp/sim/AbstractReplayRenderer.cpp index 8effc32d2f..73c88ff927 100644 --- a/src/esp/sim/AbstractReplayRenderer.cpp +++ b/src/esp/sim/AbstractReplayRenderer.cpp @@ -91,12 +91,21 @@ void AbstractReplayRenderer::setSensorTransformsFromKeyframe( } void AbstractReplayRenderer::render( - Cr::Containers::ArrayView imageViews) { - CORRADE_ASSERT(imageViews.size() == doEnvironmentCount(), - "ReplayRenderer::render(): expected" << doEnvironmentCount() - << "image views but got" - << imageViews.size(), ); - return doRender(imageViews); + Cr::Containers::ArrayView colorImageViews, + Cr::Containers::ArrayView depthImageViews) { + if (colorImageViews.size() > 0) { + ESP_CHECK(colorImageViews.size() == doEnvironmentCount(), + "ReplayRenderer::render(): expected" + << doEnvironmentCount() << "color image views but got" + << colorImageViews.size()); + } + if (depthImageViews.size() > 0) { + ESP_CHECK(depthImageViews.size() == doEnvironmentCount(), + "ReplayRenderer::render(): expected" + << doEnvironmentCount() << "depth image views but got" + << depthImageViews.size()); + } + return doRender(colorImageViews, depthImageViews); } void AbstractReplayRenderer::render( diff --git a/src/esp/sim/AbstractReplayRenderer.h b/src/esp/sim/AbstractReplayRenderer.h index 3f584738b9..d1308c816b 100644 --- a/src/esp/sim/AbstractReplayRenderer.h +++ b/src/esp/sim/AbstractReplayRenderer.h @@ -100,9 +100,12 @@ class AbstractReplayRenderer { void setSensorTransformsFromKeyframe(unsigned envIndex, const std::string& prefix); - // Renders and waits for the render to finish + // Renders into the specified CPU-resident image view arrays (one image per + // environment). Waits for the render to finish. void render(Corrade::Containers::ArrayView - imageViews); + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews); // Assumes the framebuffer color & depth is cleared void render(Magnum::GL::AbstractFramebuffer& framebuffer); @@ -164,7 +167,9 @@ class AbstractReplayRenderer { /* imageViews.size() is guaranteed to be same as doEnvironmentCount() */ virtual void doRender( Corrade::Containers::ArrayView - imageViews) = 0; + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) = 0; virtual void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) = 0; diff --git a/src/esp/sim/BatchReplayRenderer.cpp b/src/esp/sim/BatchReplayRenderer.cpp index a5346294fa..121a84b21b 100644 --- a/src/esp/sim/BatchReplayRenderer.cpp +++ b/src/esp/sim/BatchReplayRenderer.cpp @@ -252,7 +252,8 @@ void BatchReplayRenderer::doSetSensorTransformsFromKeyframe( } void BatchReplayRenderer::doRender( - Cr::Containers::ArrayView imageViews) { + Cr::Containers::ArrayView colorImageViews, + Cr::Containers::ArrayView depthImageViews) { CORRADE_ASSERT(standalone_, "BatchReplayRenderer::render(): can use this function only " "with a standalone renderer", ); @@ -267,8 +268,15 @@ void BatchReplayRenderer::doRender( Mn::Vector2i{envIndex % renderer_->tileCount().x(), envIndex / renderer_->tileCount().x()}, renderer_->tileSize()); - static_cast(*renderer_) - .colorImageInto(rectangle, imageViews[envIndex]); + + if (colorImageViews.size() > 0) { + static_cast(*renderer_) + .colorImageInto(rectangle, colorImageViews[envIndex]); + } + if (depthImageViews.size() > 0) { + static_cast(*renderer_) + .depthImageInto(rectangle, depthImageViews[envIndex]); + } } } diff --git a/src/esp/sim/BatchReplayRenderer.h b/src/esp/sim/BatchReplayRenderer.h index d7af77e9ea..fafa2fcb53 100644 --- a/src/esp/sim/BatchReplayRenderer.h +++ b/src/esp/sim/BatchReplayRenderer.h @@ -45,7 +45,9 @@ class BatchReplayRenderer : public AbstractReplayRenderer { const std::string& prefix) override; void doRender(Corrade::Containers::ArrayView - imageViews) override; + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) override; void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) override; diff --git a/src/esp/sim/ClassicReplayRenderer.cpp b/src/esp/sim/ClassicReplayRenderer.cpp index 45adffd070..a72e7b1c6e 100644 --- a/src/esp/sim/ClassicReplayRenderer.cpp +++ b/src/esp/sim/ClassicReplayRenderer.cpp @@ -8,6 +8,7 @@ #include "esp/gfx/RenderTarget.h" #include "esp/gfx/Renderer.h" #include "esp/metadata/MetadataMediator.h" +#include "esp/sensor/Sensor.h" #include "esp/sensor/SensorFactory.h" #include "esp/sim/SimulatorConfiguration.h" @@ -200,7 +201,8 @@ void ClassicReplayRenderer::doSetSensorTransformsFromKeyframe( } void ClassicReplayRenderer::doRender( - Cr::Containers::ArrayView imageViews) { + Cr::Containers::ArrayView colorImageViews, + Cr::Containers::ArrayView depthImageViews) { for (int envIndex = 0; envIndex < config_.numEnvironments; envIndex++) { auto& sensorMap = getEnvironmentSensors(envIndex); CORRADE_INTERNAL_ASSERT(sensorMap.size() == 1); @@ -211,11 +213,26 @@ void ClassicReplayRenderer::doRender( auto& sceneGraph = getSceneGraph(envIndex); + auto& sensorType = visualSensor.specification()->sensorType; + Cr::Containers::ArrayView imageViews; + switch (sensorType) { + case esp::sensor::SensorType::Color: + imageViews = colorImageViews; + break; + case esp::sensor::SensorType::Depth: + imageViews = depthImageViews; + break; + default: + break; + } + #ifdef ESP_BUILD_WITH_BACKGROUND_RENDERER - renderer_->enqueueAsyncDrawJob( - visualSensor, sceneGraph, imageViews[envIndex], - esp::gfx::RenderCamera::Flags{ - gfx::RenderCamera::Flag::FrustumCulling}); + if (imageViews != nullptr) { + renderer_->enqueueAsyncDrawJob( + visualSensor, sceneGraph, imageViews[envIndex], + esp::gfx::RenderCamera::Flags{ + gfx::RenderCamera::Flag::FrustumCulling}); + } #else // TODO what am I supposed to do here? CORRADE_ASSERT_UNREACHABLE("Not implemented yet, sorry.", ); diff --git a/src/esp/sim/ClassicReplayRenderer.h b/src/esp/sim/ClassicReplayRenderer.h index bb166f016a..d2e944b06e 100644 --- a/src/esp/sim/ClassicReplayRenderer.h +++ b/src/esp/sim/ClassicReplayRenderer.h @@ -74,7 +74,9 @@ class ClassicReplayRenderer : public AbstractReplayRenderer { const std::string& prefix) override; void doRender(Corrade::Containers::ArrayView - imageViews) override; + colorImageViews, + Corrade::Containers::ArrayView + depthImageViews) override; void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) override; diff --git a/src/tests/BatchReplayRendererTest.cpp b/src/tests/BatchReplayRendererTest.cpp index 9af01274e7..d9a6f01bbf 100644 --- a/src/tests/BatchReplayRendererTest.cpp +++ b/src/tests/BatchReplayRendererTest.cpp @@ -2,6 +2,7 @@ // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. +#include "Corrade/Containers/EnumSet.h" #include "Corrade/Utility/Assert.h" #include "Magnum/DebugTools/Screenshot.h" #include "Magnum/GL/Context.h" @@ -40,6 +41,9 @@ namespace { const std::string screenshotDir = Cr::Utility::Path::join(TEST_ASSETS, "screenshots/"); +enum class TestFlag : Magnum::UnsignedInt { Color = 1 << 0, Depth = 1 << 1 }; +typedef Corrade::Containers::EnumSet TestFlags; + struct BatchReplayRendererTest : Cr::TestSuite::Tester { explicit BatchReplayRendererTest(); @@ -66,16 +70,47 @@ Mn::MutableImageView2D getRGBView(int width, return view; } -std::vector getDefaultSensorSpecs( - const std::string& sensorName = "my_rgb") { +Mn::MutableImageView2D getDepthView(int width, + int height, + std::vector& buffer, + bool classic) { + Mn::Vector2i size(width, height); + constexpr int pixelSize = 4; + + buffer.resize(std::size_t(width * height * pixelSize)); + + // BEWARE: Classic renderer requires R32F because the depth is unprojected. + // Batch renderer directly returns the depth buffer at the moment. + auto pixelFormat = + classic ? Mn::PixelFormat::R32F : Mn::PixelFormat::Depth32F; + auto view = Mn::MutableImageView2D(pixelFormat, size, buffer); + + return view; +} + +esp::sensor::SensorSpec::ptr getDefaultSensorSpecs( + const std::string& sensorName, + const esp::sensor::SensorType sensorType) { auto pinholeCameraSpec = esp::sensor::CameraSensorSpec::create(); pinholeCameraSpec->sensorSubType = esp::sensor::SensorSubType::Pinhole; - pinholeCameraSpec->sensorType = esp::sensor::SensorType::Color; + pinholeCameraSpec->sensorType = sensorType; pinholeCameraSpec->position = {0.0f, 0.f, 0.0f}; pinholeCameraSpec->resolution = {512, 384}; pinholeCameraSpec->uuid = sensorName; - std::vector sensorSpecifications = { - pinholeCameraSpec}; + return pinholeCameraSpec; +} + +std::vector getDefaultSensorSpecs( + TestFlags flags) { + std::vector sensorSpecifications{}; + if (flags & TestFlag::Color) { + sensorSpecifications.push_back( + getDefaultSensorSpecs("rgb", esp::sensor::SensorType::Color)); + } + if (flags & TestFlag::Depth) { + sensorSpecifications.push_back( + getDefaultSensorSpecs("depth", esp::sensor::SensorType::Depth)); + } return sensorSpecifications; } @@ -83,25 +118,54 @@ const struct { const char* name; Cr::Containers::Pointer (*create)( const ReplayRendererConfiguration& configuration); +} TestUnprojectData[]{ + {"classic", + [](const ReplayRendererConfiguration& configuration) { + return Cr::Containers::Pointer{ + new esp::sim::ClassicReplayRenderer{configuration}}; + }}, + // temp only enable testUnproject for classic + //{"batch", [](const ReplayRendererConfiguration& configuration) { + // return Cr::Containers::Pointer{ + // new esp::sim::BatchReplayRenderer{configuration}}; + // } + //}, +}; + +const struct { + const char* name; + TestFlags testFlags; + Cr::Containers::Pointer (*create)( + const ReplayRendererConfiguration& configuration); } TestIntegrationData[]{ - {"classic renderer", + {"rgb - classic", TestFlag::Color, + [](const ReplayRendererConfiguration& configuration) { + return Cr::Containers::Pointer{ + new esp::sim::ClassicReplayRenderer{configuration}}; + }}, + {"rgb - batch", TestFlag::Color, + [](const ReplayRendererConfiguration& configuration) { + return Cr::Containers::Pointer{ + new esp::sim::BatchReplayRenderer{configuration}}; + }}, + {"depth - classic", TestFlag::Depth, [](const ReplayRendererConfiguration& configuration) { return Cr::Containers::Pointer{ new esp::sim::ClassicReplayRenderer{configuration}}; }}, - {"batch renderer", [](const ReplayRendererConfiguration& configuration) { + {"depth - batch", TestFlag::Depth, + [](const ReplayRendererConfiguration& configuration) { return Cr::Containers::Pointer{ new esp::sim::BatchReplayRenderer{configuration}}; - }}}; + }}, +}; BatchReplayRendererTest::BatchReplayRendererTest() { + addInstancedTests({&BatchReplayRendererTest::testUnproject}, + Cr::Containers::arraySize(TestUnprojectData)); + addInstancedTests({&BatchReplayRendererTest::testIntegration}, Cr::Containers::arraySize(TestIntegrationData)); - - // temp only enable testUnproject for classic - addInstancedTests({&BatchReplayRendererTest::testUnproject}, 1); - // addInstancedTests({&BatchReplayRendererTest::testUnproject}, - // Cr::Containers::arraySize(TestIntegrationData)); } // ctor // test recording and playback through the simulator interface @@ -159,10 +223,11 @@ void BatchReplayRendererTest::testIntegration() { auto&& data = TestIntegrationData[testCaseInstanceId()]; setTestCaseDescription(data.name); + const auto sensorSpecs = getDefaultSensorSpecs(data.testFlags); + const std::string vangogh = Cr::Utility::Path::join( SCENE_DATASETS, "habitat-test-scenes/van-gogh-room.glb"); constexpr int numEnvs = 4; - const std::string sensorName = "my_rgb"; const std::string userPrefix = "sensor_"; const std::string screenshotPrefix = "ReplayBatchRendererTest_env"; const std::string screenshotExtension = ".png"; @@ -203,10 +268,13 @@ void BatchReplayRendererTest::testIntegration() { } auto& recorder = *sim->getGfxReplayManager()->getRecorder(); - recorder.addUserTransformToKeyframe( - userPrefix + sensorName, Mn::Vector3(3.3f, 1.3f + envIndex * 0.1f, 0.f), - Mn::Quaternion::rotation(Mn::Deg(80.f + envIndex * 5.f), - Mn::Vector3(0.f, 1.f, 0.f))); + for (const auto& sensor : sensorSpecs) { + recorder.addUserTransformToKeyframe( + userPrefix + sensor->uuid, + Mn::Vector3(3.3f, 1.3f + envIndex * 0.1f, 0.f), + Mn::Quaternion::rotation(Mn::Deg(80.f + envIndex * 5.f), + Mn::Vector3(0.f, 1.f, 0.f))); + } std::string serKeyframe = esp::gfx::replay::Recorder::keyframeToString( recorder.extractKeyframe()); @@ -214,25 +282,34 @@ void BatchReplayRendererTest::testIntegration() { } ReplayRendererConfiguration batchRendererConfig; - batchRendererConfig.sensorSpecifications = getDefaultSensorSpecs(sensorName); + batchRendererConfig.sensorSpecifications = sensorSpecs; batchRendererConfig.numEnvironments = numEnvs; { Cr::Containers::Pointer renderer = data.create(batchRendererConfig); + bool isClassicRenderer = dynamic_cast( + renderer.get()) != nullptr; // Check that the context is properly created CORRADE_VERIFY(Mn::GL::Context::hasCurrent()); - std::vector> buffers(numEnvs); - std::vector imageViews; + std::vector> colorBuffers(numEnvs); + std::vector> depthBuffers(numEnvs); + std::vector colorImageViews; + std::vector depthImageViews; for (int envIndex = 0; envIndex < numEnvs; envIndex++) { - // TODO pass size as a Vector2i; use an Image instead of a std::vector - // once there's Iterable that can be implicitly - // converted from a list of Image2D. - imageViews.emplace_back(getRGBView(renderer->sensorSize(envIndex).x(), - renderer->sensorSize(envIndex).y(), - buffers[envIndex])); + if (data.testFlags & TestFlag::Color) { + colorImageViews.emplace_back(getRGBView( + renderer->sensorSize(envIndex).x(), + renderer->sensorSize(envIndex).y(), colorBuffers[envIndex])); + } + if (data.testFlags & TestFlag::Depth) { + depthImageViews.emplace_back( + getDepthView(renderer->sensorSize(envIndex).x(), + renderer->sensorSize(envIndex).y(), + depthBuffers[envIndex], isClassicRenderer)); + } } for (int envIndex = 0; envIndex < numEnvs; envIndex++) { @@ -240,16 +317,27 @@ void BatchReplayRendererTest::testIntegration() { renderer->setSensorTransformsFromKeyframe(envIndex, userPrefix); } - renderer->render(imageViews); + renderer->render(colorImageViews, depthImageViews); for (int envIndex = 0; envIndex < numEnvs; envIndex++) { CORRADE_ITERATION(envIndex); - std::string groundTruthImageFile = - screenshotPrefix + std::to_string(envIndex) + screenshotExtension; - CORRADE_COMPARE_WITH( - Mn::ImageView2D{imageViews[envIndex]}, - Cr::Utility::Path::join(screenshotDir, groundTruthImageFile), - (Mn::DebugTools::CompareImageToFile{maxThreshold, meanThreshold})); + // Test color output + if (data.testFlags & TestFlag::Color) { + std::string groundTruthImageFile = + screenshotPrefix + std::to_string(envIndex) + screenshotExtension; + CORRADE_COMPARE_WITH( + Mn::ImageView2D{colorImageViews[envIndex]}, + Cr::Utility::Path::join(screenshotDir, groundTruthImageFile), + (Mn::DebugTools::CompareImageToFile{maxThreshold, meanThreshold})); + } + // Test depth output + if (data.testFlags & TestFlag::Depth) { + const auto depth = depthImageViews[envIndex]; + float pixelA = depth.pixels()[32][32]; + float pixelB = depth.pixels()[64][64]; + CORRADE_VERIFY(pixelA > 0.0f); + CORRADE_VERIFY(pixelA != pixelB); + } } const auto colorPtr = renderer->getCudaColorBufferDevicePointer();