GPU-to-CPU batch renderer API to render color and depth sensors into images.

src/esp/bindings/SimBindings.cpp

@@ -435,12 +435,14 @@ void initSimBindings(py::module& m) {
       .def(
           "render",
           [](AbstractReplayRenderer& self,
-             std::vector<Mn::MutableImageView2D> images) {
-            self.render(images);
+             std::vector<Mn::MutableImageView2D> colorImageViews,
+             std::vector<Mn::MutableImageView2D> depthImageViews) {
+            self.render(colorImageViews, depthImageViews);
           },
-          R"(Render color sensors into the specified image vector (one per environment).
-          The images are required to be pre-allocated.
-          Blocks the thread during the GPU-to-CPU memory transfer operation.)")
+          R"(Render sensors into the specified image vectors (one per environment).
+          Blocks the thread during the GPU-to-CPU memory transfer operation.
+          Empty lists can be supplied to skip the copying render targets.
+          The images are required to be pre-allocated.)")
       .def(
           "set_sensor_transforms_from_keyframe",
           &AbstractReplayRenderer::setSensorTransformsFromKeyframe,

src/esp/sim/AbstractReplayRenderer.cpp

@@ -91,12 +91,21 @@ void AbstractReplayRenderer::setSensorTransformsFromKeyframe(
 }
 
 void AbstractReplayRenderer::render(
-    Cr::Containers::ArrayView<const Mn::MutableImageView2D> imageViews) {
-  CORRADE_ASSERT(imageViews.size() == doEnvironmentCount(),
-                 "ReplayRenderer::render(): expected" << doEnvironmentCount()
-                                                      << "image views but got"
-                                                      << imageViews.size(), );
-  return doRender(imageViews);
+    Cr::Containers::ArrayView<const Mn::MutableImageView2D> colorImageViews,
+    Cr::Containers::ArrayView<const Mn::MutableImageView2D> depthImageViews) {
+  if (colorImageViews != nullptr) {
+    ESP_CHECK(colorImageViews.size() == doEnvironmentCount(),
+              "ReplayRenderer::render(): expected"
+                  << doEnvironmentCount() << "color image views but got"
+                  << colorImageViews.size());
+  }
+  if (depthImageViews != nullptr) {
+    ESP_CHECK(depthImageViews.size() == doEnvironmentCount(),
+              "ReplayRenderer::render(): expected"
+                  << doEnvironmentCount() << "depth image views but got"
+                  << depthImageViews.size());
+  }
+  return doRender(colorImageViews, depthImageViews);
 }
 
 void AbstractReplayRenderer::render(

src/esp/sim/AbstractReplayRenderer.h

@@ -100,9 +100,12 @@ class AbstractReplayRenderer {
   void setSensorTransformsFromKeyframe(unsigned envIndex,
                                        const std::string& prefix);
 
-  // Renders and waits for the render to finish
+  // Renders into the specified CPU-resident image view arrays (one image per
+  // environment). Waits for the render to finish.
   void render(Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
-                  imageViews);
+                  colorImageViews,
+              Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
+                  depthImageViews);
 
   // Assumes the framebuffer color & depth is cleared
   void render(Magnum::GL::AbstractFramebuffer& framebuffer);
@@ -164,7 +167,9 @@ class AbstractReplayRenderer {
   /* imageViews.size() is guaranteed to be same as doEnvironmentCount() */
   virtual void doRender(
       Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
-          imageViews) = 0;
+          colorImageViews,
+      Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
+          depthImageViews) = 0;
 
   virtual void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) = 0;
 

src/esp/sim/BatchReplayRenderer.cpp

@@ -252,7 +252,8 @@ void BatchReplayRenderer::doSetSensorTransformsFromKeyframe(
 }
 
 void BatchReplayRenderer::doRender(
-    Cr::Containers::ArrayView<const Mn::MutableImageView2D> imageViews) {
+    Cr::Containers::ArrayView<const Mn::MutableImageView2D> colorImageViews,
+    Cr::Containers::ArrayView<const Mn::MutableImageView2D> depthImageViews) {
   CORRADE_ASSERT(standalone_,
                  "BatchReplayRenderer::render(): can use this function only "
                  "with a standalone renderer", );
@@ -267,8 +268,15 @@ void BatchReplayRenderer::doRender(
             Mn::Vector2i{envIndex % renderer_->tileCount().x(),
                          envIndex / renderer_->tileCount().x()},
         renderer_->tileSize());
-    static_cast<gfx_batch::RendererStandalone&>(*renderer_)
-        .colorImageInto(rectangle, imageViews[envIndex]);
+
+    if (colorImageViews) {
+      static_cast<gfx_batch::RendererStandalone&>(*renderer_)
+          .colorImageInto(rectangle, colorImageViews[envIndex]);
+    }
+    if (depthImageViews) {
+      static_cast<gfx_batch::RendererStandalone&>(*renderer_)
+          .depthImageInto(rectangle, depthImageViews[envIndex]);
+    }
   }
 }
 

src/esp/sim/BatchReplayRenderer.h

@@ -45,7 +45,9 @@ class BatchReplayRenderer : public AbstractReplayRenderer {
                                          const std::string& prefix) override;
 
   void doRender(Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
-                    imageViews) override;
+                    colorImageViews,
+                Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
+                    depthImageViews) override;
 
   void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) override;
 

src/esp/sim/ClassicReplayRenderer.cpp

@@ -8,6 +8,7 @@
 #include "esp/gfx/RenderTarget.h"
 #include "esp/gfx/Renderer.h"
 #include "esp/metadata/MetadataMediator.h"
+#include "esp/sensor/Sensor.h"
 #include "esp/sensor/SensorFactory.h"
 #include "esp/sim/SimulatorConfiguration.h"
 
@@ -201,7 +202,8 @@ void ClassicReplayRenderer::doSetSensorTransformsFromKeyframe(
 }
 
 void ClassicReplayRenderer::doRender(
-    Cr::Containers::ArrayView<const Mn::MutableImageView2D> imageViews) {
+    Cr::Containers::ArrayView<const Mn::MutableImageView2D> colorImageViews,
+    Cr::Containers::ArrayView<const Mn::MutableImageView2D> depthImageViews) {
   for (int envIndex = 0; envIndex < config_.numEnvironments; envIndex++) {
     auto& sensorMap = getEnvironmentSensors(envIndex);
     CORRADE_INTERNAL_ASSERT(sensorMap.size() == 1);
@@ -212,11 +214,28 @@ void ClassicReplayRenderer::doRender(
 
       auto& sceneGraph = getSceneGraph(envIndex);
 
+      auto& sensorType = visualSensor.specification()->sensorType;
+      Cr::Containers::ArrayView<const Mn::MutableImageView2D> imageViews;
+      switch (sensorType) {
+        case esp::sensor::SensorType::Color:
+          imageViews = colorImageViews;
+          break;
+        case esp::sensor::SensorType::Depth:
+          imageViews = depthImageViews;
+          break;
+        default:
+          imageViews = nullptr;
+          break;
+      }
+
 #ifdef ESP_BUILD_WITH_BACKGROUND_RENDERER
-      renderer_->enqueueAsyncDrawJob(
-          visualSensor, sceneGraph, imageViews[envIndex],
-          esp::gfx::RenderCamera::Flags{
-              gfx::RenderCamera::Flag::FrustumCulling});
+      // todo: investigate flags (frustum culling?)
+      if (imageViews != nullptr) {
+        renderer_->enqueueAsyncDrawJob(
+            visualSensor, sceneGraph, imageViews[envIndex],
+            esp::gfx::RenderCamera::Flags{
+                gfx::RenderCamera::Flag::FrustumCulling});
+      }
 #else
       // TODO what am I supposed to do here?
       CORRADE_ASSERT_UNREACHABLE("Not implemented yet, sorry.", );

src/esp/sim/ClassicReplayRenderer.h

@@ -74,7 +74,9 @@ class ClassicReplayRenderer : public AbstractReplayRenderer {
                                          const std::string& prefix) override;
 
   void doRender(Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
-                    imageViews) override;
+                    colorImageViews,
+                Corrade::Containers::ArrayView<const Magnum::MutableImageView2D>
+                    depthImageViews) override;
 
   void doRender(Magnum::GL::AbstractFramebuffer& framebuffer) override;
 

src/tests/BatchReplayRendererTest.cpp

@@ -240,7 +240,7 @@ void BatchReplayRendererTest::testIntegration() {
       renderer->setSensorTransformsFromKeyframe(envIndex, userPrefix);
     }
 
-    renderer->render(imageViews);
+    renderer->render(imageViews, nullptr);
 
     for (int envIndex = 0; envIndex < numEnvs; envIndex++) {
       CORRADE_ITERATION(envIndex);