Add maximumSampleCount limit to CompositeDeepScanLine

src/bin/exrcheck/main.cpp

@@ -113,7 +113,14 @@ main (int argc, char** argv)
         }
         else if (!strcmp (argv[i], "-m"))
         {
+            //
+            // note for further memory reduction, calls to the folowing could be added here
+            // CompositeDeepScanLine::setMaximumSampleCount();
+            // Header::setMaxImageSize();
+            // Header::setMaxTileSize();
+
             reduceMemory = true;
+
         }
         else if (!strcmp (argv[i], "-t"))
         {

src/lib/OpenEXR/ImfCompositeDeepScanLine.cpp

@@ -427,6 +427,23 @@ LineCompositeTask::execute ()
 
 } // namespace
 
+
+namespace {
+int64_t maximumSampleCount = 0;
+}
+
+void CompositeDeepScanLine::setMaximumSampleCount(int64_t c)
+{
+    maximumSampleCount = c;
+}
+
+int64_t CompositeDeepScanLine::getMaximumSampleCount()
+{
+    return maximumSampleCount;
+}
+
+
+
 void
 CompositeDeepScanLine::readPixels (int start, int end)
 {
@@ -508,6 +525,15 @@ CompositeDeepScanLine::readPixels (int start, int end)
         overall_sample_count += total_sizes[ptr];
     }
 
+    if (maximumSampleCount > 0 &&  overall_sample_count > maximumSampleCount)
+    {
+        throw IEX_NAMESPACE::ArgExc (
+            "Cannot composite scanline: total sample count on scanline exceeds "
+            "limit set by CompositeDeepScanLine::setMaximumSampleCount()"
+        );
+    }
+
+
     //
     // allocate arrays for pixel data
     // samples array accessed as in pixels[channel][sample]
@@ -599,4 +625,5 @@ CompositeDeepScanLine::frameBuffer () const
     return _Data->_outputFrameBuffer;
 }
 
+
 OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT

src/lib/OpenEXR/ImfCompositeDeepScanLine.h

@@ -106,6 +106,23 @@ class IMF_EXPORT_TYPE CompositeDeepScanLine
 
     struct IMF_HIDDEN Data;
 
+
+    //
+    // set the maximum number of samples that will be composited.
+    // If a single scanline has more samples, readPixels will throw
+    // an exception. This mechanism prevents the library allocating
+    // excessive memory to composite deep scanline images.
+    // A value of 0 will cause deep compositing to be disabled entirely
+    // A negative value disables the limit, allowing images with
+    // arbitrarily large sample counts to be composited
+    //
+    IMF_EXPORT
+    static void setMaximumSampleCount(int64_t sampleCount);
+
+    IMF_EXPORT
+    static int64_t getMaximumSampleCount();
+
+
 private:
     struct Data* _Data;
 

src/lib/OpenEXRUtil/ImfCheckFile.cpp

@@ -5,6 +5,7 @@
 #include "Iex.h"
 #include "ImfArray.h"
 #include "ImfChannelList.h"
+#include "ImfCompositeDeepScanLine.h"
 #include "ImfCompressor.h"
 #include "ImfDeepFrameBuffer.h"
 #include "ImfDeepScanLineInputFile.h"
@@ -1612,9 +1613,22 @@ checkOpenEXRFile (
     bool        enableCoreCheck)
 {
     bool threw = false;
+
+    uint64_t oldMaxSampleCount = CompositeDeepScanLine::getMaximumSampleCount();
+
+    if( reduceMemory || reduceTime)
+    {
+        CompositeDeepScanLine::setMaximumSampleCount(1<<20);
+    }
+
     if (enableCoreCheck)
+    {
         threw = runCoreChecks (fileName, reduceMemory, reduceTime);
+    }
     if (!threw) threw = runChecks (fileName, reduceMemory, reduceTime);
+
+    CompositeDeepScanLine::setMaximumSampleCount(oldMaxSampleCount);
+
     return threw;
 }
 
@@ -1627,13 +1641,23 @@ checkOpenEXRFile (
     bool        enableCoreCheck)
 {
     bool threw = false;
+    uint64_t oldMaxSampleCount = CompositeDeepScanLine::getMaximumSampleCount();
+
+    if( reduceMemory || reduceTime)
+    {
+        CompositeDeepScanLine::setMaximumSampleCount(1<<20);
+    }
+
     if (enableCoreCheck)
         threw = runCoreChecks (data, numBytes, reduceMemory, reduceTime);
     if (!threw)
     {
         PtrIStream stream (data, numBytes);
         threw = runChecks (stream, reduceMemory, reduceTime);
     }
+
+    CompositeDeepScanLine::setMaximumSampleCount(oldMaxSampleCount);
+
     return threw;
 }