adjust checks for core to better match c++ checks (#1632)

The core checks were not setting the same image / tile size limits and
not disabling reads at quite the same level.

Note: the core check does not read the entire image into a contiguous
slice, so does not replicate the maximum deep sample checks in the same
way, this is a source of potential false-negative failures

This should address OSS-Fuzz 66491 and 66489 (different forms of the
same failure where a large sample size allocation was happening), and
are only constrained memory (2.5Gb) issues.

Signed-off-by: Kimball Thurston <kdt3rd@gmail.com>

src/lib/OpenEXRUtil/ImfCheckFile.cpp

@@ -1219,7 +1219,7 @@ realloc_deepdata(exr_decode_pipeline_t* decode)
         bytes += totsamps * outc.user_bytes_per_element;
     }
 
-    if (bytes >= gMaxBytesPerDeepScanline * h)
+    if (bytes >= gMaxBytesPerDeepScanline)
     {
         for (int c = 0; c < decode->channel_count; c++)
         {
@@ -1302,15 +1302,16 @@ bool readCoreScanlinePart(exr_context_t f, int part, bool reduceMemory, bool red
             }
 
             doread = true;
+            if (reduceMemory && bytes >= gMaxBytesPerScanline)
+                doread = false;
+
             if (cinfo.type == EXR_STORAGE_DEEP_SCANLINE)
             {
                 decoder.decoding_user_data       = &imgdata;
                 decoder.realloc_nonimage_data_fn = &realloc_deepdata;
             }
             else
             {
-                if (reduceMemory && bytes >= gMaxBytesPerScanline) doread = false;
-
                 if (doread) imgdata.resize (bytes);
             }
             rv = exr_decoding_choose_default_routines (f, part, &decoder);
@@ -1467,16 +1468,16 @@ bool readCoreTiledPart(exr_context_t f, int part, bool reduceMemory, bool reduce
                         }
 
                         doread = true;
+                        if (reduceMemory && bytes >= gMaxTileBytes)
+                            doread = false;
+
                         if (cinfo.type == EXR_STORAGE_DEEP_TILED)
                         {
                             decoder.decoding_user_data       = &tiledata;
                             decoder.realloc_nonimage_data_fn = &realloc_deepdata;
                         }
                         else
                         {
-                            if (reduceMemory && bytes >= gMaxTileBytes)
-                                doread = false;
-
                             if (doread) tiledata.resize (bytes);
                         }
                         rv = exr_decoding_choose_default_routines (
@@ -1609,6 +1610,20 @@ runCoreChecks (const char *filename, bool reduceMemory, bool reduceTime)
 
     cinit.error_handler_fn = &core_error_handler_cb;
 
+    if (reduceMemory || reduceTime)
+    {
+        /* could use set_default functions for this, but those just
+         * initialize the context, doing it in the initializer is mt
+         * safe...
+         * exr_set_default_maximum_image_size (2048, 2048);
+         * exr_set_default_maximum_tile_size (512, 512);
+         */
+        cinit.max_image_width = 2048;
+        cinit.max_image_height = 2048;
+        cinit.max_tile_width = 512;
+        cinit.max_tile_height = 512;
+    }
+
     rv = exr_start_read (&f, filename, &cinit);
     if (rv != EXR_ERR_SUCCESS)
         return true;