Fix 0 samples in deep data (#1652)

Add some additional safety checks, but properly handle when there are 0
samples in a deep scanline / tile (chunk)

Should fix #1639 and OSS-Fuzz 66612

Signed-off-by: Kimball Thurston <kdt3rd@gmail.com>

src/lib/OpenEXRCore/decoding.c

@@ -358,8 +358,8 @@ unpack_sample_table (
             {
                 int32_t nsamps =
                     (int32_t) one_to_native32 ((uint32_t) cursampline[x]);
+                // not monotonic, violation
                 if (nsamps < prevsamp) return EXR_ERR_INVALID_SAMPLE_DATA;
-
                 cursampline[x] = nsamps - prevsamp;
                 prevsamp       = nsamps;
             }
@@ -379,6 +379,7 @@ unpack_sample_table (
             {
                 int32_t nsamps =
                     (int32_t) one_to_native32 ((uint32_t) cursampline[x]);
+                // not monotonic, violation
                 if (nsamps < prevsamp) return EXR_ERR_INVALID_SAMPLE_DATA;
 
                 cursampline[x] = nsamps;
@@ -387,13 +388,16 @@ unpack_sample_table (
 
             totsamp += (uint64_t)prevsamp;
         }
+        if (totsamp >= (uint64_t)INT32_MAX)
+            return EXR_ERR_INVALID_SAMPLE_DATA;
     }
 
     if ((totsamp * combSampSize) > decode->chunk.unpacked_size)
     {
         rv = pctxt->report_error (
             pctxt, EXR_ERR_INVALID_SAMPLE_DATA, "Corrupt sample count table");
     }
+
     return rv;
 }
 
@@ -664,14 +668,34 @@ exr_decoding_run (
         (part->storage_mode == EXR_STORAGE_DEEP_SCANLINE ||
          part->storage_mode == EXR_STORAGE_DEEP_TILED))
     {
+        if (part->comp_type == EXR_COMPRESSION_NONE &&
+            decode->sample_count_table != decode->packed_sample_count_table)
+        {
+            /* happens when we're requested to pack to 'individual' mode */
+            if (decode->sample_count_alloc_size < decode->chunk.sample_count_table_size)
+                return EXR_ERR_OUT_OF_MEMORY;
+            if (decode->chunk.sample_count_table_size > 0)
+            {
+                memcpy (decode->sample_count_table,
+                        decode->packed_sample_count_table,
+                        decode->chunk.sample_count_table_size);
+            }
+            else
+            {
+                memset (decode->sample_count_table, 0, decode->sample_count_alloc_size);
+            }
+        }
+
         rv = unpack_sample_table (pctxt, decode);
 
         if ((decode->decode_flags & EXR_DECODE_SAMPLE_DATA_ONLY)) return rv;
-    }
 
-    if (rv != EXR_ERR_SUCCESS)
-        return pctxt->report_error (
-            pctxt, rv, "Decode pipeline unable to unpack deep sample table");
+        if (rv != EXR_ERR_SUCCESS)
+            return pctxt->report_error (
+                pctxt,
+                rv,
+                "Decode pipeline unable to unpack deep sample table");
+    }
 
     if (rv == EXR_ERR_SUCCESS && decode->realloc_nonimage_data_fn)
         rv = decode->realloc_nonimage_data_fn (decode);
@@ -681,11 +705,14 @@ exr_decoding_run (
             rv,
             "Decode pipeline unable to realloc deep sample table info");
 
-    if (rv == EXR_ERR_SUCCESS && decode->unpack_and_convert_fn)
-        rv = decode->unpack_and_convert_fn (decode);
-    if (rv != EXR_ERR_SUCCESS)
-        return pctxt->report_error (
-            pctxt, rv, "Decode pipeline unable to unpack and convert data");
+    if (decode->chunk.unpacked_size > 0)
+    {
+        if (rv == EXR_ERR_SUCCESS && decode->unpack_and_convert_fn)
+            rv = decode->unpack_and_convert_fn (decode);
+        if (rv != EXR_ERR_SUCCESS)
+            return pctxt->report_error (
+                pctxt, rv, "Decode pipeline unable to unpack and convert data");
+    }
 
     return rv;
 }
@@ -706,6 +733,10 @@ exr_decoding_destroy (exr_const_context_t ctxt, exr_decode_pipeline_t* decode)
             decode->unpacked_alloc_size == 0)
             decode->unpacked_buffer = NULL;
 
+        if (decode->sample_count_table == decode->packed_sample_count_table &&
+            decode->sample_count_alloc_size == 0)
+            decode->sample_count_table = NULL;
+
         internal_decode_free_buffer (
             decode,
             EXR_TRANSCODE_BUFFER_PACKED,
@@ -726,16 +757,17 @@ exr_decoding_destroy (exr_const_context_t ctxt, exr_decode_pipeline_t* decode)
             EXR_TRANSCODE_BUFFER_SCRATCH2,
             &(decode->scratch_buffer_2),
             &(decode->scratch_alloc_size_2));
-        internal_decode_free_buffer (
-            decode,
-            EXR_TRANSCODE_BUFFER_PACKED_SAMPLES,
-            &(decode->packed_sample_count_table),
-            &(decode->packed_sample_count_alloc_size));
+
         internal_decode_free_buffer (
             decode,
             EXR_TRANSCODE_BUFFER_SAMPLES,
             (void**) &(decode->sample_count_table),
             &(decode->sample_count_alloc_size));
+        internal_decode_free_buffer (
+            decode,
+            EXR_TRANSCODE_BUFFER_PACKED_SAMPLES,
+            &(decode->packed_sample_count_table),
+            &(decode->packed_sample_count_alloc_size));
         *decode = nil;
     }
     return EXR_ERR_SUCCESS;

src/lib/OpenEXRUtil/ImfCheckFile.cpp

@@ -1247,7 +1247,7 @@ realloc_deepdata(exr_decode_pipeline_t* decode)
         bytes += totsamps * outc.user_bytes_per_element;
     }
 
-    if (bytes >= gMaxBytesPerDeepScanline)
+    if (bytes == 0 || bytes >= gMaxBytesPerDeepScanline)
     {
         for (int c = 0; c < decode->channel_count; c++)
         {