Propagate dwa core 3 1 (#1423)

* Add dwa support to core (#1402) * Fix nodiscard warning on gcc 13 * Add buffer range checks to huf code * Refactor cpuid code for use in dwa * Fix issue with big endian deep encoding of chunk table * refactor to expose byte reordering zip compression uses for dwa * expose option flag to enable headers to be more easily like old style headers * fix potential issue with piz and not able to compress data small enough * Restore piz tests * Initial implementation of DWAA/B compression for Core * write with legacy header during tests to make comparing bytes in files easier * Initial support for validating exact match in file output between C / C++ * Enable DWAA/DWAB tests * Older versions of MSVC have issues with specifying C11 for use of stdalign.h and alignas macro. Work around this by targeting the lower level macros / declspec instead of using C11 stdalign.h * Add unit test for x86 cpu id checks * Add alignment helper utility * Add missing include guards * Fix pedantic warnings * Fix compile warnings. Core should now compile with -Weverything -Wno-reserved-identifier -Wno-covered-switch-default -Wno-cast-align -Wno-overlength-strings Signed-off-by: Kimball Thurston <kdt3rd@gmail.com> * Add #include <cmath> for isnan (#1414) Signed-off-by: Cary Phillips <cary@ilm.com> * Fix initialization after merge Signed-off-by: Kimball Thurston <kdt3rd@gmail.com> --------- Signed-off-by: Kimball Thurston <kdt3rd@gmail.com> Signed-off-by: Cary Phillips <cary@ilm.com> Co-authored-by: Cary Phillips <cary@ilm.com>
AcademySoftwareFoundation · May 20, 2023 · e7233d0 · e7233d0
1 parent c3863da
commit e7233d0
Show file tree

Hide file tree

Showing 46 changed files with 6,952 additions and 432 deletions.
diff --git a/src/bin/exrinfo/main.c b/src/bin/exrinfo/main.c
@@ -104,7 +104,7 @@ process_stdin (int verbose, int allmeta, int strict)
 static int
 process_file (const char* filename, int verbose, int allmeta, int strict)
 {
-    int                       failcount;
+    int                       failcount = 0;
     exr_result_t              rv;
     exr_context_t             e     = NULL;
     exr_context_initializer_t cinit = EXR_DEFAULT_CONTEXT_INITIALIZER;
@@ -129,7 +129,7 @@ process_file (const char* filename, int verbose, int allmeta, int strict)
 int
 main (int argc, const char* argv[])
 {
-    int rv = 0, nfiles = 0, verbose = 0, allmeta = 0, strict = 0;
+    int rv = 0, verbose = 0, allmeta = 0, strict = 0;
 
     for (int a = 1; a < argc; ++a)
     {
@@ -153,7 +153,6 @@ main (int argc, const char* argv[])
         }
         else if (!strcmp (argv[a], "-"))
         {
-            ++nfiles;
             rv += process_stdin (verbose, allmeta, strict);
         }
         else if (argv[a][0] == '-')
@@ -163,7 +162,6 @@ main (int argc, const char* argv[])
         }
         else
         {
-            ++nfiles;
             rv += process_file (argv[a], verbose, allmeta, strict);
         }
     }

diff --git a/src/bin/exrmultipart/exrmultipart.cpp b/src/bin/exrmultipart/exrmultipart.cpp
@@ -573,7 +573,6 @@ combine (vector <const char*> in,
     }
 
     inputs.clear();
-    fordelete.size();
 
     cout << "\n" << "Combine Success" << endl;
 }

diff --git a/src/lib/IlmThread/IlmThreadPool.cpp b/src/lib/IlmThread/IlmThreadPool.cpp
@@ -538,7 +538,6 @@ ThreadPool::setNumThreads (int count)
         Data::ProviderPtr sp = _data->getProvider ();
         if (sp)
         {
-            bool doReset = false;
             int  curT    = sp->numThreads ();
             if (curT == count) return;
 

diff --git a/src/lib/OpenEXR/dwaLookups.h b/src/lib/OpenEXR/dwaLookups.h
@@ -3,7 +3,7 @@
 // Copyright (c) DreamWorks Animation LLC and Contributors of the OpenEXR Project
 //
 
-const unsigned short dwaCompressorNoOp[] = 
+static const unsigned short dwaCompressorNoOp[] =
 {
     0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 
     0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 
@@ -8201,7 +8201,7 @@ const unsigned short dwaCompressorNoOp[] =
 
 
 
-const unsigned short dwaCompressorToLinear[] = 
+static const unsigned short dwaCompressorToLinear[] =
 {
     0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 
     0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 
@@ -16399,7 +16399,7 @@ const unsigned short dwaCompressorToLinear[] =
 
 
 
-const unsigned short dwaCompressorToNonlinear[] = 
+static const unsigned short dwaCompressorToNonlinear[] =
 {
     0x0000, 0x1043, 0x11d7, 0x1305, 0x1400, 0x146d, 0x14cf, 0x1529, 
     0x157b, 0x15c8, 0x1611, 0x1656, 0x1697, 0x16d6, 0x1712, 0x174b, 
@@ -24597,7 +24597,7 @@ const unsigned short dwaCompressorToNonlinear[] =
 
 
 
-const unsigned int closestDataOffset[] = {
+static const unsigned int closestDataOffset[] = {
          0,      0,      1,      2,      4,      5,      7,      9, 
         12,     13,     15,     17,     20,     22,     25,     28, 
         32,     33,     35,     37,     40,     42,     45,     48, 
@@ -32793,7 +32793,7 @@ const unsigned int closestDataOffset[] = {
 };
 
 
-const unsigned short closestData[] = {
+static const unsigned short closestData[] = {
         0,     0,     0,     2,     0,     0,     4,     0, 
         4,     0,     8,     6,     0,     0,     8,     0, 
         8,     0,     8,    10,     0,     8,     0,    16, 

diff --git a/src/lib/OpenEXRCore/CMakeLists.txt b/src/lib/OpenEXRCore/CMakeLists.txt
@@ -14,7 +14,15 @@ openexr_define_library(OpenEXRCore
     internal_coding.h
     internal_constants.h
     internal_compress.h
+    internal_cpuid.h
     internal_decompress.h
+    internal_dwa_channeldata.h
+    internal_dwa_classifier.h
+    internal_dwa_compressor.h
+    internal_dwa_decoder.h
+    internal_dwa_encoder.h
+    internal_dwa_helpers.h
+    internal_dwa_simd.h
     internal_file.h
     internal_float_vector.h
     internal_huf.h

diff --git a/src/lib/OpenEXRCore/chunk.c b/src/lib/OpenEXRCore/chunk.c
@@ -240,6 +240,7 @@ struct priv_chunk_leader
             int32_t level_y;
         };
     };
+    uint8_t _pad[4];
     union
     {
         int64_t deep_data[3];
@@ -291,7 +292,7 @@ extract_chunk_leader (
     rv = ctxt->do_read (
         ctxt,
         data,
-        ntoread * sizeof (int32_t),
+        (size_t)ntoread * sizeof (int32_t),
         &nextoffset,
         NULL,
         EXR_MUST_READ_ALL);
@@ -579,7 +580,7 @@ extract_chunk_table (
             // file is incomplete (i.e. crashed during write and didn't
             // get a complete chunk table), so just do them one at a time
             if (ctxt->file_size > 0) maxoff = (uint64_t) ctxt->file_size;
-            for (size_t ci = 0; ci < part->chunk_count; ++ci)
+            for (int ci = 0; ci < part->chunk_count; ++ci)
             {
                 uint64_t cchunk = one_to_native64 (ctable[ci]);
                 if (cchunk < chunkoff || cchunk >= maxoff) complete = 0;
@@ -1269,7 +1270,7 @@ exr_read_tile_chunk_info (
         else if (fsize > 0)
         {
             uint64_t finpos = dataoff + (uint64_t) tdata[4];
-            if (finpos > fsize)
+            if (finpos > (uint64_t)fsize)
             {
                 return pctxt->print_error (
                     pctxt,
@@ -2021,7 +2022,10 @@ write_tile_chunk (
         ddata[0] = (int64_t) sample_data_size;
         ddata[1] = (int64_t) packed_size;
         ddata[2] = (int64_t) unpacked_size;
-        rv       = pctxt->do_write (
+
+        priv_from_native64 (ddata, 3);
+
+        rv = pctxt->do_write (
             pctxt, ddata, 3 * sizeof (uint64_t), &(pctxt->output_file_offset));
 
         if (rv == EXR_ERR_SUCCESS)

diff --git a/src/lib/OpenEXRCore/coding.c b/src/lib/OpenEXRCore/coding.c
@@ -136,7 +136,7 @@ internal_encode_free_buffer (
                 encode->free_fn (bufid, curbuf);
             else
             {
-                EXR_PROMOTE_CONST_CONTEXT_AND_PART_OR_ERROR_NO_LOCK (
+                EXR_PROMOTE_CONST_CONTEXT_OR_ERROR_NO_PART_NO_LOCK (
                     encode->context, encode->part_index);
 
                 pctxt->free_fn (curbuf);
@@ -159,7 +159,7 @@ internal_encode_alloc_buffer (
     void* curbuf = *buf;
     if (newsz == 0)
     {
-        EXR_PROMOTE_CONST_CONTEXT_AND_PART_OR_ERROR_NO_LOCK (
+        EXR_PROMOTE_CONST_CONTEXT_OR_ERROR_NO_PART_NO_LOCK (
             encode->context, encode->part_index);
 
         return pctxt->print_error (
@@ -177,15 +177,15 @@ internal_encode_alloc_buffer (
             curbuf = encode->alloc_fn (bufid, newsz);
         else
         {
-            EXR_PROMOTE_CONST_CONTEXT_AND_PART_OR_ERROR_NO_LOCK (
+            EXR_PROMOTE_CONST_CONTEXT_OR_ERROR_NO_PART_NO_LOCK (
                 encode->context, encode->part_index);
 
             curbuf = pctxt->alloc_fn (newsz);
         }
 
         if (curbuf == NULL)
         {
-            EXR_PROMOTE_CONST_CONTEXT_AND_PART_OR_ERROR_NO_LOCK (
+            EXR_PROMOTE_CONST_CONTEXT_OR_ERROR_NO_PART_NO_LOCK (
                 encode->context, encode->part_index);
 
             return pctxt->print_error (
@@ -218,7 +218,7 @@ internal_decode_free_buffer (
                 decode->free_fn (bufid, curbuf);
             else
             {
-                EXR_PROMOTE_CONST_CONTEXT_AND_PART_OR_ERROR_NO_LOCK (
+                EXR_PROMOTE_CONST_CONTEXT_OR_ERROR_NO_PART_NO_LOCK (
                     decode->context, decode->part_index);
 
                 pctxt->free_fn (curbuf);
@@ -256,15 +256,15 @@ internal_decode_alloc_buffer (
             curbuf = decode->alloc_fn (bufid, newsz);
         else
         {
-            EXR_PROMOTE_CONST_CONTEXT_AND_PART_OR_ERROR_NO_LOCK (
+            EXR_PROMOTE_CONST_CONTEXT_OR_ERROR_NO_PART_NO_LOCK (
                 decode->context, decode->part_index);
 
             curbuf = pctxt->alloc_fn (newsz);
         }
 
         if (curbuf == NULL)
         {
-            EXR_PROMOTE_CONST_CONTEXT_AND_PART_OR_ERROR_NO_LOCK (
+            EXR_PROMOTE_CONST_CONTEXT_OR_ERROR_NO_PART_NO_LOCK (
                 decode->context, decode->part_index);
 
             return pctxt->print_error (

diff --git a/src/lib/OpenEXRCore/context.c b/src/lib/OpenEXRCore/context.c
@@ -386,49 +386,6 @@ exr_start_inplace_header_update (
 
 /**************************************/
 
-exr_result_t
-exr_start_memory_write (
-    exr_context_t* ctxt, const exr_context_initializer_t* ctxtdata)
-{
-    int                           rv    = EXR_ERR_UNKNOWN;
-    struct _internal_exr_context* ret   = NULL;
-    exr_context_initializer_t     inits = fill_context_data (ctxtdata);
-
-    if (!ctxt)
-    {
-        inits.error_handler_fn (
-            NULL,
-            EXR_ERR_INVALID_ARGUMENT,
-            "Invalid context handle passed to start_read function");
-        return EXR_ERR_INVALID_ARGUMENT;
-    }
-
-    rv = internal_exr_alloc_context (
-        &ret,
-        &inits,
-        EXR_CONTEXT_WRITE,
-        sizeof (struct _internal_exr_filehandle));
-    if (rv == EXR_ERR_SUCCESS)
-    {
-        ret->do_write = &dispatch_write;
-
-        rv = exr_attr_string_create (
-            (exr_context_t) ret, &(ret->filename), "<memory>");
-
-        if (rv == EXR_ERR_SUCCESS)
-        {
-            if (!inits.write_fn) rv = default_init_write_file (ret);
-
-            if (rv != EXR_ERR_SUCCESS) exr_finish ((exr_context_t*) &ret);
-        }
-    }
-
-    *ctxt = (exr_context_t) ret;
-    return rv;
-}
-
-/**************************************/
-
 exr_result_t
 exr_get_file_name (exr_const_context_t ctxt, const char** name)
 {

diff --git a/src/lib/OpenEXRCore/debug.c b/src/lib/OpenEXRCore/debug.c
@@ -244,14 +244,19 @@ print_attr (const exr_attribute_t* a, int verbose)
             printf ("[ %g, %g, %g ]", a->v3d->x, a->v3d->y, a->v3d->z);
             break;
         case EXR_ATTR_OPAQUE:
+        {
+            uintptr_t faddr_unpack = (uintptr_t)a->opaque->unpack_func_ptr;
+            uintptr_t faddr_pack = (uintptr_t)a->opaque->pack_func_ptr;
+            uintptr_t faddr_destroy = (uintptr_t)a->opaque->destroy_unpacked_func_ptr;
             printf (
                 "(size %d unp size %d hdlrs %p %p %p)",
                 a->opaque->size,
                 a->opaque->unpacked_size,
-                (void*) a->opaque->unpack_func_ptr,
-                (void*) a->opaque->pack_func_ptr,
-                (void*) a->opaque->destroy_unpacked_func_ptr);
+                (void*) faddr_unpack,
+                (void*) faddr_pack,
+                (void*) faddr_destroy);
             break;
+        }
         case EXR_ATTR_UNKNOWN:
         case EXR_ATTR_LAST_KNOWN_TYPE:
         default: printf ("<ERROR Unknown type '%s'>", a->type_name); break;

diff --git a/src/lib/OpenEXRCore/decoding.c b/src/lib/OpenEXRCore/decoding.c
@@ -88,7 +88,7 @@ read_uncompressed_direct (exr_decode_pipeline_t* decode)
     int          height, start_y;
     uint64_t     dataoffset, toread;
     uint8_t*     cdata;
-    EXR_PROMOTE_READ_CONST_CONTEXT_AND_PART_OR_ERROR (
+    EXR_PROMOTE_READ_CONST_CONTEXT_OR_ERROR_NO_PART (
         decode->context, decode->part_index);
 
     dataoffset = decode->chunk.data_offset;

diff --git a/src/lib/OpenEXRCore/internal_b44.c b/src/lib/OpenEXRCore/internal_b44.c
@@ -35,6 +35,7 @@ convertToLinear (uint16_t s[16])
 static inline int
 shiftAndRound (int x, int shift)
 {
+    int a, b;
     //
     // Compute
     //
@@ -47,9 +48,9 @@ shiftAndRound (int x, int shift)
     //
 
     x <<= 1;
-    int a = (1 << shift) - 1;
+    a = (1 << shift) - 1;
     shift += 1;
-    int b = (x >> shift) & 1;
+    b = (x >> shift) & 1;
     return (x + a + b) >> shift;
 }
 
@@ -108,6 +109,8 @@ pack (const uint16_t s[16], uint8_t b[14], int flatfields, int exactmax)
     uint16_t tMax;
     int      shift = -1;
 
+    const int bias = 0x20;
+
     for (int i = 0; i < 16; ++i)
     {
         if ((s[i] & 0x7c00) == 0x7c00)
@@ -131,8 +134,6 @@ pack (const uint16_t s[16], uint8_t b[14], int flatfields, int exactmax)
     // end up between 0 and 63.
     //
 
-    const int bias = 0x20;
-
     do
     {
         shift += 1;
@@ -234,10 +235,11 @@ pack (const uint16_t s[16], uint8_t b[14], int flatfields, int exactmax)
 static inline void
 unpack14 (const uint8_t b[14], uint16_t s[16])
 {
+    uint16_t shift, bias;
     s[0] = ((uint16_t) (b[0] << 8)) | ((uint16_t) b[1]);
 
-    uint16_t shift = (b[2] >> 2);
-    uint16_t bias  = (uint16_t) (0x20u << shift);
+    shift = (b[2] >> 2);
+    bias  = (uint16_t) (0x20u << shift);
 
     s[4] =
         (uint16_t) ((uint32_t) s[0] + (uint32_t) ((((uint32_t) (b[2] << 4) | (uint32_t) (b[3] >> 4)) & 0x3fu) << shift) - bias);
@@ -387,12 +389,14 @@ compress_b44_impl (exr_encode_pipeline_t* encode, int flat_field)
             // by 4, then pad the data by repeating the
             // rightmost column and the bottom row.
             //
+            uint16_t* row0, * row1, * row2, * row3;
 
-            uint16_t* row0 = (uint16_t*) scratch;
+            row0 = (uint16_t*) scratch;
             row0 += y * nx;
-            uint16_t* row1 = row0 + nx;
-            uint16_t* row2 = row1 + nx;
-            uint16_t* row3 = row2 + nx;
+
+            row1 = row0 + nx;
+            row2 = row1 + nx;
+            row3 = row2 + nx;
 
             if (y + 3 >= ny)
             {

diff --git a/src/lib/OpenEXRCore/internal_compress.h b/src/lib/OpenEXRCore/internal_compress.h
@@ -11,6 +11,13 @@
 uint64_t internal_rle_compress (
     void* out, uint64_t outbytes, const void* src, uint64_t srcbytes);
 
+void internal_zip_deconstruct_bytes (
+    uint8_t*       scratch,
+    const uint8_t* source,
+    uint64_t       count);
+
+void internal_zip_reconstruct_bytes (uint8_t* out, uint8_t* scratch_source, uint64_t count);
+
 exr_result_t internal_exr_apply_rle (exr_encode_pipeline_t* encode);
 
 exr_result_t internal_exr_apply_zip (exr_encode_pipeline_t* encode);