From eec0dba242bedd2778c973ae4af112107b33d9c9 Mon Sep 17 00:00:00 2001 From: peterhillman Date: Wed, 7 Oct 2020 09:47:48 +1300 Subject: [PATCH] fix undefined behavior: ignore unused bits in B44 mode detection (#832) Signed-off-by: Peter Hillman --- OpenEXR/IlmImf/ImfB44Compressor.cpp | 37 ++++++++++++++++------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/OpenEXR/IlmImf/ImfB44Compressor.cpp b/OpenEXR/IlmImf/ImfB44Compressor.cpp index 3b18ba8551..0725659dbf 100644 --- a/OpenEXR/IlmImf/ImfB44Compressor.cpp +++ b/OpenEXR/IlmImf/ImfB44Compressor.cpp @@ -381,26 +381,26 @@ unpack14 (const unsigned char b[14], unsigned short s[16]) s[ 0] = (b[0] << 8) | b[1]; unsigned short shift = (b[ 2] >> 2); - unsigned short bias = (0x20 << shift); + unsigned short bias = (0x20u << shift); - s[ 4] = s[ 0] + ((((b[ 2] << 4) | (b[ 3] >> 4)) & 0x3f) << shift) - bias; - s[ 8] = s[ 4] + ((((b[ 3] << 2) | (b[ 4] >> 6)) & 0x3f) << shift) - bias; - s[12] = s[ 8] + ((b[ 4] & 0x3f) << shift) - bias; + s[ 4] = s[ 0] + ((((b[ 2] << 4) | (b[ 3] >> 4)) & 0x3fu) << shift) - bias; + s[ 8] = s[ 4] + ((((b[ 3] << 2) | (b[ 4] >> 6)) & 0x3fu) << shift) - bias; + s[12] = s[ 8] + ((b[ 4] & 0x3fu) << shift) - bias; - s[ 1] = s[ 0] + ((b[ 5] >> 2) << shift) - bias; - s[ 5] = s[ 4] + ((((b[ 5] << 4) | (b[ 6] >> 4)) & 0x3f) << shift) - bias; - s[ 9] = s[ 8] + ((((b[ 6] << 2) | (b[ 7] >> 6)) & 0x3f) << shift) - bias; - s[13] = s[12] + ((b[ 7] & 0x3f) << shift) - bias; + s[ 1] = s[ 0] + ((unsigned int) (b[ 5] >> 2) << shift) - bias; + s[ 5] = s[ 4] + ((((b[ 5] << 4) | (b[ 6] >> 4)) & 0x3fu) << shift) - bias; + s[ 9] = s[ 8] + ((((b[ 6] << 2) | (b[ 7] >> 6)) & 0x3fu) << shift) - bias; + s[13] = s[12] + ((b[ 7] & 0x3fu) << shift) - bias; - s[ 2] = s[ 1] + ((b[ 8] >> 2) << shift) - bias; - s[ 6] = s[ 5] + ((((b[ 8] << 4) | (b[ 9] >> 4)) & 0x3f) << shift) - bias; - s[10] = s[ 9] + ((((b[ 9] << 2) | (b[10] >> 6)) & 0x3f) << shift) - bias; - s[14] = s[13] + ((b[10] & 0x3f) << shift) - bias; + s[ 2] = s[ 1] + ((unsigned int)(b[ 8] >> 2) << shift) - bias; + s[ 6] = s[ 5] + ((((b[ 8] << 4) | (b[ 9] >> 4)) & 0x3fu) << shift) - bias; + s[10] = s[ 9] + ((((b[ 9] << 2) | (b[10] >> 6)) & 0x3fu) << shift) - bias; + s[14] = s[13] + ((b[10] & 0x3fu) << shift) - bias; - s[ 3] = s[ 2] + ((b[11] >> 2) << shift) - bias; - s[ 7] = s[ 6] + ((((b[11] << 4) | (b[12] >> 4)) & 0x3f) << shift) - bias; - s[11] = s[10] + ((((b[12] << 2) | (b[13] >> 6)) & 0x3f) << shift) - bias; - s[15] = s[14] + ((b[13] & 0x3f) << shift) - bias; + s[ 3] = s[ 2] + ((unsigned int)(b[11] >> 2) << shift) - bias; + s[ 7] = s[ 6] + ((((b[11] << 4) | (b[12] >> 4)) & 0x3fu) << shift) - bias; + s[11] = s[10] + ((((b[12] << 2) | (b[13] >> 6)) & 0x3fu) << shift) - bias; + s[15] = s[14] + ((b[13] & 0x3fu) << shift) - bias; for (int i = 0; i < 16; ++i) { @@ -951,7 +951,10 @@ B44Compressor::uncompress (const char *inPtr, if (inSize < 3) notEnoughData(); - if (((const unsigned char *)inPtr)[2] == 0xfc) + // + // If shift exponent is 63, call unpack14 (ignoring unused bits) + // + if (((const unsigned char *)inPtr)[2] >= (13<<2) ) { unpack3 ((const unsigned char *)inPtr, s); inPtr += 3;