[3.9] gh-98517: Fix buffer overflows in _sha3 module (GH-98519) (#98526)

This is a port of the applicable part of XKCP's fix [1] for
CVE-2022-37454 and avoids the segmentation fault and the infinite
loop in the test cases published in [2].

[1]: XKCP/XKCP@fdc6fef
[2]: https://mouha.be/sha-3-buffer-overflow/

Regression test added by: Gregory P. Smith [Google LLC] <greg@krypto.org>
(cherry picked from commit 0e4e058)

Co-authored-by: Theo Buehler <botovq@users.noreply.github.com>

Lib/test/test_hashlib.py

@@ -497,6 +497,15 @@ def test_case_md5_huge(self, size):
     def test_case_md5_uintmax(self, size):
         self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3')
 
+    @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems')
+    @bigmemtest(size=_4G - 1, memuse=1, dry_run=False)
+    def test_sha3_update_overflow(self, size):
+        """Regression test for gh-98517 CVE-2022-37454."""
+        h = hashlib.sha3_224()
+        h.update(b'\x01')
+        h.update(b'\x01'*0xffff_ffff)
+        self.assertEqual(h.hexdigest(), '80762e8ce6700f114fec0f621fd97c4b9c00147fa052215294cceeed')
+
     # use the three examples from Federal Information Processing Standards
     # Publication 180-1, Secure Hash Standard,  1995 April 17
     # http://www.itl.nist.gov/div897/pubs/fip180-1.htm

Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst

@@ -0,0 +1 @@
+Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454).

Modules/_sha3/kcp/KeccakSponge.inc

@@ -171,7 +171,7 @@ int SpongeAbsorb(SpongeInstance *instance, const unsigned char *data, size_t dat
     i = 0;
     curData = data;
     while(i < dataByteLen) {
-        if ((instance->byteIOIndex == 0) && (dataByteLen >= (i + rateInBytes))) {
+        if ((instance->byteIOIndex == 0) && (dataByteLen-i >= rateInBytes)) {
 #ifdef SnP_FastLoop_Absorb
             /* processing full blocks first */
 
@@ -199,10 +199,10 @@ int SpongeAbsorb(SpongeInstance *instance, const unsigned char *data, size_t dat
         }
         else {
             /* normal lane: using the message queue */
-
-            partialBlock = (unsigned int)(dataByteLen - i);
-            if (partialBlock+instance->byteIOIndex > rateInBytes)
+            if (dataByteLen-i > rateInBytes-instance->byteIOIndex)
                 partialBlock = rateInBytes-instance->byteIOIndex;
+            else
+                partialBlock = (unsigned int)(dataByteLen - i);
             #ifdef KeccakReference
             displayBytes(1, "Block to be absorbed (part)", curData, partialBlock);
             #endif
@@ -281,7 +281,7 @@ int SpongeSqueeze(SpongeInstance *instance, unsigned char *data, size_t dataByte
     i = 0;
     curData = data;
     while(i < dataByteLen) {
-        if ((instance->byteIOIndex == rateInBytes) && (dataByteLen >= (i + rateInBytes))) {
+        if ((instance->byteIOIndex == rateInBytes) && (dataByteLen-i >= rateInBytes)) {
             for(j=dataByteLen-i; j>=rateInBytes; j-=rateInBytes) {
                 SnP_Permute(instance->state);
                 SnP_ExtractBytes(instance->state, curData, 0, rateInBytes);
@@ -299,9 +299,10 @@ int SpongeSqueeze(SpongeInstance *instance, unsigned char *data, size_t dataByte
                 SnP_Permute(instance->state);
                 instance->byteIOIndex = 0;
             }
-            partialBlock = (unsigned int)(dataByteLen - i);
-            if (partialBlock+instance->byteIOIndex > rateInBytes)
+            if (dataByteLen-i > rateInBytes-instance->byteIOIndex)
                 partialBlock = rateInBytes-instance->byteIOIndex;
+            else
+                partialBlock = (unsigned int)(dataByteLen - i);
             i += partialBlock;
 
             SnP_ExtractBytes(instance->state, curData, instance->byteIOIndex, partialBlock);