avifRWStreamWriteBits: Check v for valid range

Make sure the input parameter `v` can be represented in `bitCount` bits.
This was checked by an assertion. Replace the assertion by an error
return.

src/stream.c

@@ -479,7 +479,7 @@ avifResult avifRWStreamWriteZeros(avifRWStream * stream, size_t byteCount)
 
 avifResult avifRWStreamWriteBits(avifRWStream * stream, uint32_t v, size_t bitCount)
 {
-    assert(((uint64_t)v >> bitCount) == 0); // (uint32_t >> 32 is undefined behavior)
+    AVIF_CHECKERR(bitCount >= 32 || (v >> bitCount) == 0, AVIF_RESULT_INVALID_ARGUMENT);
     while (bitCount) {
         if (stream->numUsedBitsInPartialByte == 0) {
             AVIF_CHECKRES(makeRoom(stream, 1)); // Book a new partial byte in the stream.

tests/gtest/avifstreamtest.cc

@@ -203,6 +203,15 @@ TEST(StreamTest, Roundtrip) {
   EXPECT_FALSE(avifROStreamSkip(&ro_stream, /*byteCount=*/1));
 }
 
+TEST(StreamTest, WriteBitsLimit) {
+  testutil::AvifRwData rw_data;
+  avifRWStream rw_stream;
+  avifRWStreamStart(&rw_stream, &rw_data);
+  EXPECT_EQ(avifRWStreamWriteBits(&rw_stream, 7, 3), AVIF_RESULT_OK);
+  EXPECT_EQ(avifRWStreamWriteBits(&rw_stream, 8, 3),
+            AVIF_RESULT_INVALID_ARGUMENT);
+}
+
 //------------------------------------------------------------------------------
 // Variable length integer implementation