avifRGBImageAllocatePixels() returns avifResult

To catch out-of-memory issues.

Change printf(ERROR) into fprintf(stderr,ERROR) in avifyuv.c.
Use RGBImageFreePixels() in RGBImageAllocatePixels().

CHANGELOG.md

@@ -16,6 +16,7 @@ List of incompatible ABI changes in this release:
   those pointers.
 * Check the return value of avifEncoderSetCodecSpecificOption().
 * The maxThreads member was added to the avifRGBImage struct.
+* Check the return value of avifRGBImageAllocatePixels().
 
 ### Added
 * Add STATIC library target avif_internal to allow tests to access functions
@@ -60,6 +61,8 @@ List of incompatible ABI changes in this release:
 * avifEncoderSetCodecSpecificOption() now returns avifResult instead of void to
   report memory allocation failures.
 * At decoding, avifIOStats now returns the same values as at encoding.
+* avifRGBImageAllocatePixels() now returns avifResult instead of void to report
+  memory allocation failures.
 
 ## [0.11.1] - 2022-10-19
 

apps/shared/avifjpeg.c

@@ -366,7 +366,10 @@ avifBool avifJPEGRead(const char * inputFilename,
         rgb.format = AVIF_RGB_FORMAT_RGB;
         rgb.chromaDownsampling = chromaDownsampling;
         rgb.depth = 8;
-        avifRGBImageAllocatePixels(&rgb);
+        if (avifRGBImageAllocatePixels(&rgb) != AVIF_RESULT_OK) {
+            fprintf(stderr, "Conversion to YUV failed: %s (out of memory)\n", inputFilename);
+            goto cleanup;
+        }
 
         int row = 0;
         while (cinfo.output_scanline < cinfo.output_height) {
@@ -557,7 +560,10 @@ avifBool avifJPEGWrite(const char * outputFilename, const avifImage * avif, int
     rgb.format = AVIF_RGB_FORMAT_RGB;
     rgb.chromaUpsampling = chromaUpsampling;
     rgb.depth = 8;
-    avifRGBImageAllocatePixels(&rgb);
+    if (avifRGBImageAllocatePixels(&rgb) != AVIF_RESULT_OK) {
+        fprintf(stderr, "Conversion to RGB failed: %s (out of memory)\n", outputFilename);
+        goto cleanup;
+    }
     if (avifImageYUVToRGB(avif, &rgb) != AVIF_RESULT_OK) {
         fprintf(stderr, "Conversion to RGB failed: %s\n", outputFilename);
         goto cleanup;

apps/shared/avifpng.c

@@ -339,7 +339,10 @@ avifBool avifPNGRead(const char * inputFilename,
     avifRGBImageSetDefaults(&rgb, avif);
     rgb.chromaDownsampling = chromaDownsampling;
     rgb.depth = imgBitDepth;
-    avifRGBImageAllocatePixels(&rgb);
+    if (avifRGBImageAllocatePixels(&rgb) != AVIF_RESULT_OK) {
+        fprintf(stderr, "Conversion to YUV failed: %s (out of memory)\n", inputFilename);
+        goto cleanup;
+    }
     rowPointers = (png_bytep *)malloc(sizeof(png_bytep) * rgb.height);
     for (uint32_t y = 0; y < rgb.height; ++y) {
         rowPointers[y] = &rgb.pixels[y * rgb.rowBytes];
@@ -416,7 +419,10 @@ avifBool avifPNGWrite(const char * outputFilename, const avifImage * avif, uint3
             colorType = PNG_COLOR_TYPE_RGB;
             rgb.format = AVIF_RGB_FORMAT_RGB;
         }
-        avifRGBImageAllocatePixels(&rgb);
+        if (avifRGBImageAllocatePixels(&rgb) != AVIF_RESULT_OK) {
+            fprintf(stderr, "Conversion to RGB failed: %s (out of memory)\n", outputFilename);
+            goto cleanup;
+        }
         if (avifImageYUVToRGB(avif, &rgb) != AVIF_RESULT_OK) {
             fprintf(stderr, "Conversion to RGB failed: %s\n", outputFilename);
             goto cleanup;

examples/avif_example_decode_file.c

@@ -58,10 +58,15 @@ int main(int argc, char * argv[])
 
         // Alternative: set rgb.pixels and rgb.rowBytes yourself, which should match your chosen rgb.format
         // Be sure to use uint16_t* instead of uint8_t* for rgb.pixels/rgb.rowBytes if (rgb.depth > 8)
-        avifRGBImageAllocatePixels(&rgb);
+        result = avifRGBImageAllocatePixels(&rgb);
+        if (result != AVIF_RESULT_OK) {
+            fprintf(stderr, "Allocation of RGB samples failed: %s (%s)\n", inputFilename, avifResultToString(result));
+            goto cleanup;
+        }
 
-        if (avifImageYUVToRGB(decoder->image, &rgb) != AVIF_RESULT_OK) {
-            fprintf(stderr, "Conversion from YUV failed: %s\n", inputFilename);
+        result = avifImageYUVToRGB(decoder->image, &rgb);
+        if (result != AVIF_RESULT_OK) {
+            fprintf(stderr, "Conversion from YUV failed: %s (%s)\n", inputFilename, avifResultToString(result));
             goto cleanup;
         }
 

examples/avif_example_decode_memory.c

@@ -80,10 +80,15 @@ int main(int argc, char * argv[])
 
         // Alternative: set rgb.pixels and rgb.rowBytes yourself, which should match your chosen rgb.format
         // Be sure to use uint16_t* instead of uint8_t* for rgb.pixels/rgb.rowBytes if (rgb.depth > 8)
-        avifRGBImageAllocatePixels(&rgb);
+        result = avifRGBImageAllocatePixels(&rgb);
+        if (result != AVIF_RESULT_OK) {
+            fprintf(stderr, "Allocation of RGB samples failed: %s (%s)\n", inputFilename, avifResultToString(result));
+            goto cleanup;
+        }
 
-        if (avifImageYUVToRGB(decoder->image, &rgb) != AVIF_RESULT_OK) {
-            fprintf(stderr, "Conversion from YUV failed: %s\n", inputFilename);
+        result = avifImageYUVToRGB(decoder->image, &rgb);
+        if (result != AVIF_RESULT_OK) {
+            fprintf(stderr, "Conversion from YUV failed: %s (%s)\n", inputFilename, avifResultToString(result));
             goto cleanup;
         }
 

examples/avif_example_encode.c

@@ -66,7 +66,11 @@ int main(int argc, char * argv[])
 
         // Alternative: set rgb.pixels and rgb.rowBytes yourself, which should match your chosen rgb.format
         // Be sure to use uint16_t* instead of uint8_t* for rgb.pixels/rgb.rowBytes if (rgb.depth > 8)
-        avifRGBImageAllocatePixels(&rgb);
+        avifResult allocationResult = avifRGBImageAllocatePixels(&rgb);
+        if (allocationResult != AVIF_RESULT_OK) {
+            fprintf(stderr, "Allocation of RGB samples failed: %s\n", avifResultToString(allocationResult));
+            goto cleanup;
+        }
 
         // Fill your RGB(A) data here
         memset(rgb.pixels, 255, rgb.rowBytes * image->height);

include/avif/avif.h

@@ -680,7 +680,7 @@ AVIF_API void avifRGBImageSetDefaults(avifRGBImage * rgb, const avifImage * imag
 AVIF_API uint32_t avifRGBImagePixelSize(const avifRGBImage * rgb);
 
 // Convenience functions. If you supply your own pixels/rowBytes, you do not need to use these.
-AVIF_API void avifRGBImageAllocatePixels(avifRGBImage * rgb);
+AVIF_API avifResult avifRGBImageAllocatePixels(avifRGBImage * rgb);
 AVIF_API void avifRGBImageFreePixels(avifRGBImage * rgb);
 
 // The main conversion functions

src/avif.c

@@ -579,14 +579,14 @@ void avifRGBImageSetDefaults(avifRGBImage * rgb, const avifImage * image)
     rgb->maxThreads = 1;
 }
 
-void avifRGBImageAllocatePixels(avifRGBImage * rgb)
+avifResult avifRGBImageAllocatePixels(avifRGBImage * rgb)
 {
-    if (rgb->pixels) {
-        avifFree(rgb->pixels);
-    }
-
-    rgb->rowBytes = rgb->width * avifRGBImagePixelSize(rgb);
-    rgb->pixels = avifAlloc((size_t)rgb->rowBytes * rgb->height);
+    avifRGBImageFreePixels(rgb);
+    const uint32_t rowBytes = rgb->width * avifRGBImagePixelSize(rgb);
+    rgb->pixels = avifAlloc((size_t)rowBytes * rgb->height);
+    AVIF_CHECKERR(rgb->pixels, AVIF_RESULT_OUT_OF_MEMORY);
+    rgb->rowBytes = rowBytes;
+    return AVIF_RESULT_OK;
 }
 
 void avifRGBImageFreePixels(avifRGBImage * rgb)

tests/avifyuv.c

@@ -151,7 +151,7 @@ int main(int argc, char * argv[])
 
                         avifImage * image = avifImageCreate(dim, dim, yuvDepth, AVIF_PIXEL_FORMAT_YUV444);
                         if (!image) {
-                            printf("ERROR: Out of memory\n");
+                            fprintf(stderr, "ERROR: Out of memory\n");
                             return 1;
                         }
                         image->colorPrimaries = cicp->cp;
@@ -168,13 +168,23 @@ int main(int argc, char * argv[])
                         avifRGBImageSetDefaults(&srcRGB, image);
                         srcRGB.format = AVIF_RGB_FORMAT_RGB;
                         srcRGB.depth = rgbDepth;
-                        avifRGBImageAllocatePixels(&srcRGB);
 
                         avifRGBImage dstRGB;
                         avifRGBImageSetDefaults(&dstRGB, image);
                         dstRGB.format = AVIF_RGB_FORMAT_RGB;
                         dstRGB.depth = rgbDepth;
-                        avifRGBImageAllocatePixels(&dstRGB);
+
+                        if ((avifRGBImageAllocatePixels(&srcRGB) != AVIF_RESULT_OK)) {
+                            avifImageDestroy(image);
+                            fprintf(stderr, "ERROR: Out of memory\n");
+                            return 1;
+                        }
+                        if ((avifRGBImageAllocatePixels(&dstRGB) != AVIF_RESULT_OK)) {
+                            avifRGBImageFreePixels(&srcRGB);
+                            avifImageDestroy(image);
+                            fprintf(stderr, "ERROR: Out of memory\n");
+                            return 1;
+                        }
 
                         uint64_t driftPixelCounts[MAX_DRIFT];
                         for (int i = 0; i < MAX_DRIFT; ++i) {
@@ -259,7 +269,10 @@ int main(int argc, char * argv[])
                                             maxDrift = drift;
                                         }
                                     } else {
-                                        printf("ERROR: Encountered a drift greater than or equal to MAX_DRIFT(%d): %d\n", MAX_DRIFT, drift);
+                                        fprintf(stderr,
+                                                "ERROR: Encountered a drift greater than or equal to MAX_DRIFT(%d): %d\n",
+                                                MAX_DRIFT,
+                                                drift);
                                         return 1;
                                     }
                                 }
@@ -307,7 +320,7 @@ int main(int argc, char * argv[])
 
         avifImage * image = avifImageCreate(originalWidth, originalHeight, 8, AVIF_PIXEL_FORMAT_YUV444);
         if (!image) {
-            printf("ERROR: Out of memory\n");
+            fprintf(stderr, "ERROR: Out of memory\n");
             return 1;
         }
 
@@ -317,7 +330,11 @@ int main(int argc, char * argv[])
             avifRGBImage srcRGB;
             avifRGBImageSetDefaults(&srcRGB, image);
             srcRGB.depth = yuvDepth;
-            avifRGBImageAllocatePixels(&srcRGB);
+            if (avifRGBImageAllocatePixels(&srcRGB) != AVIF_RESULT_OK) {
+                avifImageDestroy(image);
+                fprintf(stderr, "ERROR: Out of memory\n");
+                return 1;
+            }
             if (yuvDepth > 8) {
                 float maxChannelF = (float)((1 << yuvDepth) - 1);
                 for (uint32_t j = 0; j < srcRGB.height; ++j) {
@@ -362,7 +379,12 @@ int main(int argc, char * argv[])
                         avifRGBImageSetDefaults(&intermediateRGB, image);
                         intermediateRGB.depth = rgbDepth;
                         intermediateRGB.format = rgbFormat;
-                        avifRGBImageAllocatePixels(&intermediateRGB);
+                        if (avifRGBImageAllocatePixels(&intermediateRGB) != AVIF_RESULT_OK) {
+                            avifRGBImageFreePixels(&srcRGB);
+                            avifImageDestroy(image);
+                            fprintf(stderr, "ERROR: Out of memory\n");
+                            return 1;
+                        }
                         avifImageYUVToRGB(image, &intermediateRGB);
 
                         avifImageFreePlanes(image, AVIF_PLANES_ALL);
@@ -371,7 +393,13 @@ int main(int argc, char * argv[])
                         avifRGBImage dstRGB;
                         avifRGBImageSetDefaults(&dstRGB, image);
                         dstRGB.depth = yuvDepth;
-                        avifRGBImageAllocatePixels(&dstRGB);
+                        if (avifRGBImageAllocatePixels(&dstRGB) != AVIF_RESULT_OK) {
+                            avifRGBImageFreePixels(&intermediateRGB);
+                            avifRGBImageFreePixels(&srcRGB);
+                            avifImageDestroy(image);
+                            fprintf(stderr, "ERROR: Out of memory\n");
+                            return 1;
+                        }
                         avifImageYUVToRGB(image, &dstRGB);
 
                         avifBool moveOn = AVIF_FALSE;
@@ -463,7 +491,10 @@ int main(int argc, char * argv[])
             for (int i = 0; i < MAX_DRIFT; ++i) {
                 driftPixelCounts[i] = 0;
             }
-            avifRGBImageAllocatePixels(&rgb);
+            if (avifRGBImageAllocatePixels(&rgb) != AVIF_RESULT_OK) {
+                fprintf(stderr, "ERROR: Out of memory\n");
+                return 1;
+            }
 
             for (uint32_t a = 0; a < size; ++a) {
                 // meaningful premultiplied RGB value can't exceed A value, so stop at R = A
@@ -492,12 +523,13 @@ int main(int argc, char * argv[])
                         uint8_t * pixel = &rgb.pixels[r * sizeof(uint8_t) * 4];
                         int drift = abs((int)pixel[0] - (int)r);
                         if (drift >= MAX_DRIFT) {
-                            printf("ERROR: Premultiply round-trip difference greater than or equal to MAX_DRIFT(%d): RGB depth: %d, src: %d, dst: %d, alpha: %d.\n",
-                                   MAX_DRIFT,
-                                   rgbDepth,
-                                   pixel[0],
-                                   r,
-                                   a);
+                            fprintf(stderr,
+                                    "ERROR: Premultiply round-trip difference greater than or equal to MAX_DRIFT(%d): RGB depth: %d, src: %d, dst: %d, alpha: %d.\n",
+                                    MAX_DRIFT,
+                                    rgbDepth,
+                                    pixel[0],
+                                    r,
+                                    a);
                             return 1;
                         }
                         if (maxDrift < drift) {
@@ -508,12 +540,13 @@ int main(int argc, char * argv[])
                         uint16_t * pixel = (uint16_t *)&rgb.pixels[r * sizeof(uint16_t) * 4];
                         int drift = abs((int)pixel[0] - (int)r);
                         if (drift >= MAX_DRIFT) {
-                            printf("ERROR: Premultiply round-trip difference greater than or equal to MAX_DRIFT(%d): RGB depth: %d, src: %d, dst: %d, alpha: %d.\n",
-                                   MAX_DRIFT,
-                                   rgbDepth,
-                                   pixel[0],
-                                   r,
-                                   a);
+                            fprintf(stderr,
+                                    "ERROR: Premultiply round-trip difference greater than or equal to MAX_DRIFT(%d): RGB depth: %d, src: %d, dst: %d, alpha: %d.\n",
+                                    MAX_DRIFT,
+                                    rgbDepth,
+                                    pixel[0],
+                                    r,
+                                    a);
                             return 1;
                         }
                         if (maxDrift < drift) {

tests/gtest/aviftest_helpers.cc

@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
+#include <cstdlib>
 #include <string>
 
 #include "avif/avif.h"
@@ -21,7 +22,9 @@ AvifRgbImage::AvifRgbImage(const avifImage* yuv, int rgbDepth,
   avifRGBImageSetDefaults(this, yuv);
   depth = rgbDepth;
   format = rgbFormat;
-  avifRGBImageAllocatePixels(this);
+  if (avifRGBImageAllocatePixels(this) != AVIF_RESULT_OK) {
+    std::abort();
+  }
 }
 
 AvifRwData::AvifRwData(AvifRwData&& other) : avifRWData{other} {

tests/oss-fuzz/avif_decode_fuzzer.cc

@@ -50,7 +50,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t * Data, size_t Size)
                             rgb.depth = rgbDepths[rgbDepthsIndex];
                             rgb.chromaUpsampling = upsamplings[upsamplingsIndex];
                             rgb.avoidLibYUV = AVIF_TRUE;
-                            avifRGBImageAllocatePixels(&rgb);
+                            if (avifRGBImageAllocatePixels(&rgb) != AVIF_RESULT_OK) {
+                                continue;
+                            }
                             avifResult rgbResult = avifImageYUVToRGB(decoder->image, &rgb);
                             // Since avifImageRGBToYUV() ignores rgb.chromaUpsampling, we only need
                             // to test avifImageRGBToYUV() with a single upsamplingsIndex.
@@ -61,9 +63,10 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t * Data, size_t Size)
                                                                             decoder->image->height,
                                                                             yuvDepths[yuvDepthsIndex],
                                                                             decoder->image->yuvFormat);
-                                    avifResult yuvResult = avifImageRGBToYUV(tempImage, &rgb);
-                                    if (yuvResult != AVIF_RESULT_OK) {
+                                    if (!tempImage) {
+                                        continue;
                                     }
+                                    (void)avifImageRGBToYUV(tempImage, &rgb);
                                     avifImageDestroy(tempImage);
                                 }
                             }