Fix invalid memory access in ScalerFastBoxSampling (#513)

[src-px]->[dst-px] of 4x1px to 2x1px scaling was 0->0, 1->1, 2->1, 3->2(out of bounds) Fix it to be 0->0, 1->0, 2->1, 3->1
pion · Jul 5, 2023 · 72c1d7b · 72c1d7b
1 parent fc301a8
commit 72c1d7b
Show file tree

Hide file tree

Showing 2 changed files with 98 additions and 17 deletions.
diff --git a/pkg/io/video/scale_test.go b/pkg/io/video/scale_test.go
@@ -538,19 +538,19 @@ func TestScaleFastBoxSampling(t *testing.T) {
 			src: &image.YCbCr{
 				SubsampleRatio: image.YCbCrSubsampleRatio420,
 				Y: []uint8{
-					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
-					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
+					0xF0, 0xF8, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
+					0xF8, 0xF0, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF0, 0x10, 0x10,
 					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
 					0x00, 0x00, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00,
-					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-					0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+					0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00,
+					0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00,
 					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
 					0x00, 0x00, 0x80, 0x80, 0x30, 0x30, 0x00, 0x00, 0x30, 0x30, 0x00, 0x00,
 				},
 				Cb: []uint8{
 					0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
 					0x20, 0x20, 0x80, 0x80, 0x50, 0x50,
-					0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
+					0x80, 0x80, 0xE0, 0xE0, 0x34, 0x34,
 					0x80, 0x80, 0xE0, 0xE0, 0x30, 0x30,
 				},
 				Cr: []uint8{
@@ -568,24 +568,104 @@ func TestScaleFastBoxSampling(t *testing.T) {
 			expected: &image.YCbCr{
 				SubsampleRatio: image.YCbCrSubsampleRatio420,
 				Y: []uint8{
-					0xF0, 0x80, 0x08, 0x00, 0x78, 0x80,
-					0x08, 0x00, 0x20, 0x20, 0x20, 0x20,
-					0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-					0x00, 0x40, 0x58, 0x18, 0x18, 0x18,
+					0xF4, 0x10, 0x00, 0x00, 0xF0, 0x10,
+					0x00, 0x00, 0x40, 0x00, 0x40, 0x00,
+					0x04, 0x00, 0x00, 0x00, 0x02, 0x00,
+					0x00, 0x80, 0x30, 0x00, 0x30, 0x00,
 				},
 				Cb: []uint8{
-					0x20, 0x50, 0x68,
-					0x68, 0xB0, 0x88,
+					0x20, 0x80, 0x50,
+					0x80, 0xE0, 0x32,
 				},
 				Cr: []uint8{
-					0xE0, 0xB0, 0x98,
-					0xD0, 0x98, 0x80,
+					0xE0, 0x80, 0xB0,
+					0xF0, 0x40, 0xC0,
 				},
 				YStride: 6,
 				CStride: 3,
 				Rect:    image.Rect(0, 0, 6, 4),
 			},
 		},
+		"I420Uneven6x4to3x2": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x40,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80,
+					0x20, 0x20, 0x80,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80,
+					0xE0, 0xE0, 0x80,
+					0xFF, // dummy data to detect out of range read
+				},
+				YStride: 6,
+				CStride: 3,
+				Rect:    image.Rect(0, 0, 6, 3),
+			},
+			width:  4,
+			height: 2,
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0x10, 0x08, 0x00,
+					0x00, 0x00, 0x20, 0x40,
+				},
+				Cb: []uint8{
+					0x20, 0x80,
+				},
+				Cr: []uint8{
+					0xE0, 0x80,
+				},
+				YStride: 4,
+				CStride: 2,
+				Rect:    image.Rect(0, 0, 4, 2),
+			},
+		},
+		"I420Uneven6x3to5x2": {
+			src: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00,
+					0xF0, 0xF0, 0x10, 0x10, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x00, 0x40, 0x50,
+				},
+				Cb: []uint8{
+					0x20, 0x20, 0x80,
+					0x20, 0x20, 0x80,
+				},
+				Cr: []uint8{
+					0xE0, 0xE0, 0x80,
+					0xE0, 0xE0, 0x80,
+					0xFF, // dummy data to detect out of range read
+				},
+				YStride: 6,
+				CStride: 3,
+				Rect:    image.Rect(0, 0, 6, 3),
+			},
+			width:  5,
+			height: 2,
+			expected: &image.YCbCr{
+				SubsampleRatio: image.YCbCrSubsampleRatio420,
+				Y: []uint8{
+					0xF0, 0x10, 0x10, 0x00, 0x00,
+					0x00, 0x00, 0x00, 0x40, 0x50,
+				},
+				Cb: []uint8{
+					0x20, 0x80, 0x00,
+				},
+				Cr: []uint8{
+					0xE0, 0x80, 0x00,
+				},
+				YStride: 5,
+				CStride: 3,
+				Rect:    image.Rect(0, 0, 5, 2),
+			},
+		},
 	}
 
 	for name, c := range cases {

diff --git a/pkg/io/video/scaler_cgo.c b/pkg/io/video/scaler_cgo.c
@@ -29,7 +29,7 @@ void fastBoxSampling(
     const int sw, const int sh, const int sstride,
     uint32_t* tmp)
 {
-  memset(tmp, 0, dw * dh * ch * sizeof(tmp[0]));
+  memset(tmp, 0, dstride * dh * ch * sizeof(tmp[0]));
 
   for (int sy = 0; sy < sh; sy++)
   {
@@ -39,13 +39,14 @@ void fastBoxSampling(
     uint32_t* tmp2 = &tmp[ty * dstride];
     for (int sx = 0; sx < sw * ch; sx += ch)
     {
-      if (tx * sw < sx * dw)
-        tx += ch;
-
       for (int c = 0; c < ch; c++)
       {
         tmp2[tx + c] += 0x10000 | src2[sx + c];
       }
+      if (tx * sw < sx * dw)
+      {
+        tx += ch;
+      }
     }
   }