fix: mem cache out of range panic caused by overflow

pkg/proc/mem.go

@@ -35,7 +35,12 @@ type memCache struct {
 }
 
 func (m *memCache) contains(addr uint64, size int) bool {
-	return addr >= m.cacheAddr && addr <= (m.cacheAddr+uint64(len(m.cache)-size))
+	end := addr + uint64(size)
+	if end < addr {
+		// overflow
+		return false
+	}
+	return addr >= m.cacheAddr && end <= m.cacheAddr+uint64(len(m.cache))
 }
 
 func (m *memCache) ReadMemory(data []byte, addr uint64) (n int, err error) {
@@ -69,6 +74,10 @@ func cacheMemory(mem MemoryReadWriter, addr uint64, size int) MemoryReadWriter {
 	if size <= 0 {
 		return mem
 	}
+	if addr+uint64(size) < addr {
+		// overflow
+		return mem
+	}
 	switch cacheMem := mem.(type) {
 	case *memCache:
 		if cacheMem.contains(addr, size) {

pkg/proc/proc_general_test.go

@@ -30,6 +30,19 @@ func TestIssue554(t *testing.T) {
 	}
 }
 
+func TestIssue3760(t *testing.T) {
+	// unsigned integer overflow if len(m.cache) < size
+	mem := memCache{true, 0x20, make([]byte, 100), nil}
+	if mem.contains(0x20, 200) {
+		t.Fatalf("should be false")
+	}
+
+	cm := cacheMemory(nil, 0xffffffffffffffff, 10)
+	if cm != nil {
+		t.Fatalf("should be nil")
+	}
+}
+
 type dummyMem struct {
 	t     *testing.T
 	mem   []byte