Merge branch 'bugfix/cache_writeback_bug_v4.4' into 'release/v4.4'

esp_rom: patch Cache_WriteBack_Addr (backport v4.4)

See merge request espressif/esp-idf!24939

components/esp_rom/CMakeLists.txt

@@ -30,6 +30,10 @@ if(target STREQUAL "esp32s2" OR target STREQUAL "esp32s3")
     list(APPEND sources "patches/esp_rom_cache.c")
 endif()
 
+if(target STREQUAL "esp32s3")
+    list(APPEND sources "patches/esp_rom_cache_writeback_esp32s3.S")
+endif()
+
 idf_component_register(SRCS ${sources}
                        INCLUDE_DIRS ${include_dirs}
                        PRIV_REQUIRES ${private_required_comp})

components/esp_rom/component.mk

@@ -1,8 +1,10 @@
 COMPONENT_ADD_INCLUDEDIRS := include esp32 include/esp32
 COMPONENT_SRCDIRS := patches .
 
+COMPONENT_OBJEXCLUDE := patches/esp_rom_cache_writeback_esp32s3.o
+
 ifdef IS_BOOTLOADER_BUILD
-COMPONENT_OBJEXCLUDE := patches/esp_rom_longjmp.o
+COMPONENT_OBJEXCLUDE += patches/esp_rom_longjmp.o
 endif
 
 #Linker scripts used to link the final application.

components/esp_rom/esp32s3/esp_rom_caps.h

@@ -25,3 +25,4 @@
 #define ESP_ROM_HAS_ETS_PRINTF_BUG          (1) // ROM has ets_printf bug when disable the ROM log either by eFuse or RTC storage register
 #define ESP_ROM_HAS_FLASH_COUNT_PAGES_BUG   (1) // ROM api Cache_Count_Flash_Pages will return unexpected value
 #define ESP_ROM_HAS_CACHE_SUSPEND_WAITI_BUG (1) // ROM api Cache_Suspend_I/DCache and Cache_Freeze_I/DCache_Enable does not waiti
+#define ESP_ROM_HAS_CACHE_WRITEBACK_BUG     (1) // ROM api Cache_WriteBack_Addr access cacheline being writen back may cause cache hit with wrong value.

components/esp_rom/esp32s3/ld/esp32s3.rom.ld

@@ -381,7 +381,7 @@ PROVIDE( Cache_WriteBack_Items = 0x40001698 );
 PROVIDE( Cache_Op_Addr = 0x400016a4 );
 PROVIDE( Cache_Invalidate_Addr = 0x400016b0 );
 PROVIDE( Cache_Clean_Addr = 0x400016bc );
-PROVIDE( Cache_WriteBack_Addr = 0x400016c8 );
+PROVIDE( rom_Cache_WriteBack_Addr = 0x400016c8 );
 PROVIDE( Cache_Invalidate_ICache_All = 0x400016d4 );
 PROVIDE( Cache_Invalidate_DCache_All = 0x400016e0 );
 PROVIDE( Cache_Clean_All = 0x400016ec );

components/esp_rom/patches/esp_rom_cache.c

@@ -10,10 +10,14 @@
 #include "esp_attr.h"
 #include "esp_rom_caps.h"
 #include "soc/extmem_reg.h"
+#include "xtensa/xtruntime.h"
 #if CONFIG_IDF_TARGET_ESP32S3
 #include "esp32s3/rom/cache.h"
 #endif
 
+#define ALIGN_UP(addr, align) (((addr) + (align)-1) & ~((align)-1))
+#define ALIGN_DOWN(addr, align)  ((addr) & ~((align) - 1))
+
 // this api is renamed for patch
 extern uint32_t rom_Cache_Count_Flash_Pages(uint32_t bus, uint32_t * page0_mapped);
 IRAM_ATTR uint32_t Cache_Count_Flash_Pages(uint32_t bus, uint32_t * page0_mapped)
@@ -83,4 +87,71 @@ IRAM_ATTR void Cache_Freeze_DCache_Enable(cache_freeze_mode_t mode)
 }
 extern void Cache_Freeze_DCache_Enable(cache_freeze_mode_t mode);
 #endif
+
+#if ESP_ROM_HAS_CACHE_WRITEBACK_BUG
+/* Defined in esp_rom_cache_writeback_esp32s3.S */
+extern void cache_writeback_items_freeze(uint32_t addr, uint32_t items);
+// renamed for patch
+extern int rom_Cache_WriteBack_Addr(uint32_t addr, uint32_t size);
+IRAM_ATTR int Cache_WriteBack_Addr(uint32_t addr, uint32_t size)
+{
+    /* Do special processing for unaligned memory at the start and end of the cache writeback memory.
+     * 1. Disable the interrupt to prevent the current CPU accessing the same cacheline.
+     * 2. Enable dcache freeze to prevent the another CPU accessing the same cacheline.
+     */
+    uint32_t irq_status;
+    uint32_t start_len, end_len;
+    uint32_t start, end;
+    uint32_t dcache_line_size;
+    uint32_t autoload;
+    int ret = 0;
+    start = addr;
+    end = addr + size;
+    dcache_line_size = Cache_Get_DCache_Line_Size();
+
+    if (size == 0) {
+        return 0;
+    }
+
+    /*the start address is unaligned*/
+    if (start & (dcache_line_size -1)) {
+        addr = ALIGN_UP(start, dcache_line_size);
+        start_len = addr - start;
+        size = (size < start_len) ? 0 : (size - start_len);
+
+        /*writeback start unaligned mem, one cacheline*/
+        irq_status = XTOS_SET_INTLEVEL(XCHAL_NMILEVEL);//mask all interrupts
+        cache_writeback_items_freeze(start, 1);
+        XTOS_RESTORE_INTLEVEL(irq_status);
+
+        if (size == 0) {
+            return 0;
+        }
+    }
+
+    /*the end address is unaligned*/
+    if (end & (dcache_line_size -1)) {
+        end = ALIGN_DOWN(end, dcache_line_size);
+        end_len = addr + size - end;
+        size = (size - end_len);
+
+        /*writeback end unaligned mem, one cacheline*/
+        irq_status = XTOS_SET_INTLEVEL(XCHAL_NMILEVEL);//mask all interrupts
+        cache_writeback_items_freeze(end, 1);
+        XTOS_RESTORE_INTLEVEL(irq_status);
+
+        if (size == 0) {
+            return 0;
+        }
+    }
+
+    /*suspend autoload, avoid load cachelines being written back*/
+    autoload = Cache_Suspend_DCache_Autoload();
+    ret = rom_Cache_WriteBack_Addr(addr, size);
+    Cache_Resume_DCache_Autoload(autoload);
+
+    return ret;
+}
+extern int Cache_WriteBack_Addr(uint32_t addr, uint32_t size);
+#endif  //ESP_ROM_HAS_CACHE_WRITEBACK_BUG
 #endif

components/esp_rom/patches/esp_rom_cache_writeback_esp32s3.S

@@ -0,0 +1,135 @@
+/*
+ * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "sdkconfig.h"
+#include "esp_bit_defs.h"
+#include "soc/extmem_reg.h"
+
+#if CONFIG_IDF_TARGET_ESP32S3
+/**
+  * @brief Write back the cache items of DCache, enable cache freeze during writeback.
+  *        Operation will be done CACHE_LINE_SIZE aligned.
+  *        If the region is not in DCache addr room, nothing will be done.
+  *        Please do not call this function in your SDK application.
+  * @param  uint32_t addr: start address to write back
+  * @param  uint32_t items: cache lines to invalidate, items * cache_line_size should
+  *         not exceed the bus address size(4MB)
+  *
+  * void cache_writeback_items_freeze(uint32_t addr, uint32_t items)
+*/
+
+/*******************************************************************************
+
+This function is a cache write-back function that works around the following
+hardware errata on the ESP32-S3:
+
+- Core X manually triggers (via the EXTMEM_DCACHE_SYNC_CTRL_REG register) the
+write-back of one or more cache lines.
+- While the write-back is in progress, there are two scenarios that may cause
+cache hit error.
+    - Core X enters the interrupt handler and access the same cache line
+      being written back.
+    - Core Y access the same cache line being written back.
+
+To workaround this errata, the following steps must be taken when manually
+triggering a cache write-back:
+
+- Core X must disable interrupts so that it cannot be preempted
+- Core X must freeze the cache (via the EXTMEM_DCACHE_FREEZE_REG register) to
+prevent Core Y from accessing the same cache lines that are about to be written
+back.
+- Core X now triggers the cache write-back. During the write-back...
+    - If Core Y attempts the access any address in the cache region, Core Y will
+    busy wait until the cache is unfrozen.
+    - Core X must ensure that it does not access any address in the cache region,
+    otherwise Core X will busy wait thus causing a deadlock.
+- After the write-back is complete, Core X unfreezes the cache, and reenables
+interrupts.
+
+Notes:
+
+- Please do not modify this function, it must strictly follow the current execution
+sequence, otherwise it may cause unexpected errors.
+- This function is written in assmebly to ensure that the function itself never
+accesses any cache address while the cache is frozen. Unexpected cache access
+could occur if...
+    - the function triggers an window overflow onto a stack placed in PSRAM.
+    Thus, we only use two window panes (a0 to a8) in this function and trigger
+    all window overflows before freezing the cache.
+    - the function accesses literals/read-only variables placed in Flash.
+
+*******************************************************************************/
+
+    .section .iram1,"ax"
+    .align  4
+    /*
+    Create dedicated literal pool for this function. Mostly used to store out
+    of range movi transformations.
+    */
+    .literal_position
+    .global cache_writeback_items_freeze
+    .type   cache_writeback_items_freeze, @function
+cache_writeback_items_freeze:
+    entry   sp, 32
+
+    /* REG_WRITE(EXTMEM_DCACHE_SYNC_ADDR_REG, addr); */
+    movi a4, EXTMEM_DCACHE_SYNC_ADDR_REG
+    s32i a2, a4, 0
+    /* REG_WRITE(EXTMEM_DCACHE_SYNC_SIZE_REG, items); */
+    movi a4, EXTMEM_DCACHE_SYNC_SIZE_REG
+    s32i a3, a4, 0
+    memw    /* About to freeze the cache. Ensure all previous memory R/W are completed */
+
+    movi a2, EXTMEM_DCACHE_FREEZE_REG
+    movi a3, EXTMEM_DCACHE_SYNC_CTRL_REG
+
+    /*
+    REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_MODE);
+    REG_SET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA);
+    */
+    l32i a4, a2, 0  /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */
+    movi a5, ~(EXTMEM_DCACHE_FREEZE_MODE_M)
+    and a4, a4, a5
+    movi a5, EXTMEM_DCACHE_FREEZE_ENA_M
+    or a4, a4, a5
+    s32i a4, a2, 0  /* *(EXTMEM_DCACHE_FREEZE_REG) = a4 */
+
+    /* while (!REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); */
+    movi a5, EXTMEM_DCACHE_FREEZE_DONE_M
+_wait_freeze_done:
+    l32i a4, a2, 0  /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */
+    memw
+    bnone a4, a5, _wait_freeze_done
+
+    /* REG_SET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_WRITEBACK_ENA); */
+    l32i a4, a3, 0  /* a4 = *(EXTMEM_DCACHE_SYNC_CTRL_REG) */
+    movi a5, EXTMEM_DCACHE_WRITEBACK_ENA_M
+    or a4, a4, a5
+    s32i a4, a3, 0  /* *(EXTMEM_DCACHE_SYNC_CTRL_REG) = a4 */
+
+    /* while(!REG_GET_BIT(EXTMEM_DCACHE_SYNC_CTRL_REG, EXTMEM_DCACHE_SYNC_DONE)); */
+    movi a5, EXTMEM_DCACHE_SYNC_DONE_M
+_wait_writeback_done:
+    l32i a4, a3, 0  /* a4 = *(EXTMEM_DCACHE_SYNC_CTRL_REG) */
+    memw
+    bnone a4, a5, _wait_writeback_done
+
+    /* REG_CLR_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_ENA); */
+    l32i a4, a2, 0  /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */
+    movi a5, ~(EXTMEM_DCACHE_FREEZE_ENA_M)
+    and a4, a4, a5
+    s32i a4, a2, 0  /* *(EXTMEM_DCACHE_FREEZE_REG) = a4 */
+
+    /* while (REG_GET_BIT(EXTMEM_DCACHE_FREEZE_REG, EXTMEM_DCACHE_FREEZE_DONE)); */
+    movi a5, EXTMEM_DCACHE_FREEZE_DONE_M
+_wait_unfreeze_done:
+    l32i a4, a2, 0  /* a4 = *(EXTMEM_DCACHE_FREEZE_REG) */
+    memw
+    bany a4, a5, _wait_unfreeze_done
+
+    retw
+    .size   cache_writeback_items_freeze, . - cache_writeback_items_freeze
+#endif /* CONFIG_IDF_TARGET_ESP32S3 */

components/esp_system/port/cpu_start.c

@@ -338,6 +338,10 @@ void IRAM_ATTR call_start_cpu0(void)
     extern void rom_config_data_cache_mode(uint32_t cfg_cache_size, uint8_t cfg_cache_ways, uint8_t cfg_cache_line_size);
     rom_config_data_cache_mode(CONFIG_ESP32S3_DATA_CACHE_SIZE, CONFIG_ESP32S3_DCACHE_ASSOCIATED_WAYS, CONFIG_ESP32S3_DATA_CACHE_LINE_SIZE);
     Cache_Resume_DCache(0);
+
+    /*add lock to protect cache operation*/
+    extern void esp_cache_op_lock_init(void);
+    esp_cache_op_lock_init();
 #endif // CONFIG_IDF_TARGET_ESP32S3
 
     if (esp_efuse_check_errors() != ESP_OK) {

components/spi_flash/cache_utils.c

@@ -941,3 +941,24 @@ void IRAM_ATTR spi_flash_enable_cache(uint32_t cpuid)
     spi_flash_restore_cache(0, 0); // TODO cache_value should be non-zero
 #endif
 }
+
+#if CONFIG_IDF_TARGET_ESP32S3
+/*protect cache opreation*/
+static spinlock_t cache_op_lock = SPINLOCK_INITIALIZER;
+
+IRAM_ATTR void esp_cache_op_lock(void)
+{
+    portENTER_CRITICAL_SAFE(&cache_op_lock);
+}
+
+IRAM_ATTR void esp_cache_op_unlock(void)
+{
+    portEXIT_CRITICAL_SAFE(&cache_op_lock);
+}
+
+IRAM_ATTR void esp_cache_op_lock_init(void)
+{
+    rom_cache_op_cb.start = esp_cache_op_lock;
+    rom_cache_op_cb.end = esp_cache_op_unlock;
+}
+#endif// CONFIG_IDF_TARGET_ESP32S3