diff --git a/build_debian.sh b/build_debian.sh
index d1dc659cd88e..ac68c82109e3 100755
--- a/build_debian.sh
+++ b/build_debian.sh
@@ -447,6 +447,10 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'docke
 # Install scapy
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'scapy==2.4.4'
 
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip2 install 'PyYAML==5.4.1' --no-build-isolation
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install 'PyYAML==5.4.1' --no-build-isolation
+
 ## Note: keep pip installed for maintainance purpose
 
 # Install GCC, needed for building/installing some Python packages
diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
index 1e7a64bfcd36..ec85c215ad11 100644
--- a/dockers/docker-config-engine-buster/Dockerfile.j2
+++ b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -23,6 +23,10 @@ RUN apt-get install -y        \
 RUN pip3 install pyangbind==0.8.1
 RUN pip3 uninstall -y enum34
 
+# Fix armhf build failure
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip3 install PyYAML==5.4.1 --no-build-isolation
+
 {% if docker_config_engine_buster_debs.strip() %}
 # Copy locally-built Debian package dependencies
 {{ copy_files("debs/", docker_config_engine_buster_debs.split(' '), "/debs/") }}
diff --git a/dockers/docker-platform-monitor/Dockerfile.j2 b/dockers/docker-platform-monitor/Dockerfile.j2
index 2c144cfc55d3..38e8be5de7c4 100755
--- a/dockers/docker-platform-monitor/Dockerfile.j2
+++ b/dockers/docker-platform-monitor/Dockerfile.j2
@@ -53,6 +53,11 @@ RUN pip3 install thrift==0.13.0
 RUN pip2 install libpci
 RUN pip3 install libpci
 
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+# The the PyYAML for python3 has already installed in the base image docker-config-engine
+RUN pip2 install wheel==0.37.1
+RUN pip2 install PyYAML==5.4.1 --no-build-isolation
+
 {% if docker_platform_monitor_debs.strip() -%}
 # Copy locally-built Debian package dependencies
 {{ copy_files("debs/", docker_platform_monitor_debs.split(' '), "/debs/") }}
diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2
index eea23a2d4620..e88827cdb73a 100644
--- a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2
+++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2
@@ -15,6 +15,8 @@ RUN pip2 install --upgrade 'pip<21'
 RUN apt-get purge -y python-pip
 RUN pip2 install setuptools==40.8.0
 RUN pip2 install wheel==0.35.1
+RUN pip2 install PyYAML==5.4.1 --no-build-isolation
+
 RUN pip2 install connexion==1.1.15 \
                 setuptools==21.0.0 \
                 grpcio-tools==1.20.0 \
diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
index 669c77b1ab12..9e3fa0d3ad69 100644
--- a/sonic-slave-buster/Dockerfile.j2
+++ b/sonic-slave-buster/Dockerfile.j2
@@ -404,8 +404,9 @@ RUN pip3 uninstall -y enum34
 RUN pip2 install j2cli==0.3.10
 
 # For sonic-mgmt-framework
-RUN pip2 install "PyYAML==5.4.1"
-RUN pip3 install "PyYAML==5.4.1"
+# The option --no-build-isolation can be removed when upgrading PyYAML to 6.0.1
+RUN pip2 install "PyYAML==5.4.1" --no-build-isolation
+RUN pip3 install "PyYAML==5.4.1" --no-build-isolation
 RUN pip2 install "lxml==4.6.5"
 RUN pip3 install "lxml==4.6.5"