From b943612156f8e90535b6bbc0e7ae152d2f7d12c0 Mon Sep 17 00:00:00 2001 From: Kalimuthu Velappan Date: Mon, 5 Aug 2019 08:04:58 -0700 Subject: [PATCH] Support for DPKG local caching --- Makefile.work | 10 ++++ rules/config | 12 +++++ rules/linux-kernel.mk | 11 ++++ slave.mk | 114 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 145 insertions(+), 2 deletions(-) diff --git a/Makefile.work b/Makefile.work index 389f3f7c831f..b88499dda75d 100644 --- a/Makefile.work +++ b/Makefile.work @@ -30,6 +30,8 @@ # * Default: yes # * Values: yes, no # * KERNEL_PROCURE_METHOD: Specifying method of obtaining kernel Debian package: download or build +# * SONIC_DPKG_CACHE_METHOD: Specifying method of obtaining the Debian packages from cache: none or cache +# * SONIC_DPKG_CACHE_SOURCE: Debian package cache location when cache enabled for debian packages # ############################################################################### @@ -137,6 +139,12 @@ ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64)) endif +ifneq ($(SONIC_DPKG_CACHE_SOURCE),) + $(shell test -d $(SONIC_DPKG_CACHE_SOURCE) || mkdir -p $(SONIC_DPKG_CACHE_SOURCE) ) + DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache" +endif + + DOCKER_BASE_BUILD = docker build --no-cache \ -t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \ --build-arg http_proxy=$(http_proxy) \ @@ -171,6 +179,8 @@ SONIC_BUILD_INSTRUCTION := make \ SONIC_USE_DOCKER_BUILDKIT=$(SONIC_USE_DOCKER_BUILDKIT) \ VS_PREPARE_MEM=$(VS_PREPARE_MEM) \ KERNEL_PROCURE_METHOD=$(KERNEL_PROCURE_METHOD) \ + SONIC_DPKG_CACHE_METHOD=$(SONIC_DPKG_CACHE_METHOD) \ + SONIC_DPKG_CACHE_SOURCE=$(SONIC_DPKG_CACHE_SOURCE) \ HTTP_PROXY=$(http_proxy) \ HTTPS_PROXY=$(https_proxy) \ SONIC_ENABLE_SYSTEM_TELEMETRY=$(ENABLE_SYSTEM_TELEMETRY) \ diff --git a/rules/config b/rules/config index 31e5d40ff1f9..f30e54c53a02 100644 --- a/rules/config +++ b/rules/config @@ -93,5 +93,17 @@ DEFAULT_KERNEL_PROCURE_METHOD = build FRR_USER_UID = 300 FRR_USER_GID = 300 +# DPKG cache allows the .deb files to be stored in the cache path. This allows the submodules +# package to be cached and restored back if its commit hash is not modified and its dependencies are not modified. +# SONIC_DPKG_CACHE_METHOD - Default method of deb package caching +# none: no caching +# cache: cache from local directory +# #url: cache over remote http server (not supported) +# #registry: cache on the docker local/remote registry (not supported) +# SONIC_DPKG_CACHE_SOURCE - provides the cache location detail +SONIC_DPKG_CACHE_METHOD=cache +SONIC_DPKG_CACHE_SOURCE=${PWD}/target/cache/ + + # Default VS build memory preparation DEFAULT_VS_PREPARE_MEM = yes diff --git a/rules/linux-kernel.mk b/rules/linux-kernel.mk index fd8429eb8f3b..7e5b6ae120a6 100644 --- a/rules/linux-kernel.mk +++ b/rules/linux-kernel.mk @@ -7,8 +7,19 @@ KERNEL_SUBVERSION = 1+deb9u3 export KVERSION_SHORT KVERSION KERNEL_VERSION KERNEL_SUBVERSION + +SMPATH=$(SRC_PATH)/sonic-linux-kernel/ +SM_DEP_LIST := Makefile +SM_DEP_LIST += patch/* +SM_DEP_LIST += patch/preconfig/* +SMDEP_LIST := $(wildcard $(addprefix $(SMPATH),$(SM_DEP_LIST))) +DEP_LIST := $(SONIC_MAKEFILE_LIST) rules/linux-kernel.mk + LINUX_HEADERS_COMMON = linux-headers-$(KVERSION_SHORT)-common_$(KERNEL_VERSION)-$(KERNEL_SUBVERSION)_all.deb $(LINUX_HEADERS_COMMON)_SRC_PATH = $(SRC_PATH)/sonic-linux-kernel +$(LINUX_HEADERS_COMMON)_CACHE_MODE = GIT_COMMIT_SHA +$(LINUX_HEADERS_COMMON)_DEP_SOURCE = $(DEP_LIST) +$(LINUX_HEADERS_COMMON)_SMDEP_SOURCE = $(SMDEP_LIST) SONIC_MAKE_DEBS += $(LINUX_HEADERS_COMMON) LINUX_HEADERS = linux-headers-$(KVERSION)_$(KERNEL_VERSION)-$(KERNEL_SUBVERSION)_$(CONFIGURED_ARCH).deb diff --git a/slave.mk b/slave.mk index a80c1125682c..486156701e01 100644 --- a/slave.mk +++ b/slave.mk @@ -48,6 +48,7 @@ export BUILD_NUMBER export BUILD_TIMESTAMP export CONFIGURED_PLATFORM export CONFIGURED_ARCH +SONIC_MAKEFILE_LIST=slave.mk rules/config rules/functions ############################################################################### ## Utility rules @@ -180,6 +181,9 @@ $(info "ENABLE_SYSTEM_TELEMETRY" : "$(ENABLE_SYSTEM_TELEMETRY)") $(info "SONIC_DEBUGGING_ON" : "$(SONIC_DEBUGGING_ON)") $(info "SONIC_PROFILING_ON" : "$(SONIC_PROFILING_ON)") $(info "KERNEL_PROCURE_METHOD" : "$(KERNEL_PROCURE_METHOD)") +ifeq ($(SONIC_DPKG_CACHE_METHOD),cache) +$(info "DPKG_CACHE_PATH" : "$(SONIC_DPKG_CACHE_SOURCE)") +endif $(info "BUILD_TIMESTAMP" : "$(BUILD_TIMESTAMP)") $(info "BLDENV" : "$(BLDENV)") $(info "VS_PREPARE_MEM" : "$(VS_PREPARE_MEM)") @@ -200,6 +204,85 @@ export kernel_procure_method=$(KERNEL_PROCURE_METHOD) export vs_build_prepare_mem=$(VS_PREPARE_MEM) ############################################################################### +MOD_CACHE_LOCK_TIMEOUT = 3600 +SONIC_DPKG_CACHE_DIR=/dpkg_cache + +# Lock macro for debian package level cache +# Lock is implemented through flock command with the timeout value of 1 hour +# Lock file is created in the cache directory and corresponding lock fd is stored as part of DPKG recipe. +define MOD_LOCK + if [[ ! -f $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock ]]; then + touch $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock + chmod 777 $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock; + fi + $(eval $(1)_lock_fd=$(subst -,_,$(subst +,_,$(subst .,_,$(1))))) + exec {$($(1)_lock_fd)}<"$(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock"; + if ! flock -x -w $(MOD_CACHE_LOCK_TIMEOUT) "$${$($(1)_lock_fd)}" ; then + echo "ERROR: Lock timeout trying to read $(1) from cache."; + exit 1; + fi +endef + + +# UnLock macro for debian package level cache +define MOD_UNLOCK + eval exec "$${$($(1)_lock_fd)}<&-"; +endef + + +# Loads the deb package from debian cache +# Cache file prefix is formed using SHA value +# The SHA value is derived from one of the keyword type - GIT_COMMIT_SHA or GIT_CONTENT_SHA +# GIT_COMMIT_SHA - SHA value of the last git commit id if it is a submodule +# GIT_CONTENT_SHA - SHA value is calculated from the target dependency files content. +# Cache is loaded only when corresponding cache file is present in the cache direcory and its dependencies are not changed. +define LOAD_CACHE + $(eval MOD_SRC_PATH=$($(1)_SRC_PATH)) + $(eval MOD_HASH=$(if $(filter GIT_COMMIT_SHA,$($(1)_CACHE_MODE)),$(shell cd $(MOD_SRC_PATH) && git log -1 --format="%H") + , $(shell git hash-object $($(1)_DEP_SOURCE) $($(1)_SMDEP_SOURCE)|sha1sum|awk '{print $$1}'))) + $(eval MOD_CACHE_FILE=$(1)-$(MOD_HASH).tgz) + $(eval $(1)_MOD_CACHE_FILE=$(MOD_CACHE_FILE)) + $(eval DRV_DEB=$(foreach pkg,$(addprefix $(DEBS_PATH)/,$(1) $($(1)_DERIVED_DEBS)),$(if $(wildcard $(pkg)),,$(pkg)))) + $(eval $(1)_FILES_MODIFIED := $(if $($(1)_DEP_SOURCE),$(shell git status -s $($(1)_DEP_SOURCE))) \ + $(if $($(1)_SMDEP_SOURCE),$(shell cd $(MOD_SRC_PATH) && git status -s $(subst $(MOD_SRC_PATH)/,,$($(1)_SMDEP_SOURCE)))) ) + #$(filter-out $($(1)_DEP_SOURCE),$($(1)_SMDEP_SOURCE), $?) + + $(if $($(1)_FILES_MODIFIED), + echo "Target $(1) dependencies are modifed - load cache skipped"; + echo "Modified dependencies are : [$($(1)_FILES_MODIFIED)] "; + , + $(if $(wildcard $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE)), + $(if $(DRV_DEB), tar -xzvf $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE),echo ); + echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) is loaded from cache"; + $(eval $(1)_CACHE_LOADED := Yes) + , + echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) is not present in the cache !"; + ) + ) + echo "" +endef + +# Saves the deb package into debian cache +# A single tared-zip cache is created for .deb and its derived packages in the cache direcory. +# It saves the .deb into cache only when its dependencies are not changed +# The cache save is protected with lock. +# The SAVE_CACHE macro has dependecy with LOAD_CACHE macro +# The target specific variables -_SRC_PATH, _MOD_CACHE_FILE and _FILES_MODIFIED are +# derived from the LOAD_CACHE macro +define SAVE_CACHE + $(eval MOD_SRC_PATH=$($(1)_SRC_PATH)) + $(eval MOD_CACHE_FILE=$($(1)_MOD_CACHE_FILE)) + $(call MOD_LOCK,$(1)) + $(if $($(1)_FILES_MODIFIED), + echo "Target $(1) dependencies are modifed - save cache skipped"; + , + tar -czvf $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) $(2) $(addprefix $(DEBS_PATH)/,$($(1)_DERIVED_DEBS)); + echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) saved in the cache "; + ) + $(call MOD_UNLOCK,$(1)) + echo "" +endef + ## Local targets ############################################################################### @@ -297,8 +380,16 @@ SONIC_TARGET_LIST += $(addprefix $(FILES_PATH)/, $(SONIC_MAKE_FILES)) # $(SOME_NEW_DEB)_SRC_PATH = $(SRC_PATH)/project_name # $(SOME_NEW_DEB)_DEPENDS = $(SOME_OTHER_DEB1) $(SOME_OTHER_DEB2) ... # SONIC_MAKE_DEBS += $(SOME_NEW_DEB) -$(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) +$(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \ + $$($$*_DEP_SOURCE) $$($$*_SMDEP_SOURCE) $(HEADER) + + # Load the target deb from DPKG cache + $(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call LOAD_CACHE,$*) ) + + # Skip building the target if it is already loaded from cache + if [ -z '$($*_CACHE_LOADED)' ] ; then + # Remove target to force rebuild rm -f $(addprefix $(DEBS_PATH)/, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS)) # Apply series of patches if exist @@ -307,6 +398,12 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG) # Clean up if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi + + # Save the target deb into DPKG cache + $(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call SAVE_CACHE,$*,$@)) + + fi + $(FOOTER) SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) @@ -317,8 +414,16 @@ SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) # $(SOME_NEW_DEB)_SRC_PATH = $(SRC_PATH)/project_name # $(SOME_NEW_DEB)_DEPENDS = $(SOME_OTHER_DEB1) $(SOME_OTHER_DEB2) ... # SONIC_DPKG_DEBS += $(SOME_NEW_DEB) -$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) +$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \ + $$($$*_DEP_SOURCE) $$($$*_SMDEP_SOURCE) $(HEADER) + + # Load the target deb from DPKG cache + $(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call LOAD_CACHE,$*) ) + + # Skip building the target if it is already loaded from cache + if [ -z '$($*_CACHE_LOADED)' ] ; then + # Remove old build logs if they exist rm -f $($*_SRC_PATH)/debian/*.debhelper.log # Apply series of patches if exist @@ -335,6 +440,11 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi # Take built package(s) mv $(addprefix $($*_SRC_PATH)/../, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS)) $(DEBS_PATH) $(LOG) + + # Save the target deb into DPKG cache + $(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call SAVE_CACHE,$*,$@)) + fi + $(FOOTER) SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS))