Skip to content

Commit

Permalink
Merge pull request #13762 from sharifelgamal/iso-arm64
Browse files Browse the repository at this point in the history
Add support for building aarch64 ISO
  • Loading branch information
sharifelgamal authored May 9, 2022
2 parents f51fc4c + 9c75a9c commit acb03c4
Show file tree
Hide file tree
Showing 205 changed files with 6,147 additions and 120 deletions.
9 changes: 7 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,19 @@ _testmain.go
*.test
*.prof

/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/bin/auto-pause
/deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/usr/bin/auto-pause
/deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/usr/bin/auto-pause
/deploy/iso/minikube-iso/Config.in
/deploy/kicbase/auto-pause
/deploy/addons/auto-pause/auto-pause-hook
/out
/_gopath

#iso version file
deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/VERSION
deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/etc/VERSION
deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/etc/VERSION
deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/etc/CHANGELOG
deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/etc/CHANGELOG

/minikube

Expand Down
53 changes: 31 additions & 22 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co
KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)

# Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
ISO_VERSION ?= v1.25.2-1650391217-13814
ISO_VERSION ?= v1.25.2-1651892774-13762
# Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
DEB_REVISION ?= 0
Expand Down Expand Up @@ -93,7 +93,7 @@ GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH)
GOARM ?= 7 # the default is 5
GOPATH ?= $(shell go env GOPATH)
BUILD_DIR ?= ./out
BUILD_DIR ?= $(PWD)/out
$(shell mkdir -p $(BUILD_DIR))
CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)

Expand Down Expand Up @@ -282,17 +282,27 @@ out/e2e-%: out/minikube-%
out/e2e-windows-amd64.exe: out/e2e-windows-amd64
cp $< $@

minikube_iso: deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/bin/auto-pause # build minikube iso
echo $(ISO_VERSION) > deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/VERSION
minikube-iso-amd64: minikube-iso-x86_64
minikube-iso-arm64: minikube-iso-aarch64

minikube-iso-%: deploy/iso/minikube-iso/board/minikube/%/rootfs-overlay/usr/bin/auto-pause # build minikube iso
echo $(ISO_VERSION) > deploy/iso/minikube-iso/board/minikube/$*/rootfs-overlay/etc/VERSION
cp deploy/iso/minikube-iso/CHANGELOG deploy/iso/minikube-iso/board/minikube/$*/rootfs-overlay/etc/CHANGELOG
cp deploy/iso/minikube-iso/arch/$*/Config.in.tmpl deploy/iso/minikube-iso/Config.in
if [ ! -d $(BUILD_DIR)/buildroot ]; then \
mkdir -p $(BUILD_DIR); \
git clone --depth=1 --branch=$(BUILDROOT_BRANCH) https://github.com/buildroot/buildroot $(BUILD_DIR)/buildroot; \
cp deploy/iso/minikube-iso/go.hash $(BUILD_DIR)/buildroot/package/go/go.hash; \
fi;
$(MAKE) BR2_EXTERNAL=../../deploy/iso/minikube-iso minikube_defconfig -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS)
$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) host-python
$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS)
mv $(BUILD_DIR)/buildroot/output/images/rootfs.iso9660 $(BUILD_DIR)/minikube.iso
$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) O=$(BUILD_DIR)/buildroot/output-$* minikube_$*_defconfig
$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) O=$(BUILD_DIR)/buildroot/output-$* host-python
$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) O=$(BUILD_DIR)/buildroot/output-$*
# x86_64 ISO is still BIOS rather than EFI because of AppArmor issues for KVM, and Gen 2 issues for Hyper-V
if [ "$*" = "aarch64" ]; then \
mv $(BUILD_DIR)/buildroot/output-aarch64/images/boot.iso $(BUILD_DIR)/minikube-arm64.iso; \
else \
mv $(BUILD_DIR)/buildroot/output-x86_64/images/rootfs.iso9660 $(BUILD_DIR)/minikube-amd64.iso; \
fi;

# Change buildroot configuration for the minikube ISO
.PHONY: iso-menuconfig
Expand All @@ -301,15 +311,14 @@ iso-menuconfig: ## Configure buildroot configuration
$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) savedefconfig

# Change the kernel configuration for the minikube ISO
.PHONY: linux-menuconfig
linux-menuconfig: ## Configure Linux kernel configuration
linux-menuconfig-%: ## Configure Linux kernel configuration
$(MAKE) -C $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/ menuconfig
$(MAKE) -C $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/ savedefconfig
cp $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/defconfig deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
cp $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/defconfig deploy/iso/minikube-iso/board/minikube/$*/linux_$*_defconfig

out/minikube.iso: $(shell find "deploy/iso/minikube-iso" -type f)
out/minikube-%.iso: $(shell find "deploy/iso/minikube-iso" -type f)
ifeq ($(IN_DOCKER),1)
$(MAKE) minikube_iso
$(MAKE) minikube-iso-$*
else
docker run --rm --workdir /mnt --volume $(CURDIR):/mnt $(ISO_DOCKER_EXTRA_ARGS) \
--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
Expand All @@ -321,9 +330,6 @@ iso_in_docker:
--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
$(ISO_BUILD_IMAGE) /bin/bash

test-iso:
go test -v $(INTEGRATION_TESTS_TO_RUN) --tags=iso --minikube-start-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"

.PHONY: test-pkg
test-pkg/%: ## Trigger packaging test
go test -v -test.timeout=60m ./$* --tags="$(MINIKUBE_BUILD_TAGS)"
Expand Down Expand Up @@ -436,7 +442,7 @@ e2e-cross: e2e-linux-amd64 e2e-linux-arm64 e2e-darwin-amd64 e2e-windows-amd64.ex

.PHONY: checksum
checksum: ## Generate checksums
for f in out/minikube.iso out/minikube-linux-amd64 out/minikube-linux-arm \
for f in out/minikube-amd64.iso out/minikube-arm64.iso out/minikube-linux-amd64 out/minikube-linux-arm \
out/minikube-linux-arm64 out/minikube-linux-ppc64le out/minikube-linux-s390x \
out/minikube-darwin-amd64 out/minikube-darwin-arm64 out/minikube-windows-amd64.exe \
out/docker-machine-driver-kvm2 out/docker-machine-driver-kvm2-amd64 out/docker-machine-driver-kvm2-arm64 \
Expand Down Expand Up @@ -776,9 +782,11 @@ push-gvisor-addon-image: gvisor-addon-image
$(MAKE) push-docker IMAGE=$(REGISTRY)/gvisor-addon:$(GVISOR_TAG)

.PHONY: release-iso
release-iso: minikube_iso checksum ## Build and release .iso file
gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso
gsutil cp out/minikube.iso.sha256 gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso.sha256
release-iso: minikube-iso-aarch64 minikube-iso-x86_64 checksum ## Build and release .iso files
gsutil cp out/minikube-amd64.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-amd64.iso
gsutil cp out/minikube-amd64.iso.sha256 gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-amd64.iso.sha256
gsutil cp out/minikube-arm64.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-arm64.iso
gsutil cp out/minikube-arm64.iso.sha256 gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-arm64.iso.sha256

.PHONY: release-minikube
release-minikube: out/minikube checksum ## Minikube release
Expand Down Expand Up @@ -918,8 +926,9 @@ out/mkcmp:


# auto pause binary to be used for ISO
deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/bin/auto-pause: $(SOURCE_FILES) $(ASSET_FILES)
GOOS=linux GOARCH=$(GOARCH) go build -o $@ cmd/auto-pause/auto-pause.go
deploy/iso/minikube-iso/board/minikube/%/rootfs-overlay/usr/bin/auto-pause: $(SOURCE_FILES) $(ASSET_FILES)
@if [ "$*" != "x86_64" ] && [ "$*" != "aarch64" ]; then echo "Please enter a valid architecture. Choices are x86_64 and aarch64."; exit 1; fi
GOOS=linux GOARCH=$(subst x86_64,amd64,$(subst aarch64,arm64,$*)) go build -o $@ cmd/auto-pause/auto-pause.go


.PHONY: deploy/addons/auto-pause/auto-pause-hook
Expand Down
1 change: 0 additions & 1 deletion deploy/iso/minikube-iso/Config.in

This file was deleted.

2 changes: 2 additions & 0 deletions deploy/iso/minikube-iso/arch/aarch64/Config.in.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
source "$BR2_EXTERNAL_MINIKUBE_PATH/package/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/Config.in"
12 changes: 12 additions & 0 deletions deploy/iso/minikube-iso/arch/aarch64/package/Config.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
menu "System tools aarch64"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/buildkit-bin-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/runc-master-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/crio-bin-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/crictl-bin-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/docker-bin-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/cri-dockerd-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/cni-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/cni-plugins-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/containerd-bin-aarch64/Config.in"
source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/podman-aarch64/Config.in"
endmenu
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
config BR2_PACKAGE_BUILDKIT_BIN_AARCH64
bool "buildkit-bin"
default y
depends on BR2_aarch64
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
sha256 0a834a749e86525e0eb16e7e80ce726e7c6a8e8d26fc5cc36c712305024da4e7 buildkit-v0.9.0.linux-arm64.tar.gz
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
################################################################################
#
# buildkit-bin
#
################################################################################

BUILDKIT_BIN_AARCH64_VERSION = v0.9.0
BUILDKIT_BIN_AARCH64_COMMIT = c8bb937807d405d92be91f06ce2629e6202ac7a9
BUILDKIT_BIN_AARCH64_SITE = https://github.com/moby/buildkit/releases/download/$(BUILDKIT_BIN_AARCH64_VERSION)
BUILDKIT_BIN_AARCH64_SOURCE = buildkit-$(BUILDKIT_BIN_AARCH64_VERSION).linux-arm64.tar.gz

# https://github.com/opencontainers/runc.git
BUILDKIT_RUNC_VERSION = 12644e614e25b05da6fd08a38ffa0cfe1903fdec

define BUILDKIT_BIN_AARCH64_USERS
- -1 buildkit -1 - - - - -
endef

define BUILDKIT_BIN_AARCH64_INSTALL_TARGET_CMDS
$(INSTALL) -D -m 0755 \
$(@D)/buildctl \
$(TARGET_DIR)/usr/bin
$(INSTALL) -D -m 0755 \
$(@D)/buildkit-runc \
$(TARGET_DIR)/usr/sbin
$(INSTALL) -D -m 0755 \
$(@D)/buildkit-qemu-* \
$(TARGET_DIR)/usr/sbin
$(INSTALL) -D -m 0755 \
$(@D)/buildkitd \
$(TARGET_DIR)/usr/sbin
$(INSTALL) -D -m 644 \
$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkit.conf \
$(TARGET_DIR)/usr/lib/tmpfiles.d/buildkit.conf
$(INSTALL) -D -m 644 \
$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkitd.toml \
$(TARGET_DIR)/etc/buildkit/buildkitd.toml
endef

define BUILDKIT_BIN_AARCH64_INSTALL_INIT_SYSTEMD
$(INSTALL) -D -m 644 \
$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkit.service \
$(TARGET_DIR)/usr/lib/systemd/system/buildkit.service
$(INSTALL) -D -m 644 \
$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkit.socket \
$(TARGET_DIR)/usr/lib/systemd/system/buildkit.socket
$(INSTALL) -D -m 644 \
$(BUILDKIT_BIN_AARCH64_PKGDIR)/51-buildkit.preset \
$(TARGET_DIR)/usr/lib/systemd/system-preset/51-buildkit.preset
endef

$(eval $(generic-package))
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
config BR2_PACKAGE_CNI_AARCH64
bool "cni"
default y
depends on BR2_aarch64
depends on BR2_PACKAGE_HOST_GO_TARGET_ARCH_SUPPORTS
45 changes: 45 additions & 0 deletions deploy/iso/minikube-iso/arch/aarch64/package/cni-aarch64/cni.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
################################################################################
#
# cni
#
################################################################################

CNI_AARCH64_VERSION = v0.7.1
CNI_AARCH64_SITE = https://github.com/containernetworking/cni/archive
CNI_AARCH64_SOURCE = $(CNI_AARCH64_VERSION).tar.gz
CNI_AARCH64_LICENSE = Apache-2.0
CNI_AARCH64_LICENSE_FILES = LICENSE

CNI_AARCH64_DEPENDENCIES = host-go

CNI_AARCH64_GOPATH = $(@D)/_output
CNI_AARCH64_MAKE_ENV = \
$(GO_TARGET_ENV) \
CGO_ENABLED=0 \
GO111MODULE=off \
GOPATH="$(CNI_AARCH64_GOPATH)" \
PATH=$(CNI_AARCH64_GOPATH)/bin:$(BR_PATH) \
GOARCH=arm64

CNI_AARCH64_BUILDFLAGS = -a --ldflags '-extldflags \"-static\"'

define CNI_AARCH64_CONFIGURE_CMDS
mkdir -p $(CNI_AARCH64_GOPATH)/src/github.com/containernetworking
ln -sf $(@D) $(CNI_AARCH64_GOPATH)/src/github.com/containernetworking/cni
endef

define CNI_AARCH64_BUILD_CMDS
(cd $(@D); $(CNI_AARCH64_MAKE_ENV) go build -o bin/cnitool $(CNI_AARCH64_BUILDFLAGS) ./cnitool)
endef

define CNI_AARCH64_INSTALL_TARGET_CMDS
$(INSTALL) -D -m 0755 \
$(@D)/bin/cnitool \
$(TARGET_DIR)/opt/cni/bin/cnitool

ln -sf \
../../opt/cni/bin/cnitool \
$(TARGET_DIR)/usr/bin/cnitool
endef

$(eval $(generic-package))
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
config BR2_PACKAGE_CNI_PLUGINS_AARCH64
bool "cni-plugins"
default y
depends on BR2_aarch64
depends on BR2_PACKAGE_HOST_GO_TARGET_ARCH_SUPPORTS
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
sha256 a7881ec37e592c897bdfd2a225b4ed74caa981e3c4cdcf8f45574f8d2f111bce cni-plugins-linux-arm64-v0.8.5.tgz
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
################################################################################
#
# cni-plugins
#
################################################################################

CNI_PLUGINS_AARCH64_VERSION = v0.8.5
CNI_PLUGINS_AARCH64_SITE = https://github.com/containernetworking/plugins/releases/download/$(CNI_PLUGINS_AARCH64_VERSION)
CNI_PLUGINS_AARCH64_SOURCE = cni-plugins-linux-arm64-$(CNI_PLUGINS_AARCH64_VERSION).tgz
CNI_PLUGINS_AARCH64_LICENSE = Apache-2.0
CNI_PLUGINS_AARCH64_LICENSE_FILES = LICENSE

define CNI_PLUGINS_AARCH64_INSTALL_TARGET_CMDS
$(INSTALL) -D -m 0755 \
$(@D)/bandwidth \
$(TARGET_DIR)/opt/cni/bin/bandwidth

ln -sf \
../../opt/cni/bin/bandwidth \
$(TARGET_DIR)/usr/bin/bandwidth

$(INSTALL) -D -m 0755 \
$(@D)/bridge \
$(TARGET_DIR)/opt/cni/bin/bridge

ln -sf \
../../opt/cni/bin/bridge \
$(TARGET_DIR)/usr/bin/bridge

$(INSTALL) -D -m 0755 \
$(@D)/vlan \
$(TARGET_DIR)/opt/cni/bin/vlan

ln -sf \
../../opt/cni/bin/vlan \
$(TARGET_DIR)/usr/bin/vlan

$(INSTALL) -D -m 0755 \
$(@D)/tuning \
$(TARGET_DIR)/opt/cni/bin/tuning

ln -sf \
../../opt/cni/bin/tuning \
$(TARGET_DIR)/usr/bin/tuning

$(INSTALL) -D -m 0755 \
$(@D)/ptp \
$(TARGET_DIR)/opt/cni/bin/ptp

ln -sf \
../../opt/cni/bin/ptp \
$(TARGET_DIR)/usr/bin/ptp

$(INSTALL) -D -m 0755 \
$(@D)/portmap \
$(TARGET_DIR)/opt/cni/bin/portmap

ln -sf \
../../opt/cni/bin/portmap \
$(TARGET_DIR)/usr/bin/portmap

$(INSTALL) -D -m 0755 \
$(@D)/macvlan \
$(TARGET_DIR)/opt/cni/bin/macvlan

ln -sf \
../../opt/cni/bin/macvlan \
$(TARGET_DIR)/usr/bin/macvlan

$(INSTALL) -D -m 0755 \
$(@D)/loopback \
$(TARGET_DIR)/opt/cni/bin/loopback

ln -sf \
../../opt/cni/bin/loopback \
$(TARGET_DIR)/usr/bin/loopback

$(INSTALL) -D -m 0755 \
$(@D)/ipvlan \
$(TARGET_DIR)/opt/cni/bin/ipvlan

ln -sf \
../../opt/cni/bin/ipvlan \
$(TARGET_DIR)/usr/bin/ipvlan

$(INSTALL) -D -m 0755 \
$(@D)/host-local \
$(TARGET_DIR)/opt/cni/bin/host-local

ln -sf \
../../opt/cni/bin/host-local \
$(TARGET_DIR)/usr/bin/host-local

$(INSTALL) -D -m 0755 \
$(@D)/flannel \
$(TARGET_DIR)/opt/cni/bin/flannel

ln -sf \
../../opt/cni/bin/flannel \
$(TARGET_DIR)/usr/bin/flannel


$(INSTALL) -D -m 0755 \
$(@D)/dhcp \
$(TARGET_DIR)/opt/cni/bin/dhcp

ln -sf \
../../opt/cni/bin/dhcp \
$(TARGET_DIR)/usr/bin/dhcp

$(INSTALL) -D -m 0755 \
$(@D)/firewall \
$(TARGET_DIR)/opt/cni/bin/firewall

ln -sf \
../../opt/cni/bin/firewall \
$(TARGET_DIR)/usr/bin/firewall
endef

$(eval $(generic-package))
Loading

0 comments on commit acb03c4

Please sign in to comment.