diff --git a/build-coredistools.cmd b/build-coredistools.cmd index 613ba6d4..04da1b47 100644 --- a/build-coredistools.cmd +++ b/build-coredistools.cmd @@ -16,6 +16,19 @@ if /i "%TargetOSArchitecture%" == "win-arm64" ( set LLVMTargetsToBuild=ARM;X86 ) else ( echo ERROR: Unknown target OS and architecture: %TargetOSArchitecture% + echo Use one of win-arm64, win-x64, win-x86. + exit /b 1 +) + +set BuildFlavor=%2 +if "%BuildFlavor%"=="" set BuildFlavor=Release + +if /i "%BuildFlavor%" == "Release" ( + @REM ok +) else if /i "%BuildFlavor%" == "Debug" ( + @REM ok +) else ( + echo ERROR: Unknown build flavor: %BuildFlavor% exit /b 1 ) @@ -37,11 +50,19 @@ if %ERRORLEVEL% neq 0 ( where /q llvm-tblgen.exe -if %ERRORLEVEL% neq 0 ( +if %ERRORLEVEL% equ 0 goto found_llvm_tblgen + +@REM We expect it to be in the `bin` directory, so add that to the PATH if it's there. +if not exist %RootDirectory%bin\llvm-tblgen.exe ( echo ERROR: llvm-tblgen.exe is not found in the PATH exit /b 1 ) +echo Found llvm-tblgen.exe in %RootDirectory%bin; adding that directory to PATH. +set PATH=%RootDirectory%bin;%PATH% + +:found_llvm_tblgen + for /f %%I in ('where llvm-tblgen.exe') do ( set LLVMTableGen=%%~I ) @@ -51,11 +72,22 @@ if not exist "%BinariesDirectory%" ( ) pushd "%BinariesDirectory%" + +@REM To use the Debug CRT, use: +@REM -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreadedDebug +@REM +@REM To build a Debug version (asserts, debug info): +@REM -DCMAKE_BUILD_TYPE=Debug +@REM To build a Release version (no asserts, no debug info): +@REM -DCMAKE_BUILD_TYPE=Release +@REM +@REM Misc. LLVM CMake documentation: https://llvm.org/docs/CMake.html cmake.exe ^ -G "Visual Studio 17 2022" ^ -A %GeneratorPlatform% ^ -DCMAKE_INSTALL_PREFIX="%StagingDirectory%" ^ + -DCMAKE_BUILD_TYPE=%BuildFlavor% ^ -DLLVM_DEFAULT_TARGET_TRIPLE=%LLVMDefaultTargetTriple% ^ -DLLVM_EXTERNAL_PROJECTS=coredistools ^ -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR="%SourcesDirectory%\coredistools" ^ @@ -64,18 +96,19 @@ cmake.exe ^ -DLLVM_TABLEGEN="%LLVMTableGen%" ^ -DLLVM_TARGETS_TO_BUILD=%LLVMTargetsToBuild% ^ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON ^ - -DLLVM_USE_CRT_DEBUG=MTd ^ - -DLLVM_USE_CRT_RELEASE=MT ^ + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ^ "%SourcesDirectory%\llvm-project\llvm" popd if %ERRORLEVEL% neq 0 goto :CMakeNonZeroExitStatus +@REM Use `--config Release` for release build, `--config Debug` for debug build + cmake.exe ^ --build "%BinariesDirectory%" ^ --target coredistools ^ - --config Release + --config %BuildFlavor% if %ERRORLEVEL% neq 0 goto :CMakeNonZeroExitStatus diff --git a/build-coredistools.sh b/build-coredistools.sh index da44cad0..9dca3afc 100755 --- a/build-coredistools.sh +++ b/build-coredistools.sh @@ -3,6 +3,9 @@ TargetOSArchitecture=$1 CrossRootfsDirectory=$2 +# Set this to 1 to build using CBL-Mariner +CrossBuildUsingMariner=1 + EnsureCrossRootfsDirectoryExists () { if [ ! -d "$CrossRootfsDirectory" ]; then echo "Invalid or unspecified CrossRootfsDirectory: $CrossRootfsDirectory" @@ -13,12 +16,44 @@ EnsureCrossRootfsDirectoryExists () { CMakeOSXArchitectures= LLVMTargetsToBuild="AArch64;ARM;X86" +# Figure out which `strip` to use. Prefer `llvm-strip` if it is available. +# `llvm-strip` is available in CBL-Mariner container; `strip` is available on macOS. +StripTool=$(command -v llvm-strip) +if [ -z "$StripTool" ]; then + StripTool=$(command -v strip) + if [ -z "$StripTool" ]; then + echo "Strip tool not found" + exit 1 + fi +fi + +TblGenTool=$(command -v llvm-tblgen) +if [ -z "$TblGenTool" ]; then + echo "llvm-tblgen tool not found" + exit 1 +fi + +C_COMPILER=$(command -v clang) +if [ -z "$C_COMPILER" ]; then + echo "C compiler not found" + # Keep going in case cmake can find one? +fi + +CXX_COMPILER=$(command -v clang++) +if [ -z "$CXX_COMPILER" ]; then + echo "C++ compiler not found" + # Keep going in case cmake can find one? +fi + +echo "Using C compiler: $C_COMPILER" +echo "Using C++ compiler: $CXX_COMPILER" + case "$TargetOSArchitecture" in linux-arm) CMakeCrossCompiling=ON LLVMDefaultTargetTriple=thumbv7-linux-gnueabihf LLVMHostTriple=arm-linux-gnueabihf - LLVMTargetsToBuild=ARM + LLVMTargetsToBuild="ARM" EnsureCrossRootfsDirectoryExists ;; @@ -29,8 +64,13 @@ case "$TargetOSArchitecture" in ;; linux-x64) - CMakeCrossCompiling=OFF LLVMHostTriple=x86_64-linux-gnu + if [ $CrossBuildUsingMariner -eq 1 ]; then + CMakeCrossCompiling=ON + EnsureCrossRootfsDirectoryExists + else + CMakeCrossCompiling=OFF + fi ;; linux-loongarch64) @@ -63,7 +103,7 @@ SourcesDirectory=$RootDirectory/src BinariesDirectory=$RootDirectory/obj/$TargetOSArchitecture StagingDirectory=$RootDirectory/artifacts/$TargetOSArchitecture -which cmake >/dev/null 2>&1 +command -v cmake >/dev/null 2>&1 if [ "$?" -ne 0 ]; then echo "ERROR: cmake is not found in the PATH" @@ -77,47 +117,76 @@ fi pushd "$BinariesDirectory" if [ -z "$CrossRootfsDirectory" ]; then + BUILD_FLAGS="-target $LLVMHostTriple" cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ - -DCMAKE_C_COMPILER=$(which clang) \ - -DCMAKE_C_FLAGS="-target $LLVMHostTriple" \ - -DCMAKE_CXX_COMPILER=$(which clang++) \ - -DCMAKE_CXX_FLAGS="-target $LLVMHostTriple" \ + -DCMAKE_C_COMPILER=${C_COMPILER} \ + -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \ - -DCMAKE_STRIP=$(which strip) \ + -DCMAKE_STRIP=$StripTool \ + -DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \ + -DLLVM_ENABLE_TERMINFO=OFF \ + -DLLVM_EXTERNAL_PROJECTS=coredistools \ + -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \ + -DLLVM_HOST_TRIPLE=$LLVMHostTriple \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_TABLEGEN=$TblGenTool \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ + $SourcesDirectory/llvm-project/llvm +elif [ $CrossBuildUsingMariner -eq 1 ]; then + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" + # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=${C_COMPILER} \ + -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ + -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \ + -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ + -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ + -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ + -DCMAKE_STRIP=$StripTool \ -DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \ -DLLVM_ENABLE_TERMINFO=OFF \ -DLLVM_EXTERNAL_PROJECTS=coredistools \ -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \ -DLLVM_HOST_TRIPLE=$LLVMHostTriple \ -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_TABLEGEN=$(which llvm-tblgen) \ + -DLLVM_TABLEGEN=$TblGenTool \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm else + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory -target $LLVMHostTriple" cmake \ -G "Unix Makefiles" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ - -DCMAKE_C_COMPILER=$(which clang) \ - -DCMAKE_C_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \ - -DCMAKE_CXX_COMPILER=$(which clang++) \ - -DCMAKE_CXX_FLAGS="-target $LLVMHostTriple --sysroot=$CrossRootfsDirectory" \ + -DCMAKE_C_COMPILER=${C_COMPILER} \ + -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ -DCMAKE_INSTALL_PREFIX=$StagingDirectory \ -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ - -DCMAKE_STRIP=/usr/$LLVMHostTriple/bin/strip \ + -DCMAKE_STRIP=$StripTool \ -DLLVM_DEFAULT_TARGET_TRIPLE=$LLVMDefaultTargetTriple \ -DLLVM_ENABLE_TERMINFO=OFF \ -DLLVM_EXTERNAL_PROJECTS=coredistools \ -DLLVM_EXTERNAL_COREDISTOOLS_SOURCE_DIR=$SourcesDirectory/coredistools \ -DLLVM_HOST_TRIPLE=$LLVMHostTriple \ -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_TABLEGEN=$(which llvm-tblgen) \ + -DLLVM_TABLEGEN=$TblGenTool \ -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ -DLLVM_TOOL_COREDISTOOLS_BUILD=ON \ $SourcesDirectory/llvm-project/llvm diff --git a/build-tblgen.sh b/build-tblgen.sh new file mode 100755 index 00000000..c616649a --- /dev/null +++ b/build-tblgen.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# Build the Linux/Mac llvm-tblgen tool. Note that this will be run during the +# Linux/Mac coredistools build. So, we only build the versions that we'll use +# during those builds. Thus, we need a linux-x64 version (used for +# linux-x64, linux-arm, linux-arm64 builds) and osx-x64 version (used for +# osx-x64 and osx-arm64 builds). +# +# The linux-x64 build is itself a cross-build when using CBL-Mariner container to build. + +TargetOSArchitecture=$1 +CrossRootfsDirectory=$2 + +# Set this to 1 to build using CBL-Mariner +CrossBuildUsingMariner=1 + +EnsureCrossRootfsDirectoryExists () { + if [ ! -d "$CrossRootfsDirectory" ]; then + echo "Invalid or unspecified CrossRootfsDirectory: $CrossRootfsDirectory" + exit 1 + fi +} + +CMakeOSXArchitectures= +LLVMTargetsToBuild="AArch64;ARM;X86" + +case "$TargetOSArchitecture" in + linux-x64) + LLVMHostTriple=x86_64-linux-gnu + if [ $CrossBuildUsingMariner -eq 1 ]; then + CMakeCrossCompiling=ON + EnsureCrossRootfsDirectoryExists + else + CMakeCrossCompiling=OFF + fi + ;; + + linux-loongarch64) + CMakeCrossCompiling=OFF + LLVMHostTriple=loongarch64-linux-gnu + LLVMTargetsToBuild="LoongArch" + ;; + + osx-x64) + CMakeCrossCompiling=OFF + CMakeOSXArchitectures=x86_64 + LLVMHostTriple=x86_64-apple-darwin + ;; + + *) + echo "Unknown target OS and architecture: $TargetOSArchitecture" + exit 1 +esac + +RootDirectory="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +SourcesDirectory=$RootDirectory/src +BinariesDirectory=$RootDirectory/obj +StagingDirectory=$RootDirectory/bin + +command -v cmake >/dev/null 2>&1 + +if [ "$?" -ne 0 ]; then + echo "ERROR: cmake is not found in the PATH" + exit 1 +fi + +if [ ! -d $BinariesDirectory ]; then + mkdir -p $BinariesDirectory +fi + +pushd "$BinariesDirectory" + +if [ -z "$CrossRootfsDirectory" ]; then + BUILD_FLAGS="" + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=$(command -v clang) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_INSTALL_PREFIX=$RootDirectory \ + -DCMAKE_OSX_ARCHITECTURES=$CMakeOSXArchitectures \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + $SourcesDirectory/llvm-project/llvm +elif [ $CrossBuildUsingMariner -eq 1 ]; then + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory" + # CBL-Mariner doesn't have `ld` so need to tell clang to use `lld` with "-fuse-ld=lld" + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$RootDirectory \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=$(command -v clang) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \ + -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \ + -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ + -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + $SourcesDirectory/llvm-project/llvm +else + BUILD_FLAGS="--sysroot=$CrossRootfsDirectory" + cmake \ + -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$RootDirectory \ + -DCMAKE_CROSSCOMPILING=$CMakeCrossCompiling \ + -DCMAKE_C_COMPILER=$(command -v clang) \ + -DCMAKE_CXX_COMPILER=$(command -v clang++) \ + -DCMAKE_C_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_CXX_FLAGS="${BUILD_FLAGS}" \ + -DCMAKE_INCLUDE_PATH=$CrossRootfsDirectory/usr/include \ + -DCMAKE_LIBRARY_PATH=$CrossRootfsDirectory/usr/lib/$LLVMHostTriple \ + -DLLVM_TARGETS_TO_BUILD=$LLVMTargetsToBuild \ + $SourcesDirectory/llvm-project/llvm +fi + +popd + +if [ "$?" -ne 0 ]; then + echo "ERROR: cmake exited with code $1" + exit 1 +fi + +cmake \ + --build $BinariesDirectory \ + --target llvm-tblgen \ + --config Release + +if [ "$?" -ne 0 ]; then + echo "ERROR: cmake exited with code $1" + exit 1 +fi + +if [ ! -d $StagingDirectory ]; then + mkdir -p $StagingDirectory +fi + +# Copy llvm-tblgen from BinariesDirectory to StagingDirectory +find $BinariesDirectory -name llvm-tblgen -type f -exec cp -v {} $StagingDirectory \; + +exit 0 diff --git a/coredistools.yml b/coredistools.yml index 776a6646..9e6e69b6 100644 --- a/coredistools.yml +++ b/coredistools.yml @@ -1,3 +1,5 @@ +# Changes to build with CBL-Mariner are marked "CBL-Mariner" + pr: branches: include: @@ -9,6 +11,7 @@ pr: - build-coredistools.cmd - build-coredistools.sh - build-tblgen.cmd + - build-tblgen.sh - coredistools.yml - pack-coredistools.cmd @@ -26,19 +29,39 @@ trigger: - coredistools.yml - pack-coredistools.cmd +# CBL-Mariner: resources: + # The base CBL-Mariner tags used to build .NET are: + # mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 + # mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + # mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 + # However, these are based on Ubuntu 16.04 libs and do not have libstdc++ 7. + # The following containers are based on a later set of libs. containers: - - container: ubuntu-18.04-arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 - - container: ubuntu-18.04-arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 + - container: linux_x64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-amd64 + - container: linux_arm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm + - container: linux_arm64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm64 + +# # Use the following for legacy Ubuntu 18.04 based Arm/Arm64 cross build (x64 build is not cross) +# resources: +# containers: +# - container: linux_arm +# image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 +# - container: linux_arm64 +# image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 variables: LLVMRepositoryUri: https://github.com/llvm/llvm-project.git LLVMSourceBundle: llvm-project.bundle - LLVMSourceVersion: llvmorg-13.0.1 + LLVMSourceVersion: llvmorg-17.0.6 jobs: + +################################ Check out LLVM source tree; publish to artifacts for future jobs + - job: checkout_llvm displayName: Checkout LLVM @@ -61,9 +84,56 @@ jobs: artifact: $(LLVMSourceBundle) displayName: Publish LLVM bundle -- job: crossbuild_coredistools_linux +################################ Build llvm-tblgen on Linux (x64) + +- job: build_tblgen_linux dependsOn: checkout_llvm + displayName: Build llvm-tblgen linux + + # CBL-Mariner: + container: linux_x64 + + pool: + # CBL-Mariner: + vmImage: ubuntu-latest + # non-CBL-Mariner: + # vmImage: ubuntu-20.04 + + workspace: + clean: all + + steps: + - template: /eng/download-checkout-llvm.yml + + # CBL-Mariner: + + - script: | + sudo tdnf install -y ncurses-compat + displayName: Install libtinfo.so.5 dependency of llvm-tblgen + + - script: | + ./build-tblgen.sh linux-x64 /crossrootfs/x64 + displayName: Build llvm-tblgen + + # non-CBL-Mariner: + #- script: ./build-tblgen.sh linux-x64 + # displayName: Build llvm-tblgen + + - publish: $(Build.SourcesDirectory)/bin/llvm-tblgen + artifact: tblgen-linux + displayName: Publish llvm-tblgen + +################################ Cross-build coredistools for linux-arm, linux-arm64 (and linux-x64 on CBL-Mariner) + +- job: crossbuild_coredistools_linux + timeoutInMinutes: 60 + dependsOn: + - checkout_llvm + - build_tblgen_linux + # CBL-Mariner: displayName: Build coredistools Linux + # non-CBL-Mariner: + # displayName: Build coredistools Linux container: $[ variables['ContainerImage'] ] @@ -72,13 +142,19 @@ jobs: strategy: matrix: + # CBL-Mariner (x64 section only): + x64: + ContainerImage: linux_x64 + CrossRootfsDirectory: /crossrootfs/x64 + TargetOSArchitecture: linux-x64 + arm: - ContainerImage: ubuntu-18.04-arm + ContainerImage: linux_arm CrossRootfsDirectory: /crossrootfs/arm TargetOSArchitecture: linux-arm arm64: - ContainerImage: ubuntu-18.04-arm64 + ContainerImage: linux_arm64 CrossRootfsDirectory: /crossrootfs/arm64 TargetOSArchitecture: linux-arm64 @@ -88,27 +164,79 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - template: /eng/download-llvm-release.yml - parameters: - os: linux - release: $(LLVMSourceVersion) + - download: current + artifact: tblgen-linux + displayName: Download llvm-tblgen + + - script: | + chmod +x $(Pipeline.Workspace)/tblgen-linux/llvm-tblgen + displayName: Make llvm-tblgen executable + + - script: | + echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" + displayName: Add llvm-tblgen to the PATH + + - script: | + sudo tdnf install -y ncurses-compat + displayName: Install libtinfo.so.5 dependency of llvm-tblgen - - script: ./build-coredistools.sh $(TargetOSArchitecture) $(CrossRootfsDirectory) + - script: | + ./build-coredistools.sh $(TargetOSArchitecture) $(CrossRootfsDirectory) displayName: Build coredistools - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools -- job: build_coredistools_linux_x64 +################################ Cross-build coredistools for linux-x64: only non-CBL-Mariner + +# - job: build_coredistools_linux +# timeoutInMinutes: 60 +# dependsOn: +# - checkout_llvm +# - build_tblgen_linux +# displayName: Build coredistools Linux x64 +# +# pool: +# vmImage: ubuntu-20.04 +# +# variables: +# TargetOSArchitecture: linux-x64 +# +# workspace: +# clean: all +# +# steps: +# - template: /eng/download-checkout-llvm.yml +# +# - download: current +# artifact: tblgen-linux +# displayName: Download llvm-tblgen +# +# - script: | +# chmod +x $(Pipeline.Workspace)/tblgen-linux/llvm-tblgen +# displayName: Make llvm-tblgen executable +# +# - script: | +# echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-linux" +# displayName: Add llvm-tblgen to the PATH +# +# - script: | +# ./build-coredistools.sh $(TargetOSArchitecture) +# displayName: Build coredistools +# +# - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so +# artifact: coredistools-$(TargetOSArchitecture) +# displayName: Publish coredistools + +################################ Build llvm-tblgen on Mac (x64) + +- job: build_tblgen_macos dependsOn: checkout_llvm - displayName: Build coredistools Linux x64 + displayName: Build llvm-tblgen macOS pool: - vmImage: ubuntu-18.04 - - variables: - TargetOSArchitecture: linux-x64 + vmImage: macOS-latest workspace: clean: all @@ -116,20 +244,20 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - template: /eng/download-llvm-release.yml - parameters: - os: linux - release: $(LLVMSourceVersion) + - script: ./build-tblgen.sh osx-x64 + displayName: Build llvm-tblgen - - script: ./build-coredistools.sh $(TargetOSArchitecture) - displayName: Build coredistools + - publish: $(Build.SourcesDirectory)/bin/llvm-tblgen + artifact: tblgen-macos + displayName: Publish llvm-tblgen - - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.so - artifact: coredistools-$(TargetOSArchitecture) - displayName: Publish coredistools +################################ Build coredistools for macos-x64, macos-arm64 - job: build_coredistools_macos - dependsOn: checkout_llvm + timeoutInMinutes: 60 + dependsOn: + - checkout_llvm + - build_tblgen_macos displayName: Build coredistools macOS pool: @@ -139,10 +267,10 @@ jobs: matrix: x64: TargetOSArchitecture: osx-x64 - VMImage: macOS-10.15 + VMImage: macOS-latest arm64: TargetOSArchitecture: osx-arm64 - VMImage: macOS-11 + VMImage: macOS-latest workspace: clean: all @@ -150,18 +278,28 @@ jobs: steps: - template: /eng/download-checkout-llvm.yml - - template: /eng/download-llvm-release.yml - parameters: - os: macos - release: $(LLVMSourceVersion) + - download: current + artifact: tblgen-macos + displayName: Download llvm-tblgen + + - script: | + chmod +x $(Pipeline.Workspace)/tblgen-macos/llvm-tblgen + displayName: Make llvm-tblgen executable + + - script: | + echo "##vso[task.prependpath]$(Pipeline.Workspace)/tblgen-macos" + displayName: Add llvm-tblgen to the PATH - - script: ./build-coredistools.sh $(TargetOSArchitecture) + - script: | + ./build-coredistools.sh $(TargetOSArchitecture) displayName: Build coredistools - publish: $(Build.SourcesDirectory)/artifacts/$(TargetOSArchitecture)/bin/libcoredistools.dylib artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools +################################ Build llvm-tblgen on Windows + - job: build_tblgen_windows dependsOn: checkout_llvm displayName: Build llvm-tblgen Windows @@ -182,7 +320,10 @@ jobs: artifact: tblgen-windows displayName: Publish llvm-tblgen +################################ Build coredistools for win-x64, win-x86, win-arm64 + - job: build_coredistools_windows + timeoutInMinutes: 60 dependsOn: - checkout_llvm - build_tblgen_windows @@ -220,10 +361,13 @@ jobs: artifact: coredistools-$(TargetOSArchitecture) displayName: Publish coredistools +################################ Build coredistools NuGet packages + - job: build_coredistools_nuget_packages dependsOn: - crossbuild_coredistools_linux - - build_coredistools_linux_x64 + # only non-CBL-Mariner: + # - build_coredistools_linux - build_coredistools_macos - build_coredistools_windows displayName: Build coredistools NuGet packages @@ -238,6 +382,10 @@ jobs: - download: current displayName: Download artifacts + # Install nuget.exe if it's not already there. + - task: NuGetToolInstaller@1 + displayName: Install nuget.exe + - script: | for %%I in (linux-arm linux-arm64 linux-x64 osx-arm64 osx-x64 win-arm64 win-x64 win-x86) do ( mkdir "$(Build.BinariesDirectory)\%%I" diff --git a/doc/building-coredistools.md b/doc/building-coredistools.md index dd99f6bb..9651e5f4 100644 --- a/doc/building-coredistools.md +++ b/doc/building-coredistools.md @@ -8,48 +8,46 @@ git clone https://github.com/dotnet/jitutils.git cd jitutils ``` -2. Checkout the LLVM project repository: +2. Checkout the LLVM project repository into a subdirectory named src/llvm-project: ``` -git clone --depth 1 --branch llvmorg-13.0.1 https://github.com/llvm/llvm-project.git src\llvm-project +git clone --depth 1 --branch llvmorg-17.0.6 https://github.com/llvm/llvm-project.git src\llvm-project ``` -4. Build `llvm-tblgen.exe`: +3. Build `llvm-tblgen.exe`: ``` build-tblgen.cmd ``` -5. Add the `bin` subdirectory to the `PATH`: +This builds llvm-tblgen.exe and puts it in the `bin` subdirectory. + +4. Add the `bin` subdirectory to the `PATH`: ``` set "PATH=%cd%\bin;%PATH%" -```` +``` -6. Build `coredistools.dll` for a combination of target OS and architecture. +This puts the just built lldb-tblgen.exe on the `PATH`. -For example, the following command will result in `coredistools.dll` binary that can be run on Windows x64: +5. Build `coredistools.dll` for a combination of target OS and architecture. +Build Windows x64, Windows x86, and Windows ARM64 binaries: ``` build-coredistools.cmd win-x64 +build-coredistools.cmd win-x86 +build-coredistools.cmd win-arm64 ``` -The file will be copied to subdirectory `artifacts` after the command finishes: +The file will be copied to subdirectory `artifacts` after the command finishes. E.g., for win-x64: ``` dir /A:-D /B /S artifacts\win-x64 F:\echesako\git\jitutils\artifacts\win-x64\bin\coredistools.dll F:\echesako\git\jitutils\artifacts\win-x64\lib\coredistools.lib ``` -7. Build Windows x86, Windows ARM and Windows ARM64 binaries: -``` -build-coredistools.cmd win-x86 -build-coredistools.cmd win-arm -build-coredistools.cmd win-arm64 -``` - ### Building Debug binaries -The `build-coredistools.cmd` script is set up to build a Release build. To create a Debug build with a PDB file, +The `build-coredistools.cmd` script is set up to build a Release build. To create a Debug build with a PDB file for debugging, change the `--config Release` line to `--config Debug`. -## Building on Linux +## Building on Linux / Mac 1. Checkout the jitutils repository: ``` @@ -59,29 +57,29 @@ cd jitutils 2. Checkout the LLVM project repository: ``` -git clone --depth 1 --branch llvmorg-13.0.1 https://github.com/llvm/llvm-project.git src/llvm-project +git clone --depth 1 --branch llvmorg-17.0.6 https://github.com/llvm/llvm-project.git src/llvm-project ``` -3. Download LLVM release from GitHub: - +3. Build `llvm-tblgen` in Docker: +You need to install the `ncurses-compat` package because the Mariner container we use doesn't have libtinfo.so.5, which the built +llvm-tblgen needs to be able to run. (Note that we build in the Mariner container, but we also run some built binaries, namely llvm-tblgen, +as part of the build process.) ``` -python3 eng/download-llvm-release.py -release llvmorg-13.0.1 -os linux +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-amd64 +sudo tdnf install -y ncurses-compat +./build-tblgen.sh linux-x64 /crossrootfs/x64 ``` -4. Locate under the current directory file `llvm-tblgen` -``` -find -name llvm-tblgen -./clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin/llvm-tblgen -``` -and add its parent directory location to the `PATH`: +This builds llvm-tblgen and puts it in the `bin` subdirectory. +4. Add `llvm-tblgen` to the PATH: ``` -export PATH=$(pwd)/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +export PATH=$(pwd)/bin:$PATH ``` -5. Build `libcoredistools.so` for Linux x64: +5. Build `libcoredistools.so` for Linux x64 (in the same Docker container): ``` -./build-coredistools.sh linux-x64 +./build-coredistools.sh linux-x64 /crossrootfs/x64 ``` The file will be copied to subdirectory `artifacts` after the command finishes: @@ -94,15 +92,16 @@ find ./artifacts -name libcoredistools.so 6. Build `libcoredistools.so` for Linux arm64 under Docker: ``` -docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-arm64-20220312201346-b2c2436 -export PATH=$(pwd)/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm64 +sudo tdnf install -y ncurses-compat +export PATH=$(pwd)/bin:$PATH ./build-coredistools.sh linux-arm64 /crossrootfs/arm64 ``` 7. Build `libcoredistools.so` for Linux arm under Docker: ``` -docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code -u $(id -u):$(id -g) mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-20220312201346-b9de666 -export PATH=$(pwd)/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04/bin:$PATH +docker run -it --rm --entrypoint /bin/bash -v ~/git/jitutils:/opt/code -w /opt/code mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-ubuntu-18.04-arm +sudo tdnf install -y ncurses-compat +export PATH=$(pwd)/bin:$PATH ./build-coredistools.sh linux-arm /crossrootfs/arm ``` - diff --git a/eng/download-llvm-release.py b/eng/download-llvm-release.py deleted file mode 100755 index 78fbac96..00000000 --- a/eng/download-llvm-release.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the .NET Foundation under one or more agreements. -# The .NET Foundation licenses this file to you under the MIT license. -# -import argparse -import io -import os -import sys -import tarfile -from urllib import request -from urllib.error import URLError, HTTPError - -Release_urls = { - 'llvmorg-13.0.1': { - 'linux': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/clang+llvm-13.0.1-x86_64-linux-gnu-ubuntu-18.04.tar.xz', - 'macos': 'https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/clang+llvm-13.0.1-x86_64-apple-darwin.tar.xz' - } -} - -def download_llvm_release(release_url, output_dir): - try: - with request.urlopen(release_url) as response: - downloaded_bytes = response.read() - except (URLError, HTTPError) as err: - print(err) - sys.exit(1) - - with io.BytesIO(downloaded_bytes) as downloaded_fileobj: - with tarfile.open(fileobj=downloaded_fileobj, mode='r:xz') as archive: - archive.extractall(path=output_dir) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-release', required=True, choices=Release_urls.keys()) - parser.add_argument('-os', required=True, choices=['linux', 'macos']) - parser.add_argument('-output-dir', dest='output_dir', default=os.getcwd()) - args = parser.parse_args() - - release_url = Release_urls[args.release][args.os] - download_llvm_release(release_url, args.output_dir) diff --git a/eng/download-llvm-release.yml b/eng/download-llvm-release.yml deleted file mode 100644 index 6ad1cd62..00000000 --- a/eng/download-llvm-release.yml +++ /dev/null @@ -1,12 +0,0 @@ -parameters: -- name: os - type: string -- name: release - type: string - -steps: -- script: python3 $(Build.SourcesDirectory)/eng/download-llvm-release.py -os ${{ parameters.os }} -release ${{ parameters.release }} -output-dir $(Pipeline.Workspace) - displayName: Download LLVM release from GitHub - -- script: echo '##vso[task.prependpath]'$(find $(Pipeline.Workspace) -name "clang+llvm-*" -type d)/bin - displayName: Add Clang and LLVM to the PATH diff --git a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec index f6b3a69c..24cd6f3f 100644 --- a/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec +++ b/src/coredistools/.nuget/Microsoft.NETCore.CoreDisTools.nuspec @@ -1,8 +1,8 @@ - Microsoft.NETCore.CoreDisTools - 1.1.0 + Microsoft.NETCore.CoreDisTools + 1.4.0 Microsoft.NETCore Instruction-wise Disassembler Microsoft Microsoft diff --git a/src/coredistools/coredistools.cpp b/src/coredistools/coredistools.cpp index f302b205..0ff8232f 100644 --- a/src/coredistools/coredistools.cpp +++ b/src/coredistools/coredistools.cpp @@ -1,4 +1,4 @@ -//===-------- coredistools.cpp - Dissassembly tools for CoreClr -----------===// +//===-------- coredistools.cpp - Disassembly tools for CoreClr ------------===// // // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. @@ -11,8 +11,6 @@ /// //===----------------------------------------------------------------------===// -#include "llvm/ADT/Optional.h" -#include "llvm/ADT/Triple.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" @@ -22,12 +20,12 @@ #include "llvm/MC/MCObjectFileInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/TargetParser/Host.h" #include "llvm/Support/Format.h" -#include "llvm/Support/Host.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Support/TargetRegistry.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Support/DataTypes.h" @@ -51,23 +49,26 @@ class BlockInfo { bool isEmpty() const { return BlockSize == 0; } - // A pointer to the code block to diassemble. + // A pointer to the code block to disassemble. const uint8_t *Ptr; + // The size of the code block to compare. uint64_t BlockSize; + // The original base address of the code block. uintptr_t Addr; + // An identifying string, debug output only const char *Name; }; -// A block iterator reoresents a code-point within a code block. +// A block iterator represents a code-point within a code block. // It represents an instruction within the block, and can move // forward to subsequent instructions via the advance() method. // The iterator can be in two modes: // Not-Decoded: Before DecodeInstruction() is called at this // code point -// Decoded: After the current instrction is Decoded. In this state, +// Decoded: After the current instruction is Decoded. In this state, // Inst is a valid MCInst and InstrSize is an actual non-zero // length of the current instruction. class BlockIterator : public BlockInfo { @@ -184,8 +185,6 @@ struct CorDisasm { protected: enum TargetArch TheTargetArch; const PrintControl *Print; - bool isThumb2MoveImmediateOpcode(unsigned int opcode) const; - bool isThumb2MoveTopOpcode(unsigned int opcode) const; private: bool setTarget(); @@ -212,19 +211,15 @@ struct CorDisasm { static const int X86NumPrefixes = 19; static const OpcodeMap X86Prefix[X86NumPrefixes]; - - // The following constants is a workaround and the opcode numbers - // were copied from TableGen-erated ${LLVM_BINARY_DIR}/lib/Target/ARM/ARMGenInstrInfo.inc - static const unsigned int Thumb2MoveImmediateOpcode = 4069; // Corresponds to t2MOVi16 - static const unsigned int Thumb2MoveTopOpcode = 4067; // Correspond to t2MOVTi16 }; struct CorAsmDiff : public CorDisasm { public: CorAsmDiff(enum TargetArch Target, const PrintControl *PControl = &DefaultPrintControl, - const OffsetComparator Comp = DefaultEqualityComparator) - : CorDisasm(Target, PControl), Comparator(Comp) {} + const OffsetComparator Comp = DefaultEqualityComparator, + const OffsetMunger Munge = nullptr) + : CorDisasm(Target, PControl), Comparator(Comp), Munger(Munge) {} bool nearDiff(const BlockInfo &LeftBlock, const BlockInfo &RightBlock, const void *UserData) const; @@ -233,9 +228,8 @@ struct CorAsmDiff : public CorDisasm { bool fail(const char *Mesg, const BlockIterator &Left, const BlockIterator &Right) const; - bool tryDecodeThumb2MoveImm32(const BlockIterator& Iter, unsigned int& Reg, unsigned int& Imm32) const; - OffsetComparator Comparator; + OffsetMunger Munger; }; // clang-format off @@ -262,6 +256,11 @@ CorDisasm::OpcodeMap const CorDisasm::X86Prefix[CorDisasm::X86NumPrefixes] = { }; // clang-format on +#if !defined(_MSC_VER) +// Disable "warning: default label in switch which covers all enumeration values [-Wcovered-switch-default]" +#pragma clang diagnostic ignored "-Wcovered-switch-default" +#endif + bool CorDisasm::setTarget() { // Figure out the target triple. @@ -365,7 +364,7 @@ bool CorDisasm::init() { string FeaturesStr; // No additional target specific attributes. if (TheTargetArch == Target_Arm64) { - Mcpu = "cortex-a76"; + Mcpu = "neoverse-n2"; } STI.reset(TheTarget->createMCSubtargetInfo(TargetTriple, Mcpu, FeaturesStr)); @@ -531,57 +530,6 @@ void CorDisasm::dumpBlock(const BlockInfo &Block) const { Print->Dump("-----------------------------------------------"); } -bool CorDisasm::isThumb2MoveImmediateOpcode(unsigned int Opcode) const { - return (Opcode == Thumb2MoveImmediateOpcode); -} - -bool CorDisasm::isThumb2MoveTopOpcode(unsigned int Opcode) const { - return (Opcode == Thumb2MoveTopOpcode); -} - -bool CorAsmDiff::tryDecodeThumb2MoveImm32(const BlockIterator& Curr, unsigned int& Reg, unsigned int& Imm32) const { - assert(Curr.isDecoded()); - - if (!isThumb2MoveImmediateOpcode(Curr.Inst.getOpcode())) { - return false; - } - - BlockIterator Next = Curr; - - Next.advance(); - - if (Next.isEmpty()) { - return false; - } - - decodeInstruction(Next); - - if (!Next.isDecoded()) { - return false; - } - - if (!isThumb2MoveTopOpcode(Next.Inst.getOpcode())) { - return false; - } - - const MCInst& MovImm = Curr.Inst; - const MCInst& MovTop = Next.Inst; - - if (MovImm.getOperand(0).getReg() != MovTop.getOperand(0).getReg()) { - return false; - } - - Reg = MovImm.getOperand(0).getReg(); - - const unsigned Lo16 = (unsigned int)MovImm.getOperand(1).getImm(); - const unsigned Hi16 = (unsigned int)MovTop.getOperand(2).getImm(); - - // Reconstruct 32-bit value that is loaded using movw/movt instruction pair. - Imm32 = Lo16 | (Hi16 << 16); - - return true; -} - // Compares two code sections for syntactic equality. This is the core of the // asm diffing logic. // @@ -633,31 +581,37 @@ bool CorAsmDiff::nearDiff(const BlockInfo &LeftBlock, return fail("Instruction Size Mismatch", Left, Right); } - if (TheTargetArch == Target_Thumb) { - unsigned int RegL = 0; - unsigned int RegR = 0; - - unsigned int Imm32L = 0; - unsigned int Imm32R = 0; + if (Munger != nullptr) + { + // Need to call the munger first + + uint64_t ImmL = 0; + uint64_t ImmR = 0; + uint32_t SkipL = 0; + uint32_t SkipR = 0; + + if (Munger(UserData, Left.BlockOffset(), Left.InstrSize, &ImmL, &ImmR, &SkipL, &SkipR)) + { + const bool constMatches = ((ImmL == ImmR) || Comparator(UserData, Left.BlockOffset(), Left.InstrSize, ImmL, ImmR)); + if (!constMatches) + { + return fail("Munged Immediate Operand Value Mismatch", Left, Right); + } - // On Thumb2 movw/movt instruction pair can be used to load a 32-bit value to a register. - // If this is the case, we should treat such instruction pair as **one** pseudo-instruction and try decoding them together. - if (tryDecodeThumb2MoveImm32(Left, RegL, Imm32L) && tryDecodeThumb2MoveImm32(Right, RegR, Imm32R)) { - if ((RegL == RegR) && ((Imm32L == Imm32R) || Comparator(UserData, Left.BlockOffset(), Left.InstrSize, Imm32L, Imm32R))) { - // When movw/movt instructions pairs load the same or "equivalent" 32-bit values - // to the same register advance both iterators to the positions after movt. + Left.advance(); + Right.advance(); - Left.advance(); + for (uint32_t i = 0; i < SkipL; i++) { decodeInstruction(Left); Left.advance(); + } - Right.advance(); + for (uint32_t i = 0; i < SkipR; i++) { decodeInstruction(Right); Right.advance(); - - continue; } - // Otherwise, do nothing and allow the comparison below to fail. + + continue; } } @@ -776,6 +730,8 @@ DllIface CorDisasm *NewDisasm(enum TargetArch Target, return nullptr; } +DllIface void FinishDisasm(const CorDisasm *Disasm) { delete Disasm; } + DllIface CorDisasm *InitBufferedDiffer(enum TargetArch Target, const OffsetComparator Comparator) { return NewDiffer(Target, &BufferedPrintControl, Comparator); @@ -784,7 +740,14 @@ DllIface CorDisasm *InitBufferedDiffer(enum TargetArch Target, DllIface CorAsmDiff *NewDiffer(enum TargetArch Target, const PrintControl *PControl, const OffsetComparator Comparator) { - CorAsmDiff *AsmDiff = new CorAsmDiff(Target, PControl, Comparator); + return NewDiffer2(Target, PControl, Comparator, nullptr); +} + +DllIface CorAsmDiff *NewDiffer2(enum TargetArch Target, + const PrintControl *PControl, + const OffsetComparator Comparator, + const OffsetMunger Munger) { + CorAsmDiff *AsmDiff = new CorAsmDiff(Target, PControl, Comparator, Munger); if (AsmDiff->init()) { return AsmDiff; @@ -794,8 +757,6 @@ DllIface CorAsmDiff *NewDiffer(enum TargetArch Target, return nullptr; } -DllIface void FinishDisasm(const CorDisasm *Disasm) { delete Disasm; } - DllIface void FinishDiff(const CorAsmDiff *AsmDiff) { delete AsmDiff; } DllIface size_t DisasmInstruction(const CorDisasm *Disasm, diff --git a/src/coredistools/coredistools.exports b/src/coredistools/coredistools.exports index 6506557c..9579e591 100644 --- a/src/coredistools/coredistools.exports +++ b/src/coredistools/coredistools.exports @@ -2,11 +2,12 @@ InitDisasm InitBufferedDisasm NewDisasm FinishDisasm -DisasmInstruction -DumpInstruction InitBufferedDiffer NewDiffer +NewDiffer2 FinishDiff +DisasmInstruction +DumpInstruction NearDiffCodeBlocks DumpCodeBlock DumpDiffBlocks diff --git a/src/coredistools/coredistools.h b/src/coredistools/coredistools.h index f567ba3c..f26693f9 100644 --- a/src/coredistools/coredistools.h +++ b/src/coredistools/coredistools.h @@ -1,11 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -//===--------- coredistools.h - Dissassembly tools for CoreClr ------------===// +//===--------- coredistools.h - Disassembly tools for CoreClr ------------===// // -// Core Disassembly Tools API Version 1.0.1-prerelease +// Core Disassembly Tools API Version 1.4.0 // Disassembly tools required by CoreCLR for utilities like -// GCStress and SuperPMI +// GCStress, SuperPMI, and R2RDump. //===----------------------------------------------------------------------===// #if !defined(_COREDISTOOLS_H_) @@ -61,7 +61,7 @@ struct PrintControl { // The type of a custom function provided by the user to determine // if two offsets are considered equivalent wrt diffing code blocks. // Offset1 and Offset2 are the two offsets to be compared. -// BlockOffset is the offest of the instructions (that contain Offset1 +// BlockOffset is the offset of the instructions (that contain Offset1 // and Offset2) from the beginning of their respective code blocks. // InstructionLength is the length of the current instruction being // compared for equivalency. @@ -69,6 +69,20 @@ typedef bool(__cdecl *OffsetComparator)(const void *UserData, size_t BlockOffset size_t InstructionLength, uint64_t Offset1, uint64_t Offset2); +// If an OffsetMunger function is defined, it is called before the OffsetComparator. +// If it returns `true` then: +// 1. the instructions are considered equivalent +// 2. the offsets have been decoded and "munged" (changed), and +// *Offset1 and *Offset2 are set to the values to use. +// 3. *SkipInstructions1 instructions in code stream 1 are skipped +// 4. *SkipInstructions2 instructions in code stream 2 are skipped +// +// This is typically used on arm32 to treat "movw/movt" as a single instruction +// generating a single constant. Similarly, for arm64 mov/movk/movk/movk sequences. +typedef bool(__cdecl *OffsetMunger)(const void *UserData, size_t BlockOffset, + size_t InstructionLength, uint64_t* Offset1, uint64_t* Offset2, + uint32_t* SkipInstructions1, uint32_t* SkipInstructions2); + // The Export/Import definitions for CoreDistools library are defined below. // A typedef for each interface function's type is defined in order to aid // the importer. @@ -77,6 +91,10 @@ typedef bool(__cdecl *OffsetComparator)(const void *UserData, size_t BlockOffset typedef CorDisasm * __cdecl InitDisasm_t(enum TargetArch Target); DllIface InitDisasm_t InitDisasm; +// Initialize the disassembler, using buffered print controls +typedef CorDisasm * __cdecl InitBufferedDisasm_t(enum TargetArch Target); +DllIface InitBufferedDisasm_t InitBufferedDisasm; + // Initialize the disassembler using custom print controls typedef CorDisasm * __cdecl NewDisasm_t(enum TargetArch Target, const PrintControl *PControl); @@ -86,6 +104,28 @@ DllIface NewDisasm_t NewDisasm; typedef void __cdecl FinishDisasm_t(const CorDisasm *Disasm); DllIface FinishDisasm_t FinishDisasm; +// Initialize a code differ using buffered output. +typedef CorDisasm * __cdecl InitBufferedDiffer_t(enum TargetArch Target, + const OffsetComparator Comparator); +DllIface InitBufferedDiffer_t InitBufferedDiffer; + +// Initialize the Code Differ +typedef CorAsmDiff * __cdecl NewDiffer_t(enum TargetArch Target, + const PrintControl *PControl, + const OffsetComparator Comparator); +DllIface NewDiffer_t NewDiffer; + +// Initialize the Code Differ, with an offset munger. +typedef CorAsmDiff * __cdecl NewDiffer2_t(enum TargetArch Target, + const PrintControl *PControl, + const OffsetComparator Comparator, + const OffsetMunger Munger); +DllIface NewDiffer2_t NewDiffer2; + +// Delete the Code Differ +typedef void __cdecl FinishDiff_t(const CorAsmDiff *AsmDiff); +DllIface FinishDiff_t FinishDiff; + // DisasmInstruction -- Disassemble one instruction // Arguments: // Disasm -- The Disassembler @@ -101,22 +141,27 @@ typedef size_t __cdecl DisasmInstruction_t(const CorDisasm *Disasm, const uint8_t *Bytes, size_t Maxlength); DllIface DisasmInstruction_t DisasmInstruction; -// Initialize the Code Differ -typedef CorAsmDiff * __cdecl NewDiffer_t(enum TargetArch Target, - const PrintControl *PControl, - const OffsetComparator Comparator); -DllIface NewDiffer_t NewDiffer; - -// Delete the Code Differ -typedef void __cdecl FinishDiff_t(const CorAsmDiff *AsmDiff); -DllIface FinishDiff_t FinishDiff; +// DumpInstruction -- Disassemble one instruction and output it +// Arguments: +// Disasm -- The Disassembler +// Address -- The address at which the bytes of the instruction +// are intended to execute +// Bytes -- Pointer to the actual bytes which need to be disassembled +// MaxLength -- Number of bytes available in Bytes buffer +// Returns: +// -- The Size of the disassembled instruction +// -- Zero on failure +typedef size_t __cdecl DumpInstruction_t(const CorDisasm *Disasm, + const uint8_t *Address, const uint8_t *Bytes, + size_t Maxlength); +DllIface DumpInstruction_t DumpInstruction; // NearDiffCodeBlocks -- Compare two code blocks for semantic // equivalence // Arguments: // AsmDiff -- The Asm-differ // UserData -- Any data the user wishes to pass through into -// the OffsetComparator +// the OffsetComparator/OffsetMunger // Address1 -- Address at which first block will execute // Bytes1 -- Pointer to the actual bytes of the first block // Size1 -- The size of the first block @@ -146,4 +191,12 @@ typedef void __cdecl DumpDiffBlocks_t(const CorAsmDiff *AsmDiff, const uint8_t *Bytes2, size_t Size2); DllIface DumpDiffBlocks_t DumpDiffBlocks; +// Get a pointer to the buffered output buffer. +typedef const char* __cdecl GetOutputBuffer_t(); +DllIface GetOutputBuffer_t GetOutputBuffer; + +// Clear the buffered output buffer. +typedef void __cdecl ClearOutputBuffer_t(); +DllIface ClearOutputBuffer_t ClearOutputBuffer; + #endif // !defined(_COREDISTOOLS_H_)