From c13aa5bf34b3fdf65ce4f0fdaacbb764b4955a5e Mon Sep 17 00:00:00 2001 From: Alexander Arlt Date: Tue, 27 Sep 2022 23:08:22 -0300 Subject: [PATCH] [tests] Refactor scripts/ASTImportTest.sh. --- scripts/ASTImportTest.sh | 248 ++++++++++++++++++++++++--------------- scripts/common.sh | 74 ++++++++++-- test/cmdlineTests.sh | 4 +- 3 files changed, 218 insertions(+), 108 deletions(-) diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index daf5a61632d7..1b7906efb531 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -1,105 +1,174 @@ #!/usr/bin/env bash +# ------------------------------------------------------------------------------ +# vim:ts=4:et +# This file is part of solidity. +# +# solidity is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# solidity is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with solidity. If not, see +# +# (c) solidity contributors. +# ------------------------------------------------------------------------------ +# Bash script to test the import/exports. +# ast import/export tests: +# - first exporting a .sol file to JSON, then loading it into the compiler +# and exporting it again. The second JSON should be identical to the first. + +set -euo pipefail -set -e - -# Bash script to test the ast-import option of the compiler by -# first exporting a .sol file to JSON, then loading it into the compiler -# and exporting it again. The second JSON should be identical to the first READLINK=readlink if [[ "$OSTYPE" == "darwin"* ]]; then READLINK=greadlink fi REPO_ROOT=$(${READLINK} -f "$(dirname "$0")"/..) SOLIDITY_BUILD_DIR=${SOLIDITY_BUILD_DIR:-${REPO_ROOT}/build} -SOLC=${SOLIDITY_BUILD_DIR}/solc/solc -SPLITSOURCES=${REPO_ROOT}/scripts/splitSources.py +SOLC=$SOLIDITY_BUILD_DIR/solc/solc +SPLITSOURCES=$REPO_ROOT/scripts/splitSources.py + +# shellcheck source=scripts/common.sh +source "${REPO_ROOT}/scripts/common.sh" + +function print_usage +{ + fail "Usage: ${0} ast [--exit-on-error]." +} + +function print_used_commands +{ + local test_directory="$1" + local export_command="$2" + local import_command="$3" + printError "You can find the files used for this test here: ${test_directory}" + printError "Used commands for test:" + printError "# export" + echo "$ ${export_command}" >&2 + printError "# import" + echo "$ ${import_command}" >&2 +} + +function print_stderr_stdout +{ + local error_message="$1" + local stderr_file="$2" + local stdout_file="$3" + printError "${error_message}" + printError "" + printError "stderr:" + cat "${stderr_file}" >&2 + printError "" + printError "stdout:" + cat "${stdout_file}" >&2 +} + +IMPORT_TEST_TYPE= +EXIT_ON_ERROR=0 +for PARAM in "$@" +do + case "$PARAM" in + ast) IMPORT_TEST_TYPE="ast" ;; + --exit-on-error) EXIT_ON_ERROR=1 ;; + *) print_usage ;; + esac +done SYNTAXTESTS_DIR="${REPO_ROOT}/test/libsolidity/syntaxTests" ASTJSONTESTS_DIR="${REPO_ROOT}/test/libsolidity/ASTJSON" -NSOURCES="$(find "$SYNTAXTESTS_DIR" -type f | wc -l)" - -# DEV_DIR="${REPO_ROOT}/../tmp/contracts/" -# NSOURCES="$(find $DEV_DIR -type f | wc -l)" #TODO use find command FAILED=0 UNCOMPILABLE=0 TESTED=0 -if [[ "$(find . -maxdepth 0 -type d -empty)" == "" ]]; then - echo "Test directory not empty. Skipping!" - exit 1 -fi +function ast_import_export_equivalence +{ + local sol_file="$1" + local input_files=( "${@:2}" ) + + # export ast - save ast json as expected result (silently) + local export_command="$SOLC --combined-json ast --pretty-json --json-indent 4 ${input_files[*]} > expected.json 2> stderr_export.txt" + if ! $SOLC --combined-json ast --pretty-json --json-indent 4 "${input_files[@]}" > expected.json 2> stderr_export.txt + then + print_stderr_stdout "ERROR: AST export failed for input file $sol_file." ./stderr_export.txt ./expected.json + print_used_commands "$(pwd)" "${export_command}" "${import_command}" + return 1 + fi -# function tests whether exporting and importing again leaves the JSON ast unchanged + # (re)import ast - and export it again as obtained result (silently) + local import_command="$SOLC --import-ast --combined-json ast --pretty-json --json-indent 4 expected.json > obtained.json 2> stderr_import.txt" + if ! $SOLC --import-ast --combined-json ast --pretty-json --json-indent 4 expected.json > obtained.json 2> stderr_import.txt + then + print_stderr_stdout "ERROR: AST export failed for input file $sol_file." ./stderr_import.txt ./obtained.json + print_used_commands "$(pwd)" "${export_command}" "${import_command}" + return 1 + fi + + # compare expected and obtained ast's + if diff_files expected.json obtained.json + then + echo -n "✅" + else + printError "❌ ERROR: AST reimport failed for ${sol_file}" + if [[ $EXIT_ON_ERROR == 1 ]] + then + print_used_commands "$(pwd)" "${export_command}" "${import_command}" + return 1 + fi + FAILED=$((FAILED + 1)) + fi + TESTED=$((TESTED + 1)) +} + +# function tests whether exporting and importing again is equivalent. # Results are recorded by adding to FAILED or UNCOMPILABLE. -# Also, in case of a mismatch a diff and the respective ASTs are printed +# Also, in case of a mismatch a diff is printed # Expected parameters: # $1 name of the file to be exported and imported # $2 any files needed to do so that might be in parent directories function testImportExportEquivalence { - local nth_input_file="$1" - IFS=" " read -r -a all_input_files <<< "$2" + local sol_file="$1" + local input_files=( "${@:2}" ) - if $SOLC "$nth_input_file" "${all_input_files[@]}" > /dev/null 2>&1 + # if compilable + if "$SOLC" --bin "${input_files[@]}" > /dev/null 2>&1 then - ! [[ -e stderr.txt ]] || { echo "stderr.txt already exists. Refusing to overwrite."; exit 1; } - - # save exported json as expected result (silently) - $SOLC --combined-json ast --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null - # import it, and export it again as obtained result (silently) - if ! $SOLC --import-ast --combined-json ast --pretty-json expected.json > obtained.json 2> stderr.txt - then - # For investigating, use exit 1 here so the script stops at the - # first failing test - # exit 1 - FAILED=$((FAILED + 1)) - echo -e "ERROR: AST reimport failed for input file $nth_input_file" - echo - echo "Compiler stderr:" - cat ./stderr.txt - echo - echo "Compiler stdout:" - cat ./obtained.json - return 1 - fi - DIFF="$(diff expected.json obtained.json)" - if [ "$DIFF" != "" ] - then - if [ "$DIFFVIEW" == "" ] - then - echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" - echo "Expected:" - cat ./expected.json - echo "Obtained:" - cat ./obtained.json - else - # Use user supplied diff view binary - $DIFFVIEW expected.json obtained.json - fi - FAILED=$((FAILED + 1)) - return 2 - fi - TESTED=$((TESTED + 1)) - rm expected.json obtained.json - rm -f stderr.txt + case "$IMPORT_TEST_TYPE" in + ast) ast_import_export_equivalence "${sol_file}" "${input_files[@]}" ;; + *) fail "Unknown import test type '${IMPORT_TEST_TYPE}'. Aborting." ;; + esac else - # echo "contract $solfile could not be compiled " UNCOMPILABLE=$((UNCOMPILABLE + 1)) fi - # return 0 } -echo "Looking at $NSOURCES .sol files..." WORKINGDIR=$PWD -# for solfile in $(find $DEV_DIR -name *.sol) +command_available "$SOLC" --version +command_available jq --version + +case "$IMPORT_TEST_TYPE" in + ast) TEST_DIRS=("${SYNTAXTESTS_DIR}" "${ASTJSONTESTS_DIR}") ;; + *) print_usage ;; +esac + # boost_filesystem_bug specifically tests a local fix for a boost::filesystem # bug. Since the test involves a malformed path, there is no point in running -# AST tests on it. See https://github.com/boostorg/filesystem/issues/176 -# shellcheck disable=SC2044 -for solfile in $(find "$SYNTAXTESTS_DIR" "$ASTJSONTESTS_DIR" -name "*.sol" -and -not -name "boost_filesystem_bug.sol") +# tests on it. See https://github.com/boostorg/filesystem/issues/176 +IMPORT_TEST_FILES=$(find "${TEST_DIRS[@]}" -name "*.sol" -and -not -name "boost_filesystem_bug.sol") + +NSOURCES="$(echo "${IMPORT_TEST_FILES}" | wc -l)" +echo "Looking at ${NSOURCES} .sol files..." + +for solfile in $IMPORT_TEST_FILES do - echo -n "." + echo -n "·" # create a temporary sub-directory FILETMP=$(mktemp -d) cd "$FILETMP" @@ -108,48 +177,39 @@ do OUTPUT=$("$SPLITSOURCES" "$solfile") SPLITSOURCES_RC=$? set -e - if [ ${SPLITSOURCES_RC} == 0 ] + + if [[ $SPLITSOURCES_RC == 0 ]] then - # echo $OUTPUT - NSOURCES=$((NSOURCES - 1)) - for i in $OUTPUT; - do - testImportExportEquivalence "$i" "$OUTPUT" - NSOURCES=$((NSOURCES + 1)) - done - elif [ ${SPLITSOURCES_RC} == 1 ] + IFS=' ' read -ra OUTPUT_ARRAY <<< "$OUTPUT" + testImportExportEquivalence "$solfile" "${OUTPUT_ARRAY[@]}" + elif [ $SPLITSOURCES_RC == 1 ] then - testImportExportEquivalence "$solfile" - elif [ ${SPLITSOURCES_RC} == 2 ] + testImportExportEquivalence "$solfile" "$solfile" + elif [ $SPLITSOURCES_RC == 2 ] then # The script will exit with return code 2, if an UnicodeDecodeError occurred. # This is the case if e.g. some tests are using invalid utf-8 sequences. We will ignore # these errors, but print the actual output of the script. - echo -e "\n${OUTPUT}\n" - testImportExportEquivalence "$solfile" + printError "\n\n${OUTPUT}\n\n" + testImportExportEquivalence "$solfile" "$solfile" else # All other return codes will be treated as critical errors. The script will exit. - echo -e "\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." - echo -e "\n${OUTPUT}\n" - - cd "$WORKINGDIR" - # Delete temporary files - rm -rf "$FILETMP" + printError "\n\nGot unexpected return code ${SPLITSOURCES_RC} from ${SPLITSOURCES}. Aborting." + printError "\n\n${OUTPUT}\n\n" exit 1 fi - cd "$WORKINGDIR" + cd "${WORKINGDIR}" # Delete temporary files - rm -rf "$FILETMP" + rm -rf "${FILETMP}" done -echo "" +echo -if [ "$FAILED" = 0 ] +if (( FAILED == 0 )) then - echo "SUCCESS: $TESTED syntaxTests passed, $FAILED failed, $UNCOMPILABLE could not be compiled ($NSOURCES sources total)." + echo "SUCCESS: ${TESTED} tests passed, ${FAILED} failed, ${UNCOMPILABLE} could not be compiled (${NSOURCES} sources total)." else - echo "FAILURE: Out of $NSOURCES sources, $FAILED failed, ($UNCOMPILABLE could not be compiled)." - exit 1 + fail "FAILURE: Out of ${NSOURCES} sources, ${FAILED} failed, (${UNCOMPILABLE} could not be compiled)." fi diff --git a/scripts/common.sh b/scripts/common.sh index 98f63ecb80c5..bce5461879aa 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -26,18 +26,18 @@ set -e # changes directory. The paths returned by `caller` are relative to it. _initial_work_dir=$(pwd) -if [ "$CIRCLECI" ] +if [[ ${CIRCLECI:-} != "" ]] then export TERM="${TERM:-xterm}" - function printTask { echo "$(tput bold)$(tput setaf 2)$1$(tput setaf 7)"; } - function printError { >&2 echo "$(tput setaf 1)$1$(tput setaf 7)"; } - function printWarning { >&2 echo "$(tput setaf 11)$1$(tput setaf 7)"; } - function printLog { echo "$(tput setaf 3)$1$(tput setaf 7)"; } + function printTask { echo -e "$(tput bold)$(tput setaf 2)$1$(tput setaf 7)"; } + function printError { >&2 echo -e "$(tput setaf 1)$1$(tput setaf 7)"; } + function printWarning { >&2 echo -e "$(tput setaf 11)$1$(tput setaf 7)"; } + function printLog { echo -e "$(tput setaf 3)$1$(tput setaf 7)"; } else - function printTask { echo "$(tput bold)$(tput setaf 2)$1$(tput sgr0)"; } - function printError { >&2 echo "$(tput setaf 1)$1$(tput sgr0)"; } - function printWarning { >&2 echo "$(tput setaf 11)$1$(tput sgr0)"; } - function printLog { echo "$(tput setaf 3)$1$(tput sgr0)"; } + function printTask { echo -e "$(tput bold)$(tput setaf 2)$1$(tput sgr0)"; } + function printError { >&2 echo -e "$(tput setaf 1)$1$(tput sgr0)"; } + function printWarning { >&2 echo -e "$(tput setaf 11)$1$(tput sgr0)"; } + function printLog { echo -e "$(tput setaf 3)$1$(tput sgr0)"; } fi function checkDputEntries @@ -193,16 +193,56 @@ function msg_on_error fi } + function diff_values { (( $# >= 2 )) || fail "diff_values requires at least 2 arguments." local value1="$1" local value2="$2" - shift - shift + shift 2 - diff --unified=0 <(echo "$value1") <(echo "$value2") "$@" + if ! diff --unified=0 <(echo "$value1") <(echo "$value2") "$@" + then + if [ "${DIFFVIEW:-}" == "" ] + then + printError "ERROR: values differ:" + printError "Expected:" + printError "${value1}" + printError "Obtained:" + printError "${value2}" + else + # Use user supplied diff view binary + printError "ERROR: values differ." + "$DIFFVIEW" "${file1}" "${file2}" + fi + return 1 + fi +} + +function diff_files +{ + (( $# >= 2 )) || fail "diff_files requires at least 2 arguments." + + local file1="$1" + local file2="$2" + + if ! diff "${file1}" "${file2}" + then + if [ "${DIFFVIEW:-}" == "" ] + then + printError "ERROR: files differ: ${file1} vs. ${file2}" + printError "Expected:" + cat "${file1}" >&2 + printError "Obtained:" + cat "${file2}" >&2 + else + # Use user supplied diff view binary + "$DIFFVIEW" "${file1}" "${file2}" + fi + return 1 + fi + return 0 } function safe_kill @@ -277,3 +317,13 @@ function split_on_empty_lines_into_numbered_files awk -v RS= "{print > (\"${path_prefix}_\"NR \"${path_suffix}\")}" } + +function command_available +{ + local program="$1" + local parameters=${*:2} + if ! "${program}" "${parameters}" > /dev/null 2>&1 + then + fail "'${program}' not found or not executed successfully with parameter(s) '${parameters}'. aborting." + fi +} diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index b320da05e0f4..38b35ea953d5 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -662,11 +662,11 @@ SOLTMPDIR=$(mktemp -d) ) rm -r "$SOLTMPDIR" -printTask "Testing AST import..." +printTask "Testing AST import/export..." SOLTMPDIR=$(mktemp -d) ( cd "$SOLTMPDIR" - if ! "$REPO_ROOT/scripts/ASTImportTest.sh" + if ! "$REPO_ROOT/scripts/ASTImportTest.sh" ast then rm -r "$SOLTMPDIR" fail