From 5c396f4683aa44b5c4fb7f6467d2d8159fea3f90 Mon Sep 17 00:00:00 2001 From: nishant-sachdeva Date: Wed, 18 Sep 2024 11:51:47 +0530 Subject: [PATCH] LIT testing broken into modular functions --- src/IR2Vec.cpp | 1 - src/test-suite/CMakeLists.txt | 3 +- src/test-suite/sanity_check.sh.cmake | 149 ++++++++++-------- src/test-suite/test-fa.lit | 7 + .../{test-ir2vec.lit => test-sym.lit} | 3 +- 5 files changed, 94 insertions(+), 69 deletions(-) create mode 100644 src/test-suite/test-fa.lit rename src/test-suite/{test-ir2vec.lit => test-sym.lit} (67%) diff --git a/src/IR2Vec.cpp b/src/IR2Vec.cpp index 4ce25ed0..ef447e98 100644 --- a/src/IR2Vec.cpp +++ b/src/IR2Vec.cpp @@ -12,7 +12,6 @@ #include "Symbolic.h" #include "version.h" #include "llvm/Support/CommandLine.h" -#include #include #include diff --git a/src/test-suite/CMakeLists.txt b/src/test-suite/CMakeLists.txt index c1e5d114..e4cced32 100644 --- a/src/test-suite/CMakeLists.txt +++ b/src/test-suite/CMakeLists.txt @@ -77,4 +77,5 @@ file(COPY index-llvm17.files DESTINATION ./) configure_file(lit.site.cfg.py.in lit.site.cfg.py @ONLY) file(COPY test-lit.py DESTINATION ./) -file(COPY test-ir2vec.lit DESTINATION ./) +file(COPY test-fa.lit DESTINATION ./) +file(COPY test-sym.lit DESTINATION ./) diff --git a/src/test-suite/sanity_check.sh.cmake b/src/test-suite/sanity_check.sh.cmake index db6e9b52..95ef3909 100644 --- a/src/test-suite/sanity_check.sh.cmake +++ b/src/test-suite/sanity_check.sh.cmake @@ -35,35 +35,90 @@ functions=("main" "buildMatchingMachine" "search" "BellamFord" "BFS" "isBCUtil" "selectKItems" "getMinDiceThrows" "countSort" "subset_sum" "SolveSudoku" "SCC" "solveKTUtil" "topologicalSort" "transitiveClosure" "insertSuffix" "tugOfWar" "isUgly" "Union" "printVertexCover" "findMaxProfit" "solveWordWrap") -perform_vector_comparison() { - LEVEL=$1 - FILE_PREFIX=$2 +perform_program_vector_comparison() { + LEVEL="p" + FILE_PREFIX="p" echo -e "${BLUE}${BOLD}Running ir2vec on ${FILE_PREFIX}-level for ${EncodingType} encoding type" ORIG_FILE=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/ir2vec.txt - VIR_FILE=ir2vec_${FILE_PREFIX}.txt - + VIR_FILE=ir2vec_${FILE_PREFIX}_${PASS}.txt # SQLite specific variables. if [[ "$ENABLE_SQLITE" == "ON" ]]; then - SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt + SQLITE_VIR=sqlite3_${FILE_PREFIX}_${PASS}.txt SQLITE_INPUT=./sqlite3.ll SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt fi + # if file prefix is p or f, run the first while loop, else, run the second while loop + while IFS= read -r d; do + ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${VIR_FILE} ${d} &> /dev/null + done < index-${SEED_VERSION}.files + wait + + # SQLITE is currently only tested against the program (p) level + if [[ "$ENABLE_SQLITE" == "ON" && "$FILE_PREFIX" == "p" ]]; then + ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null + fi + + TEMP=temp_${EncodingType}_${SEED_VERSION}_${FILE_PREFIX} + if ls *${VIR_FILE} 1> /dev/null 2>&1; then + mkdir -p ${TEMP} + mv *${VIR_FILE} ${TEMP}/ + + d=$(diff <(sed -e 's/^ *#[0-9]* *//g' ${ORIG_FILE}) <(sed -e 's/^ *#[0-9]* *//g' ${TEMP}/${VIR_FILE})) + if [ "$d" == "" ]; then + echo -e "${GREEN}${BOLD}[Test Passed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}" + else + echo -e "$(tput bold)${RED}[Test Failed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}" + rm -rf ${TEMP} + exit 1 + fi + else + echo -e "$(tput bold)${RED}[Error] No embeddings are generated.${NC}" + rm -rf ${TEMP} + exit 1 + fi + + # SQLite tests only if its enabled + if [[ "$ENABLE_SQLITE" == "ON" ]]; then + if [[ ! -e "$SQLITE_VIR" ]]; then + echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" + rm -rf ${TEMP} + exit 1 + fi + mv ${SQLITE_VIR} ${TEMP}/ + + d=$(diff <(sed -e 's/^ *#[0-9]* *//g' ${SQLITE_ORIG}) <(sed -e 's/^ *#[0-9]* *//g' ${TEMP}/${SQLITE_VIR})) + + if [ "$d" == "" ]; then + echo -e "${GREEN}${BOLD}[Test Passed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}" + else + echo -e "$(tput bold)${RED}[Test Failed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}" + rm -rf ${TEMP} + exit 1 + fi + fi + rm -rf ${TEMP} +} + +perform_vector_comparison() { + LEVEL=$1 + FILE_PREFIX=$2 + + echo -e "${BLUE}${BOLD}Running ir2vec on ${FILE_PREFIX}-level for ${EncodingType} encoding type" + + ORIG_FILE=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/ir2vec.txt + VIR_FILE=ir2vec_${FILE_PREFIX}_${PASS}.txt + # if file prefix is p or f, run the first while loop, else, run the second while loop - if [[ "$FILE_PREFIX" == "p" || "$FILE_PREFIX" == "f" ]]; then + if [[ "$FILE_PREFIX" == "f" ]]; then while IFS= read -r d; do ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${VIR_FILE} ${d} &> /dev/null done < index-${SEED_VERSION}.files wait - - # SQLITE is currently only tested against the program (p) level - if [[ "$ENABLE_SQLITE" == "ON" && "$FILE_PREFIX" == "p" ]]; then - ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null - fi else while IFS= read -r d_on do @@ -76,62 +131,26 @@ perform_vector_comparison() { fi TEMP=temp_${EncodingType}_${SEED_VERSION}_${FILE_PREFIX} - if [[ "$LEVEL" == "p" ]]; then - if ls *${VIR_FILE} 1> /dev/null 2>&1; then - mkdir -p ${TEMP} - mv *${VIR_FILE} ${TEMP}/ - - d=$(diff <(sed -e 's/^ *#[0-9]* *//g' ${ORIG_FILE}) <(sed -e 's/^ *#[0-9]* *//g' ${TEMP}/${VIR_FILE})) - if [ "$d" == "" ]; then - echo -e "${GREEN}${BOLD}[Test Passed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}" - else - echo -e "$(tput bold)${RED}[Test Failed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}" - exit 1 - fi - else - echo -e "$(tput bold)${RED}[Error] No embeddings are generated.${NC}" - exit 1 - fi - - # SQLite tests only if its enabled - if [[ "$ENABLE_SQLITE" == "ON" ]]; then - if [[ ! -e "$SQLITE_VIR" ]]; then - echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}" - exit 1 - fi - mv ${SQLITE_VIR} ${TEMP}/ - - d=$(diff <(sed -e 's/^ *#[0-9]* *//g' ${SQLITE_ORIG}) <(sed -e 's/^ *#[0-9]* *//g' ${TEMP}/${SQLITE_VIR})) - - if [ "$d" == "" ]; then - echo -e "${GREEN}${BOLD}[Test Passed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}" - else - echo -e "$(tput bold)${RED}[Test Failed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}" - exit 1 - fi - fi - - else - if ls *${VIR_FILE} 1> /dev/null 2>&1 + if ls *${VIR_FILE} 1> /dev/null 2>&1; then + mkdir -p ${TEMP} + mv *${VIR_FILE} ${TEMP}/ + # removing demangled file and function names before '=' + sed 's/.*=//' ${ORIG_FILE} > orig_file_${FILE_PREFIX}.txt + sed 's/.*=//' ${TEMP}/${VIR_FILE}> vir_file_${FILE_PREFIX}.txt + d_f=$(diff orig_file_${FILE_PREFIX}.txt vir_file_${FILE_PREFIX}.txt ) + + if [ "$d_f" == "" ] then - mkdir -p ${TEMP} - mv *${VIR_FILE} ${TEMP}/ - # removing demangled file and function names before '=' - sed 's/.*=//' ${ORIG_FILE} > orig_file_${FILE_PREFIX}.txt - sed 's/.*=//' ${TEMP}/${VIR_FILE}> vir_file_${FILE_PREFIX}.txt - d_f=$(diff orig_file_${FILE_PREFIX}.txt vir_file_${FILE_PREFIX}.txt ) - - if [ "$d_f" == "" ] - then - echo -e "${GREEN}${BOLD}[Test Passed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}" - - else - echo -e "$(tput bold)${RED}[Test Failed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}" - exit 1 - fi + echo -e "${GREEN}${BOLD}[Test Passed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}" else - echo -e "$(tput bold)${RED}[Error] No embeddings are generated.${NC}" + echo -e "$(tput bold)${RED}[Test Failed] Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}" + rm -rf ${TEMP} exit 1 fi + else + echo -e "$(tput bold)${RED}[Error] No embeddings are generated.${NC}" + rm -rf ${TEMP} + exit 1 fi + rm -rf ${TEMP} } diff --git a/src/test-suite/test-fa.lit b/src/test-suite/test-fa.lit new file mode 100644 index 00000000..eed156ab --- /dev/null +++ b/src/test-suite/test-fa.lit @@ -0,0 +1,7 @@ +// RUN: bash %s FA llvm17 + +source sanity_check.sh + +perform_program_vector_comparison +perform_vector_comparison "f" "f" +perform_vector_comparison "f" "onDemand" diff --git a/src/test-suite/test-ir2vec.lit b/src/test-suite/test-sym.lit similarity index 67% rename from src/test-suite/test-ir2vec.lit rename to src/test-suite/test-sym.lit index bdae9a62..8392e402 100644 --- a/src/test-suite/test-ir2vec.lit +++ b/src/test-suite/test-sym.lit @@ -1,8 +1,7 @@ -// RUN: bash %s FA llvm17 // RUN: bash %s SYM llvm17 source sanity_check.sh -perform_vector_comparison "p" "p" +perform_program_vector_comparison perform_vector_comparison "f" "f" perform_vector_comparison "f" "onDemand"