From 56b5e61d516b1f0065705dcefed8b79047dede92 Mon Sep 17 00:00:00 2001 From: jparisu Date: Fri, 11 Nov 2022 10:13:26 +0100 Subject: [PATCH 1/5] Add ASAN CI tests for fastdds and Discovery Server Signed-off-by: jparisu --- .github/workflows/asan.yml | 67 ------ .github/workflows/{ => asan}/asan_colcon.meta | 12 + .github/workflows/asan_log_parser.py | 56 ----- .github/workflows/sanitizer-tests.yaml | 132 +++++++++++ .github/workflows/utils/log_parser.py | 220 ++++++++++++++++++ .../workflows/utils/specific_errors_filter.sh | 23 ++ 6 files changed, 387 insertions(+), 123 deletions(-) delete mode 100644 .github/workflows/asan.yml rename .github/workflows/{ => asan}/asan_colcon.meta (65%) delete mode 100644 .github/workflows/asan_log_parser.py create mode 100644 .github/workflows/sanitizer-tests.yaml create mode 100644 .github/workflows/utils/log_parser.py create mode 100644 .github/workflows/utils/specific_errors_filter.sh diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml deleted file mode 100644 index c8b33d0407c..00000000000 --- a/.github/workflows/asan.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Address Sanitizer analysis - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - schedule: - - cron: '0 1 * * *' - -jobs: - - asan-test: - if: ${{ !(contains(github.event.pull_request.labels.*.name, 'no-test') || - contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} - runs-on: ubuntu-20.04 - - steps: - - name: Sync eProsima/Fast-DDS repository - uses: actions/checkout@v3 - with: - path: src/Fast-DDS - - - name: Install apt packages - uses: ./src/Fast-DDS/.github/actions/install-apt-packages - - - name: Install GTest - uses: ./src/Fast-DDS/.github/actions/install-gtest-linux - - - name: Install Python packages - uses: ./src/Fast-DDS/.github/actions/install-python-packages - - - name: Fetch Fast DDS dependencies - uses: ./src/Fast-DDS/.github/actions/fetch-fastdds-repos - - - name: Build workspace - run: | - cat src/Fast-DDS/.github/workflows/asan_colcon.meta - colcon build \ - --event-handlers=console_direct+ \ - --metas src/Fast-DDS/.github/workflows/asan_colcon.meta - - - name: Run tests Fast DDS - run: | - source install/setup.bash && \ - colcon test \ - --packages-select fastrtps \ - --event-handlers=console_direct+ \ - --return-code-on-test-failure \ - --ctest-args \ - --label-exclude xfail \ - --timeout 60 - - - name: Upload Logs - uses: actions/upload-artifact@v1 - with: - name: asan-logs - path: log/ - if: always() - - - name: Report ASAN errors - if: always() - run: | - echo -n "**ASAN Errors**: " >> $GITHUB_STEP_SUMMARY - echo $(sed 's/==.*==ERROR:/==.*==ERROR:\n/g' log/latest_test/fastrtps/stdout_stderr.log | grep -c "==.*==ERROR:") >> $GITHUB_STEP_SUMMARY - python3 src/Fast-DDS/.github/workflows/asan_log_parser.py diff --git a/.github/workflows/asan_colcon.meta b/.github/workflows/asan/asan_colcon.meta similarity index 65% rename from .github/workflows/asan_colcon.meta rename to .github/workflows/asan/asan_colcon.meta index 4f773e8dbc5..103c274a39a 100644 --- a/.github/workflows/asan_colcon.meta +++ b/.github/workflows/asan/asan_colcon.meta @@ -17,6 +17,18 @@ "-DSANITIZER=Address", "-DCMAKE_CXX_FLAGS='-Werror'" ] + }, + + "discovery-server": + { + "cmake-args": + [ + "-DCMAKE_BUILD_TYPE=Debug", + "-DSANITIZER=Address", + + "-D__TODO__='Following line should be remove'", + "-DCMAKE_CXX_FLAGS='-fsanitize=address'", + ] } } } diff --git a/.github/workflows/asan_log_parser.py b/.github/workflows/asan_log_parser.py deleted file mode 100644 index c0071dbc8e7..00000000000 --- a/.github/workflows/asan_log_parser.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2022 Proyectos y Sistemas de Mantenimiento SL (eProsima). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Script to parse the colcon test output file.""" - -import re -import os - -from tomark import Tomark - -# Python summary file saved in GITHUB_STEP_SUMMARY environment variable -SUMMARY_FILE = os.getenv('GITHUB_STEP_SUMMARY') -LOG_FILE = 'log/latest_test/fastrtps/stdout_stderr.log' - -# Save the lines with failed tests -saved_lines = [] -with open(LOG_FILE, 'r') as file: - for line in reversed(file.readlines()): - saved_lines.append(line) - if (re.search('.*The following tests FAILED:.*', line)): - break - -# Exit if no test failed -if (not saved_lines): - exit(0) - -failed_tests = [] -for test in saved_lines: - if (re.search(r'\d* - .* \(.+\)', test)): - split_test = test.split() - failed_tests.insert( - 0, - dict({ - 'ID': split_test[0], - 'Name': split_test[2], - 'Type': test[test.find('(')+1:test.find(')')] - })) - -# Convert python dict to markdown table -md_table = Tomark.table(failed_tests) -print(md_table) - -# Save table of failed test to GitHub action summary file -with open(SUMMARY_FILE, 'a') as file: - file.write(f'\n{md_table}') diff --git a/.github/workflows/sanitizer-tests.yaml b/.github/workflows/sanitizer-tests.yaml new file mode 100644 index 00000000000..3c0791debe5 --- /dev/null +++ b/.github/workflows/sanitizer-tests.yaml @@ -0,0 +1,132 @@ +name: Sanitizer analysis + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + schedule: + - cron: '0 1 * * *' + +jobs: + + asan-test: + + if: ${{ !(contains(github.event.pull_request.labels.*.name, 'no-test') || + contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} + + runs-on: ubuntu-20.04 + + steps: + - name: Sync eProsima/Fast-DDS repository + uses: actions/checkout@v3 + with: + path: src/Fast-DDS + + - name: Install apt packages + uses: ./src/Fast-DDS/.github/actions/install-apt-packages + + - name: Install GTest + uses: ./src/Fast-DDS/.github/actions/install-gtest-linux + + - name: Install Python packages + uses: ./src/Fast-DDS/.github/actions/install-python-packages + + - name: Fetch Fast DDS dependencies + uses: ./src/Fast-DDS/.github/actions/fetch-fastdds-repos + + - name: Build workspace + run: | + cat src/Fast-DDS/.github/workflows/asan/asan_colcon.meta + colcon build \ + --event-handlers=console_direct+ \ + --metas src/Fast-DDS/.github/workflows/asan/asan_colcon.meta + + - name: Run tests Fast DDS + run: | + source install/setup.bash && \ + colcon test \ + --packages-select fastrtps \ + --event-handlers=console_direct+ \ + --return-code-on-test-failure \ + --ctest-args \ + --label-exclude xfail \ + --timeout 60 + + - name: Upload Logs + uses: actions/upload-artifact@v1 + with: + name: asan-logs + path: log/ + if: always() + + - name: Report ASAN errors + if: always() + run: | + bash src/Fast-DDS/.github/workflows/utils/specific_errors_filter.sh "==ERROR:" log/latest_test/fastrtps/stdout_stderr.log _tmp_specific_error_file.log + python3 src/Fast-DDS/.github/workflows/utils/log_parser.py --log-file log/latest_test/fastrtps/stdout_stderr.log --specific-error-file _tmp_specific_error_file.log --output-file $GITHUB_STEP_SUMMARY --sanitizer=asan + + + asan-discovery-server-test: + + if: ${{ !(contains(github.event.pull_request.labels.*.name, 'no-test') || + contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} + + runs-on: ubuntu-20.04 + + steps: + - name: Sync eProsima/Fast-DDS repository + uses: actions/checkout@v3 + with: + path: src/Fast-DDS + + - name: Install apt packages + uses: ./src/Fast-DDS/.github/actions/install-apt-packages + + - name: Install GTest + uses: ./src/Fast-DDS/.github/actions/install-gtest-linux + + - name: Install Python packages + uses: ./src/Fast-DDS/.github/actions/install-python-packages + + - name: Fetch repositories + run: | + mkdir -p src + cd src + git clone https://github.com/eProsima/foonathan_memory_vendor.git + git clone https://github.com/eProsima/Fast-CDR.git + git clone https://github.com/eProsima/Discovery-Server.git + git clone https://github.com/google/googletest.git --branch release-1.10.0 + cd .. + + - name: Build workspace + run: | + cat src/Fast-DDS/.github/workflows/asan/asan_colcon.meta + colcon build \ + --event-handlers=console_direct+ \ + --metas src/Fast-DDS/.github/workflows/asan/asan_colcon.meta + + - name: Run tests Fast DDS + run: | + source install/setup.bash && \ + colcon test \ + --packages-select discovery-server \ + --event-handlers=console_direct+ \ + --return-code-on-test-failure \ + --ctest-args \ + --label-exclude xfail \ + --timeout 60 + + - name: Upload Logs + uses: actions/upload-artifact@v1 + with: + name: asan-logs + path: log/ + if: always() + + - name: Report ASAN errors + if: always() + run: | + bash src/Fast-DDS/.github/workflows/utils/specific_errors_filter.sh "==ERROR:" log/latest_test/discovery-server/stdout_stderr.log _tmp_specific_error_file.log + python3 src/Fast-DDS/.github/workflows/utils/log_parser.py --log-file log/latest_test/discovery-server/stdout_stderr.log --specific-error-file _tmp_specific_error_file.log --output-file $GITHUB_STEP_SUMMARY --sanitizer=asan diff --git a/.github/workflows/utils/log_parser.py b/.github/workflows/utils/log_parser.py new file mode 100644 index 00000000000..39c3dc32d30 --- /dev/null +++ b/.github/workflows/utils/log_parser.py @@ -0,0 +1,220 @@ +# Copyright 2022 Proyectos y Sistemas de Mantenimiento SL (eProsima). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to parse the colcon test output file.""" + +import argparse +import re + +from tomark import Tomark + +DESCRIPTION = """Script to read a test log and return a summary table""" +USAGE = ('python3 log_parser.py') + + +def parse_options(): + """ + Parse arguments. + + :return: The arguments parsed. + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + add_help=True, + description=(DESCRIPTION), + usage=(USAGE) + ) + + required_args = parser.add_argument_group('required arguments') + required_args.add_argument( + '-i', + '--log-file', + type=str, + required=True, + help='Path to test log file.' + ) + required_args.add_argument( + '-e', + '--specific-error-file', + type=str, + required=True, + help='Path to file with ASAN or TSAN specific errors.' + ) + required_args.add_argument( + '-o', + '--output-file', + type=str, + required=True, + help='Path to output file.' + ) + + parser.add_argument( + '-s', + '--sanitizer', + type=str, + required=True, + help='Sanitizer to use. [ASAN|TSAN] no case sensitive.' + ) + + return parser.parse_args() + + +def failure_test_list( + log_file_path: str): + + # failed tests + saved_lines = [] + with open(log_file_path, 'r') as file: + for line in reversed(file.readlines()): + saved_lines.append(line) + if (re.search('.*The following tests FAILED:.*', line)): + break + + # Exit if no test failed + if (not saved_lines): + return {} + + failed_tests = [] + for test in saved_lines: + if (re.search(r'\d* - .* \(.+\)', test)): + split_test = test.strip().split() + failed_tests.insert( + 0, + dict({ + 'ID': split_test[-4], + 'Name': split_test[-2], + 'Type': test[test.find('(')+1:test.find(')')] + })) + + if len(failed_tests) == 0: + failed_tests.append( + dict({ + 'ID': '-1', + 'Name': 'No tests failed', + 'Type': 'Hooray' + }) + ) + + return failed_tests + + +def _common_line_splitter( + line: str, + text_to_split_start: str, + text_to_split_end: str = None) -> str: + start_index = line.find(text_to_split_start) + if start_index == -1: + return line + elif text_to_split_end: + end_index = line.find(text_to_split_end) + if end_index != -1: + return line[(start_index + + len(text_to_split_start)): + end_index].strip() + return line[(start_index + len(text_to_split_start)):].strip() + + +def asan_line_splitter( + line: str): + return _common_line_splitter( + line=line, + text_to_split_start='==ERROR: ') + + +def tsan_line_splitter( + line: str): + return _common_line_splitter( + line=line, + text_to_split_start='WARNING: ThreadSanitizer: ', + text_to_split_end=' (pid=') + + +def common_specific_errors_list( + errors_file_path: str, + line_splitter): + result = [ + {'Error': k, 'Repetitions': v} + for k, v + in common_specific_errors_dict( + errors_file_path, + line_splitter).items()] + + if len(result) == 0: + result.append({'Error': 'No errors', 'Repetitions': '0'}) + + return result + + +def common_specific_errors_dict( + errors_file_path: str, + line_splitter): + + # failed tests + errors = {} + with open(errors_file_path, 'r') as file: + for line in file.readlines(): + error_id = line_splitter(line) + if error_id in errors: + errors[error_id] += 1 + else: + errors[error_id] = 1 + + return errors + + +def print_list_to_markdown( + title: str, + result: list, + output_file_path: str): + + # Convert python dict to markdown table + md_table = Tomark.table(result) + print(md_table) + + # Save table of failed test to output summary file + with open(output_file_path, 'a') as file: + file.write(f'\n## {title}\n') + file.write(f'\n{md_table}') + + +def main(): + + # Parse arguments + args = parse_options() + + # Get specific ASAN or TSAN variables + asan = args.sanitizer.lower() == 'asan' + line_splitter = (asan_line_splitter if asan else tsan_line_splitter) + file_title = ('ASAN' if asan else 'TSAN') + ' Errors Summary' + + # Execute specific errors parse + specific_errors = common_specific_errors_list( + errors_file_path=args.specific_error_file, + line_splitter=line_splitter) + print_list_to_markdown( + title=file_title, + result=specific_errors, + output_file_path=args.output_file) + + # Execute failed tests + tests_failed = failure_test_list( + log_file_path=args.log_file) + print_list_to_markdown( + title='Tests failed', + result=tests_failed, + output_file_path=args.output_file) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/utils/specific_errors_filter.sh b/.github/workflows/utils/specific_errors_filter.sh new file mode 100644 index 00000000000..c86897abfd4 --- /dev/null +++ b/.github/workflows/utils/specific_errors_filter.sh @@ -0,0 +1,23 @@ +# Copyright 2022 Proyectos y Sistemas de Mantenimiento SL (eProsima). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LINE_TO_FIND=${1} +LOG_FILE=${2} +OUTPUT_FILE=${3} + +# Store in OUTPUT_FILE lines in LOG_FILE that contain substring LINE_TO_FIND +# It is done with grep as it is able to do it in a very efficient way, while python +# requires a much longer process +# || true works in case there are no matches +grep "${LINE_TO_FIND}" ${LOG_FILE} > ${OUTPUT_FILE} || true; From 64c41839ea4b22aa54a2413307ff2d21d28a9ad6 Mon Sep 17 00:00:00 2001 From: jparisu Date: Fri, 11 Nov 2022 12:48:48 +0100 Subject: [PATCH 2/5] add py dependencies Signed-off-by: jparisu --- .github/actions/install-python-packages/action.yml | 5 ++++- .github/workflows/asan/asan_colcon.meta | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/install-python-packages/action.yml b/.github/actions/install-python-packages/action.yml index 4db9bdd1d40..837107a8f67 100644 --- a/.github/actions/install-python-packages/action.yml +++ b/.github/actions/install-python-packages/action.yml @@ -11,5 +11,8 @@ runs: vcstool \ setuptools \ gcovr \ - tomark + tomark \ + xmltodict \ + jsondiff \ + pandas shell: bash diff --git a/.github/workflows/asan/asan_colcon.meta b/.github/workflows/asan/asan_colcon.meta index 103c274a39a..6ce812b06ed 100644 --- a/.github/workflows/asan/asan_colcon.meta +++ b/.github/workflows/asan/asan_colcon.meta @@ -25,9 +25,6 @@ [ "-DCMAKE_BUILD_TYPE=Debug", "-DSANITIZER=Address", - - "-D__TODO__='Following line should be remove'", - "-DCMAKE_CXX_FLAGS='-fsanitize=address'", ] } } From c9c10c55dc4ec4a50be26c7c7cc6f54a6e416614 Mon Sep 17 00:00:00 2001 From: jparisu Date: Mon, 14 Nov 2022 12:23:10 +0100 Subject: [PATCH 3/5] adjust timeout and make report ASAN fail only if asan errors Signed-off-by: jparisu --- .github/workflows/sanitizer-tests.yaml | 4 +++- .github/workflows/utils/log_parser.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sanitizer-tests.yaml b/.github/workflows/sanitizer-tests.yaml index 3c0791debe5..c950d6cec62 100644 --- a/.github/workflows/sanitizer-tests.yaml +++ b/.github/workflows/sanitizer-tests.yaml @@ -52,7 +52,8 @@ jobs: --return-code-on-test-failure \ --ctest-args \ --label-exclude xfail \ - --timeout 60 + --timeout 300 + continue-on-error: true - name: Upload Logs uses: actions/upload-artifact@v1 @@ -117,6 +118,7 @@ jobs: --ctest-args \ --label-exclude xfail \ --timeout 60 + continue-on-error: true - name: Upload Logs uses: actions/upload-artifact@v1 diff --git a/.github/workflows/utils/log_parser.py b/.github/workflows/utils/log_parser.py index 39c3dc32d30..94c32f04044 100644 --- a/.github/workflows/utils/log_parser.py +++ b/.github/workflows/utils/log_parser.py @@ -150,10 +150,12 @@ def common_specific_errors_list( errors_file_path, line_splitter).items()] + n_errors = len(result) + if len(result) == 0: result.append({'Error': 'No errors', 'Repetitions': '0'}) - return result + return result, n_errors def common_specific_errors_dict( @@ -199,7 +201,7 @@ def main(): file_title = ('ASAN' if asan else 'TSAN') + ' Errors Summary' # Execute specific errors parse - specific_errors = common_specific_errors_list( + specific_errors, n_errors = common_specific_errors_list( errors_file_path=args.specific_error_file, line_splitter=line_splitter) print_list_to_markdown( @@ -208,13 +210,15 @@ def main(): output_file_path=args.output_file) # Execute failed tests - tests_failed = failure_test_list( + tests_failed, _ = failure_test_list( log_file_path=args.log_file) print_list_to_markdown( title='Tests failed', result=tests_failed, output_file_path=args.output_file) + return n_errors + if __name__ == '__main__': main() From 02488ff93abe2c66db2597e04bc7fa06d284c0d9 Mon Sep 17 00:00:00 2001 From: jparisu Date: Mon, 14 Nov 2022 15:08:06 +0100 Subject: [PATCH 4/5] fix typos in workflow Signed-off-by: jparisu --- .github/workflows/sanitizer-tests.yaml | 2 +- .github/workflows/utils/log_parser.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sanitizer-tests.yaml b/.github/workflows/sanitizer-tests.yaml index c950d6cec62..e23dfc49c1f 100644 --- a/.github/workflows/sanitizer-tests.yaml +++ b/.github/workflows/sanitizer-tests.yaml @@ -117,7 +117,7 @@ jobs: --return-code-on-test-failure \ --ctest-args \ --label-exclude xfail \ - --timeout 60 + --timeout 300 continue-on-error: true - name: Upload Logs diff --git a/.github/workflows/utils/log_parser.py b/.github/workflows/utils/log_parser.py index 94c32f04044..80a6b4cb2bc 100644 --- a/.github/workflows/utils/log_parser.py +++ b/.github/workflows/utils/log_parser.py @@ -81,10 +81,6 @@ def failure_test_list( if (re.search('.*The following tests FAILED:.*', line)): break - # Exit if no test failed - if (not saved_lines): - return {} - failed_tests = [] for test in saved_lines: if (re.search(r'\d* - .* \(.+\)', test)): @@ -97,16 +93,18 @@ def failure_test_list( 'Type': test[test.find('(')+1:test.find(')')] })) + n_errors = len(failed_tests) + if len(failed_tests) == 0: - failed_tests.append( + failed_tests.insert( + 0, dict({ 'ID': '-1', 'Name': 'No tests failed', 'Type': 'Hooray' - }) - ) + })) - return failed_tests + return failed_tests, n_errors def _common_line_splitter( From f74043efa48f62a5c8243be53dae9140cd93a03b Mon Sep 17 00:00:00 2001 From: jparisu Date: Mon, 14 Nov 2022 16:44:53 +0100 Subject: [PATCH 5/5] new fix typos Signed-off-by: jparisu --- .github/workflows/sanitizer-tests.yaml | 2 +- .github/workflows/utils/log_parser.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sanitizer-tests.yaml b/.github/workflows/sanitizer-tests.yaml index e23dfc49c1f..011272d0eb5 100644 --- a/.github/workflows/sanitizer-tests.yaml +++ b/.github/workflows/sanitizer-tests.yaml @@ -123,7 +123,7 @@ jobs: - name: Upload Logs uses: actions/upload-artifact@v1 with: - name: asan-logs + name: asan-ds-logs path: log/ if: always() diff --git a/.github/workflows/utils/log_parser.py b/.github/workflows/utils/log_parser.py index 80a6b4cb2bc..280d604335d 100644 --- a/.github/workflows/utils/log_parser.py +++ b/.github/workflows/utils/log_parser.py @@ -84,12 +84,22 @@ def failure_test_list( failed_tests = [] for test in saved_lines: if (re.search(r'\d* - .* \(.+\)', test)): + + # Remove strange chars and break lines and separate by space split_test = test.strip().split() + + # NOTE: if there are spaces in the test failure (e.g. Subprocess Aborted) + # the parse is more difficult as the values to take from split are different + # Thus, it uses this variable to get the correct # of spaces in error name + n_spaces_in_error = len(test[test.find('(')+1:test.find(')')].split()) - 1 + + # Insert as failed test (in first place, because we are reading them inversed) + # the new test failed with the name and id taken from split line failed_tests.insert( 0, dict({ - 'ID': split_test[-4], - 'Name': split_test[-2], + 'ID': split_test[- 4 - n_spaces_in_error], + 'Name': split_test[- 2 - n_spaces_in_error], 'Type': test[test.find('(')+1:test.find(')')] }))