diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 7c704e36be9b..d81df7280b90 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -665,6 +665,10 @@ end_tc(Mod, Fun, Args) -> %% Have to keep end_tc/3 for backwards compatibility issues end_tc(Mod, Fun, Args, '$end_tc_dummy'). end_tc(?MODULE,error_in_suite,{Result,[Args]},Return) -> + case proplists:get_value(force_failed, Args) of + undefined -> ok; + _ -> add_to_stats(failed) + end, %% this clause gets called if CT has encountered a suite that %% can't be executed FinalNotify = @@ -1221,9 +1225,9 @@ get_all(Mod, ConfTests) -> expand_tests(Mod, Tests) catch throw:{error,Error} -> - [{?MODULE,error_in_suite,[[{error,Error}]]}]; + [{?MODULE,error_in_suite,[[{error,Error},{force_failed,true}]]}]; _:Error:S -> - [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] + [{?MODULE,error_in_suite,[[{error,{Error,S}},{force_failed,true}]]}] end; Skip = {skip,_Reason} -> Skip; diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl index 4c3d279a826e..f1d783fc8da5 100644 --- a/lib/common_test/test/ct_misc_1_SUITE.erl +++ b/lib/common_test/test/ct_misc_1_SUITE.erl @@ -60,7 +60,7 @@ end_per_testcase(TestCase, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [beam_me_up, {group,parse_table}]. + [beam_me_up, {group,parse_table}, groups_bad_1]. groups() -> [{parse_table,[parallel], @@ -169,6 +169,29 @@ parse_table_one_column_multiline(Config) when is_list(Config) -> String = ["|test|","|test","value|"], {{"test"},[{"test\nvalue"}]} = ct:parse_table(String). + + +%%%----------------------------------------------------------------- +%%% + +groups_bad_1(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "bad_groups_SUITE"), + {Opts,ERPid} = setup([{suite,Suite}, + {label,groups_bad_1}], Config), + + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(bad_groups, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = test_events(groups_bad_1), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -226,4 +249,32 @@ test_events(beam_me_up) -> {?eh,tc_done,{beam_2_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} + ]; + +test_events(groups_bad_1) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,0,0}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error, + 'Invalid reference to group unexist in bad_groups_SUITE:all/0'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,0,0}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error, + 'Invalid reference to group unexist in bad_groups_SUITE:all/0'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_misc_1_SUITE_data/bad_groups_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE_data/bad_groups_SUITE.erl new file mode 100644 index 000000000000..8c5180ddf7a1 --- /dev/null +++ b/lib/common_test/test/ct_misc_1_SUITE_data/bad_groups_SUITE.erl @@ -0,0 +1,37 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% + + +-module(bad_groups_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +all() -> + [ + {group, exist}, + {group, unexist} + ]. + +groups() -> + [ + {exist, [], [test1]} + ]. + +test1(_) -> + ok.