Merge pull request #7487 from hizel/ct-error-in-suite

ct: update auto skip stats when error in suite handled
OTP-18728

lib/common_test/src/ct_framework.erl

@@ -665,6 +665,10 @@ end_tc(Mod, Fun, Args) ->
     %% Have to keep end_tc/3 for backwards compatibility issues
     end_tc(Mod, Fun, Args, '$end_tc_dummy').
 end_tc(?MODULE,error_in_suite,{Result,[Args]},Return) ->
+    case proplists:get_value(force_failed, Args) of
+		undefined -> ok;
+		_ -> add_to_stats(failed)
+    end,
     %% this clause gets called if CT has encountered a suite that
     %% can't be executed
     FinalNotify =
@@ -1221,9 +1225,9 @@ get_all(Mod, ConfTests) ->
                     expand_tests(Mod, Tests)
             catch
                 throw:{error,Error} ->
-                    [{?MODULE,error_in_suite,[[{error,Error}]]}];
+                    [{?MODULE,error_in_suite,[[{error,Error},{force_failed,true}]]}];
                 _:Error:S ->
-                    [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+                    [{?MODULE,error_in_suite,[[{error,{Error,S}},{force_failed,true}]]}]
             end;
         Skip = {skip,_Reason} ->
 	    Skip;

lib/common_test/test/ct_misc_1_SUITE.erl

@@ -60,7 +60,7 @@ end_per_testcase(TestCase, Config) ->
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [beam_me_up, {group,parse_table}].
+    [beam_me_up, {group,parse_table}, groups_bad_1].
 
 groups() -> 
     [{parse_table,[parallel], 
@@ -169,6 +169,29 @@ parse_table_one_column_multiline(Config) when is_list(Config) ->
     String = ["|test|","|test","value|"],
 
     {{"test"},[{"test\nvalue"}]} = ct:parse_table(String).
+
+
+%%%-----------------------------------------------------------------
+%%%
+
+groups_bad_1(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "bad_groups_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},
+      {label,groups_bad_1}], Config),
+
+  ok = ct_test_support:run(Opts, Config),
+  Events = ct_test_support:get_events(ERPid, Config),
+
+  ct_test_support:log_events(bad_groups,
+           reformat(Events, ?eh),
+           ?config(priv_dir, Config),
+           Opts),
+
+  TestEvents = test_events(groups_bad_1),
+  ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+
 
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
@@ -226,4 +249,32 @@ test_events(beam_me_up) ->
      {?eh,tc_done,{beam_2_SUITE,end_per_suite,ok}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
+    ];
+
+test_events(groups_bad_1) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,0,0}},
+     {?eh,tc_start,{ct_framework,error_in_suite}},
+     {?eh,test_stats,{0,1,{0,0}}},
+     {?eh,tc_done,
+        {ct_framework,error_in_suite,
+            {failed,
+                {error,
+                    'Invalid reference to group unexist in bad_groups_SUITE:all/0'}}}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]},
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,0,0}},
+     {?eh,tc_start,{ct_framework,error_in_suite}},
+     {?eh,test_stats,{0,1,{0,0}}},
+     {?eh,tc_done,
+        {ct_framework,error_in_suite,
+            {failed,
+                {error,
+                    'Invalid reference to group unexist in bad_groups_SUITE:all/0'}}}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
     ].

lib/common_test/test/ct_misc_1_SUITE_data/bad_groups_SUITE.erl

@@ -0,0 +1,37 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+
+
+-module(bad_groups_SUITE).
+-compile(export_all).
+-compile(nowarn_export_all).
+
+all() ->
+  [
+    {group, exist},
+    {group, unexist}
+  ].
+
+groups() ->
+  [
+   {exist, [], [test1]}
+  ].
+
+test1(_) ->
+  ok.