From 5277ce7dce4ad9de8afe86dd02f79627800ecf14 Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Sat, 30 Apr 2022 13:27:50 +0300 Subject: [PATCH] tests: support for marking _test.v files with `// vtest flaky: true`, to reduce false positives from the CI --- cmd/tools/modules/testing/common.v | 18 ++++++++++++++++-- vlib/context/cancel_test.v | 1 + vlib/net/tcp_test.v | 1 + vlib/os/notify/notify_test.v | 1 + vlib/os/process_test.v | 1 + vlib/sync/channel_select_6_test.v | 1 + vlib/sync/channel_select_test.v | 1 + vlib/v/live/live_test.v | 1 + vlib/v/pref/options_test.v | 1 + vlib/v/tests/go_anon_fn_variable_call_test.v | 1 + vlib/v/tests/go_array_wait_test.v | 1 + vlib/v/tests/go_wait_option_test.v | 1 + vlib/v/tests/init_global_test.v | 1 + vlib/v/tests/keep_args_alive_test.v | 1 + vlib/v/tests/semaphore_timed_test.v | 1 + 15 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cmd/tools/modules/testing/common.v b/cmd/tools/modules/testing/common.v index 93d90c1e4..9ac7d6f91 100644 --- a/cmd/tools/modules/testing/common.v +++ b/cmd/tools/modules/testing/common.v @@ -19,6 +19,8 @@ pub const hide_oks = os.getenv('VTEST_HIDE_OK') == '1' pub const fail_fast = os.getenv('VTEST_FAIL_FAST') == '1' +pub const fail_flaky = os.getenv('VTEST_FAIL_FLAKY') == '1' + pub const test_only = os.getenv('VTEST_ONLY').split_any(',') pub const test_only_fn = os.getenv('VTEST_ONLY_FN').split_any(',') @@ -362,7 +364,7 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr { details := get_test_details(file) os.setenv('VTEST_RETRY_MAX', '$details.retry', true) for retry := 1; retry <= details.retry; retry++ { - ts.append_message(.info, ' retrying $retry/$details.retry of $relative_file ...') + ts.append_message(.info, ' [stats] retrying $retry/$details.retry of $relative_file ; known flaky: $details.flaky ...') os.setenv('VTEST_RETRY', '$retry', true) status = os.system(cmd) if status == 0 { @@ -372,6 +374,10 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr { } time.sleep(500 * time.millisecond) } + if details.flaky && !testing.fail_flaky { + ts.append_message(.info, ' *FAILURE* of the known flaky test file $relative_file is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: $details.retry .') + goto test_passed_system + } ts.failed = true ts.benchmark.fail() tls_bench.fail() @@ -399,7 +405,7 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr { details := get_test_details(file) os.setenv('VTEST_RETRY_MAX', '$details.retry', true) for retry := 1; retry <= details.retry; retry++ { - ts.append_message(.info, ' retrying $retry/$details.retry of $relative_file ...') + ts.append_message(.info, ' retrying $retry/$details.retry of $relative_file ; known flaky: $details.flaky ...') os.setenv('VTEST_RETRY', '$retry', true) r = os.execute(cmd) if r.exit_code == 0 { @@ -408,6 +414,10 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr { } } } + if details.flaky && !testing.fail_flaky { + ts.append_message(.info, ' *FAILURE* of the known flaky test file $relative_file is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: $details.retry .') + goto test_passed_system + } ts.failed = true ts.benchmark.fail() tls_bench.fail() @@ -562,6 +572,7 @@ pub fn setup_new_vtmp_folder() string { pub struct TestDetails { pub mut: retry int + flaky bool // when flaky tests fail, the whole run is still considered successfull, unless VTEST_FAIL_FLAKY is 1 } pub fn get_test_details(file string) TestDetails { @@ -571,6 +582,9 @@ pub fn get_test_details(file string) TestDetails { if line.starts_with('// vtest retry:') { res.retry = line.all_after(':').trim_space().int() } + if line.starts_with('// vtest flaky:') { + res.flaky = line.all_after(':').trim_space().bool() + } } return res } diff --git a/vlib/context/cancel_test.v b/vlib/context/cancel_test.v index cd5033c34..f9d055d2d 100644 --- a/vlib/context/cancel_test.v +++ b/vlib/context/cancel_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 import context diff --git a/vlib/net/tcp_test.v b/vlib/net/tcp_test.v index 12e6f2b67..4d0a81a60 100644 --- a/vlib/net/tcp_test.v +++ b/vlib/net/tcp_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 8 import net import os diff --git a/vlib/os/notify/notify_test.v b/vlib/os/notify/notify_test.v index 4791b4585..3003ff8de 100644 --- a/vlib/os/notify/notify_test.v +++ b/vlib/os/notify/notify_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 import os import os.notify diff --git a/vlib/os/process_test.v b/vlib/os/process_test.v index d7c5436bb..a219ee8b3 100644 --- a/vlib/os/process_test.v +++ b/vlib/os/process_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 import os import time diff --git a/vlib/sync/channel_select_6_test.v b/vlib/sync/channel_select_6_test.v index dc00d347d..cce63fecf 100644 --- a/vlib/sync/channel_select_6_test.v +++ b/vlib/sync/channel_select_6_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 15 // This test case runs concurrent 3 instances of `do_select` that diff --git a/vlib/sync/channel_select_test.v b/vlib/sync/channel_select_test.v index 70dce3066..21f357106 100644 --- a/vlib/sync/channel_select_test.v +++ b/vlib/sync/channel_select_test.v @@ -1,5 +1,6 @@ module sync +// vtest flaky: true // vtest retry: 6 // ATTENTION! Do not use this file as an example! diff --git a/vlib/v/live/live_test.v b/vlib/v/live/live_test.v index 86d40895d..4804d3704 100644 --- a/vlib/v/live/live_test.v +++ b/vlib/v/live/live_test.v @@ -1,6 +1,7 @@ import os import time +// vtest flaky: true // vtest retry: 4 /* diff --git a/vlib/v/pref/options_test.v b/vlib/v/pref/options_test.v index fbfd635f3..1752dffe7 100644 --- a/vlib/v/pref/options_test.v +++ b/vlib/v/pref/options_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 import os import time diff --git a/vlib/v/tests/go_anon_fn_variable_call_test.v b/vlib/v/tests/go_anon_fn_variable_call_test.v index f4cfe526c..8d48fdaee 100644 --- a/vlib/v/tests/go_anon_fn_variable_call_test.v +++ b/vlib/v/tests/go_anon_fn_variable_call_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 fn sum1(a int, b int) int { diff --git a/vlib/v/tests/go_array_wait_test.v b/vlib/v/tests/go_array_wait_test.v index cdf1686b3..fbe6a5086 100644 --- a/vlib/v/tests/go_array_wait_test.v +++ b/vlib/v/tests/go_array_wait_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 fn f(x f64) f64 { diff --git a/vlib/v/tests/go_wait_option_test.v b/vlib/v/tests/go_wait_option_test.v index 1979d8fb8..ed910f3cb 100644 --- a/vlib/v/tests/go_wait_option_test.v +++ b/vlib/v/tests/go_wait_option_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 fn f(n int) ?f64 { diff --git a/vlib/v/tests/init_global_test.v b/vlib/v/tests/init_global_test.v index 3e61844d6..a635bbaaf 100644 --- a/vlib/v/tests/init_global_test.v +++ b/vlib/v/tests/init_global_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 4 import sync diff --git a/vlib/v/tests/keep_args_alive_test.v b/vlib/v/tests/keep_args_alive_test.v index 55fd4fb9d..b1b0b3f29 100644 --- a/vlib/v/tests/keep_args_alive_test.v +++ b/vlib/v/tests/keep_args_alive_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 4 /* * To verify the effect of "[keep_args_alive]", this attribute may be commented out. diff --git a/vlib/v/tests/semaphore_timed_test.v b/vlib/v/tests/semaphore_timed_test.v index 5fd82b689..e80ef74e7 100644 --- a/vlib/v/tests/semaphore_timed_test.v +++ b/vlib/v/tests/semaphore_timed_test.v @@ -1,3 +1,4 @@ +// vtest flaky: true // vtest retry: 3 import sync import time -- 2.30.2