vxx2 / cmd / tools / vtest-self.v
448 lines · 440 sloc · 16.63 KB · b0cf6a55ead9842330509ac394064d5ecc9a50bd
Raw
1module main
2
3import os
4import testing
5
6struct Config {
7 run_just_essential bool = '${os.getenv('VTEST_JUST_ESSENTIAL')}${os.getenv('VTEST_SANDBOXED_PACKAGING')}' != ''
8 is_musl_ci bool = os.getenv('V_CI_MUSL') != ''
9 is_ubuntu_musl_ci bool = os.getenv('V_CI_UBUNTU_MUSL') != ''
10 is_sandboxed_packaging bool = os.getenv('VTEST_SANDBOXED_PACKAGING') != ''
11 github_job string = os.getenv('GITHUB_JOB')
12mut:
13 test_dirs []string = ['cmd', 'vlib']
14 is_asan_compiler bool
15 is_msan_compiler bool
16 is_ubsan_compiler bool
17 // Options relating to the v command itself (passed in the prefix) `v [...args] test-self`.
18 werror bool
19 sanitize_memory bool
20 sanitize_address bool
21 sanitize_undefined bool
22}
23
24const vroot = os.dir(os.real_path(os.getenv_opt('VEXE') or { @VEXE }))
25
26const temporarily_disabled_self_test_vlib_dirs = ['v2_toberemoved', 'v3']
27
28const essential_list = [
29 'cmd/tools/vvet/vet_test.v',
30 'cmd/tools/vdoc/document/doc_test.v',
31 'vlib/arrays/arrays_test.v',
32 'vlib/bitfield/bitfield_test.v',
33 //
34 'vlib/builtin/int_test.v',
35 'vlib/builtin/array_test.v',
36 'vlib/builtin/float_test.v',
37 'vlib/builtin/byte_test.v',
38 'vlib/builtin/rune_test.v',
39 'vlib/builtin/builtin_test.c.v',
40 'vlib/builtin/map_of_floats_test.v',
41 'vlib/builtin/string_int_test.v',
42 'vlib/builtin/utf8_test.v',
43 'vlib/builtin/map_test.v',
44 'vlib/builtin/string_test.v',
45 'vlib/builtin/sorting_test.v',
46 'vlib/builtin/gated_array_string_test.v',
47 'vlib/builtin/isnil_test.v',
48 'vlib/builtin/string_match_glob_test.v',
49 'vlib/builtin/string_strip_margin_test.v',
50 //
51 'vlib/cli/command_test.v',
52 'vlib/crypto/md5/md5_test.v',
53 'vlib/dl/dl_test.v',
54 'vlib/encoding/base64/base64_test.v',
55 'vlib/encoding/utf8/validate/encoding_utf8_test.v',
56 'vlib/encoding/utf8/utf8_util_test.v',
57 'vlib/flag/flag_test.v',
58 'vlib/json/tests/json_decode_test.v',
59 'vlib/math/math_test.v',
60 'vlib/net/tcp_test.v',
61 'vlib/net/http/http_test.v',
62 'vlib/net/http/server_test.v',
63 'vlib/net/http/request_test.v',
64 'vlib/io/io_test.v',
65 'vlib/io/os_file_reader_test.v',
66 'vlib/os/process_test.v',
67 'vlib/os/file_test.v',
68 'vlib/os/notify/notify_test.c.v',
69 'vlib/os/filepath_test.v',
70 'vlib/os/environment_test.v',
71 'vlib/os/glob_test.v',
72 'vlib/os/os_test.c.v',
73 'vlib/rand/random_numbers_test.v',
74 'vlib/rand/wyrand/wyrand_test.v',
75 'vlib/runtime/runtime_test.v',
76 'vlib/semver/semver_test.v',
77 'vlib/sync/stdatomic/atomic_test.v',
78 'vlib/sync/thread_test.v',
79 'vlib/sync/waitgroup_test.v',
80 'vlib/sync/pool/pool_test.v',
81 'vlib/strings/builder_test.v',
82 'vlib/strconv/atof_test.c.v',
83 'vlib/strconv/atoi_test.v',
84 'vlib/strconv/f32_f64_to_string_test.v',
85 'vlib/strconv/format_test.v',
86 'vlib/strconv/number_to_base_test.v',
87 'vlib/time/time_test.v',
88 'vlib/toml/tests/toml_test.v',
89 'vlib/v/compiler_errors_test.v',
90 'vlib/v/fmt/fmt_keep_test.v',
91 'vlib/v/fmt/fmt_test.v',
92 'vlib/v/gen/c/coutput_test.v',
93 'vlib/v/gen/js/program_test.v',
94 'vlib/v/pkgconfig/pkgconfig_test.v',
95 'vlib/v/slow_tests/inout/compiler_test.v',
96 'vlib/x/json2/tests/json2_test.v',
97]
98const skip_with_fsanitize_memory = [
99 'do_not_remove',
100 'cmd/tools/vpm/dependency_test.v', // known flaky, for fsanitize_memory, due to using mbedtls
101 'cmd/tools/vpm/install_test.v', // known flaky, for fsanitize_memory, due to using mbedtls
102 'cmd/tools/vpm/install_version_input_test.v', // known flaky, for fsanitize_memory, due to using mbedtls
103 'cmd/tools/vpm/install_version_test.v', // known flaky, for fsanitize_memory, due to using mbedtls
104 'cmd/tools/vpm/update_test.v', // known flaky, for fsanitize_memory, due to using mbedtls
105 'vlib/net/tcp_simple_client_server_test.v',
106 'vlib/net/http/cookie_test.v',
107 'vlib/net/http/http_test.v',
108 'vlib/net/http/http_proxy_test.v',
109 'vlib/net/http/status_test.v',
110 'vlib/net/http/header_test.v',
111 'vlib/net/http/server_test.v',
112 'vlib/net/http/transport_test.v', // mbedtls TLS 1.3 handshake trips MSan in thirdparty bignum code
113 'vlib/net/mbedtls/mbedtls_head_with_content_length_test.v',
114 'vlib/net/ssl/ssl_read_all_test.v',
115 'vlib/net/udp_test.v',
116 'vlib/net/tcp_test.v',
117 'vlib/orm/orm_test.v',
118 'vlib/orm/orm_sql_or_blocks_test.v',
119 'vlib/orm/orm_create_and_drop_test.v',
120 'vlib/orm/orm_dynamic_test.v',
121 'vlib/orm/orm_insert_test.v',
122 'vlib/orm/orm_insert_reserved_name_test.v',
123 'vlib/orm/orm_sum_type_insert_test.v',
124 'vlib/orm/orm_fn_calls_test.v',
125 'vlib/orm/orm_last_id_test.v',
126 'vlib/orm/orm_string_interpolation_in_where_test.v',
127 'vlib/orm/orm_interface_test.v',
128 'vlib/orm/orm_mut_db_test.v',
129 'vlib/orm/orm_null_test.v',
130 'vlib/orm/orm_result_test.v',
131 'vlib/orm/orm_custom_operators_test.v',
132 'vlib/orm/orm_fk_test.v',
133 'vlib/orm/orm_nested_struct_test.v',
134 'vlib/orm/orm_references_test.v',
135 'vlib/orm/orm_option_array_test.v',
136 'vlib/orm/orm_option_time_test.v',
137 'vlib/orm/orm_order_by_custom_field_test.v',
138 'vlib/orm/orm_serial_attribute_test.v',
139 'vlib/orm/orm_option_subselect_test.v',
140 'vlib/orm/orm_save_test.v',
141 'vlib/orm/orm_upsert_test.v',
142 'vlib/orm/orm_func_test.v',
143 'vlib/db/sqlite/sqlite_test.v',
144 'vlib/db/sqlite/sqlite_orm_test.v',
145 'vlib/db/sqlite/sqlite_comptime_field_test.v',
146 'vlib/db/sqlite/parent_child_test.v',
147 'vlib/db/sqlite/sqlite_vfs_lowlevel_test.v',
148 'vlib/db/sqlite/sqlite_f32_test.v',
149 'vlib/v/tests/orm_enum_test.v',
150 'vlib/v/tests/orm_sub_struct_test.v',
151 'vlib/v/tests/orm_sub_array_struct_test.v',
152 'vlib/v/tests/orm_generic_struct_select_test.v',
153 'vlib/v/tests/orm_joined_tables_select_test.v',
154 'vlib/v/tests/orm_if_expr_value_test.v',
155 'vlib/v/tests/orm_option_field_issue_18333_test.v',
156 'vlib/v/tests/sql_statement_inside_fn_call_test.v',
157 'vlib/v/tests/orm_stmt_wrong_return_checking_test.v',
158 'vlib/v/tests/orm_table_name_test.v',
159 'vlib/v/tests/orm_array_field_test.v',
160 'vlib/v/tests/orm_bulk_insert_update_test.v', // MSan flags uninstrumented sqlite3BtreeOpen in libsqlite3.so
161 'vlib/v/tests/orm_handle_error_for_select_from_not_created_table_test.v',
162 'vlib/v/tests/orm_create_several_tables_test.v',
163 'vlib/v/tests/orm_update_test.v',
164 'vlib/v/tests/orm_or_test.v',
165 'vlib/net/http/request_test.v',
166 'vlib/net/http/response_test.v',
167 'vlib/net/websocket/websocket_test.v',
168 'vlib/net/smtp/smtp_test.v',
169 'vlib/v/tests/websocket_logger_interface_should_compile_test.v',
170 'vlib/v/tests/fns/fn_literal_type_test.v',
171 'vlib/v/tests/unions/union_implementing_interface_test.v',
172 'vlib/x/async/timeout_test.v', // spawn + channels + IError default field; tripped by MSan padding tracking
173 'vlib/x/sessions/tests/db_store_test.v',
174]
175const skip_with_fsanitize_address = [
176 'do_not_remove',
177 'vlib/compress/zstd/zstd_test.v', // ASan reports leaks from zstd library
178 'vlib/crypto/argon2/argon2_test.v', // ASan flags large alloc on test setup
179 'vlib/gg/text_rendering_test.v', // depends on freetype/font assets not available under sanitize CI
180 'vlib/json/tests/json_decode_with_sumtype_test.v', // ASan flake on sumtype decode buffer reuse
181 'vlib/net/mbedtls/mbedtls_read_timeout_test.v', // network timing test, ASan-incompatible
182 'vlib/net/websocket/websocket_test.v',
183 'vlib/orm/orm_create_and_drop_test.v',
184 'vlib/orm/orm_insert_test.v',
185 'vlib/orm/orm_insert_reserved_name_test.v',
186 'vlib/orm/orm_sum_type_insert_test.v',
187 'vlib/orm/orm_references_test.v',
188 'vlib/v/tests/websocket_client_default_read_timeout_test.v', // network timing
189 'vlib/v/tests/websocket_logger_interface_should_compile_test.v',
190 'vlib/v/tests/orm_enum_test.v',
191 'vlib/v/tests/orm_sub_array_struct_test.v',
192 'vlib/v/tests/orm_handle_error_for_select_from_not_created_table_test.v',
193 'vlib/v/tests/orm_create_several_tables_test.v',
194 'vlib/v/tests/orm_update_test.v',
195 'vlib/v/tests/orm_or_test.v',
196 'vlib/v/tests/shared_library_system_link_test.v', // ASan keeps Boehm GC symbols visible, breaking the export-symbol assertion
197 'vlib/veb/sse/sse_test.v', // long-lived event stream + sockets, ASan flake
198 'vlib/v2_toberemoved/gen/cleanc/flag_enum_codegen_test.v', // v2 self-host, ASan-incompatible
199 'vlib/v2_toberemoved/gen/x64/x64_backend_runtime_regression_manual_test.v', // V2 x64 backend runtime regression manual tests, ASan-incompatible
200]
201const skip_with_fsanitize_undefined = [
202 'do_not_remove',
203 'vlib/crypto/argon2/argon2_test.v', // UBSan flags shift in argon2 reference impl
204 'vlib/orm/orm_create_and_drop_test.v',
205 'vlib/orm/orm_insert_test.v',
206 'vlib/orm/orm_insert_reserved_name_test.v',
207 'vlib/orm/orm_sum_type_insert_test.v',
208 'vlib/orm/orm_references_test.v',
209 'vlib/v/tests/orm_enum_test.v',
210 'vlib/v/tests/orm_sub_array_struct_test.v',
211 'vlib/v/tests/orm_handle_error_for_select_from_not_created_table_test.v',
212 'vlib/v/tests/orm_create_several_tables_test.v',
213 'vlib/v/tests/orm_update_test.v',
214 'vlib/v/tests/orm_or_test.v',
215 'vlib/v/tests/project_with_cpp_code/compiling_cpp_files_with_a_cplusplus_compiler_test.c.v', // fails compilation with: undefined reference to vtable for __cxxabiv1::__function_type_info'
216 'vlib/v/tests/shared_library_system_link_test.v', // UBSan keeps Boehm GC symbols visible, breaking the export-symbol assertion
217 'vlib/v2_toberemoved/gen/cleanc/flag_enum_codegen_test.v', // v2 self-host, UBSan-incompatible
218 'vlib/v2_toberemoved/gen/x64/x64_backend_runtime_regression_manual_test.v', // V2 x64 backend runtime regression manual tests, UBSan-incompatible
219 'vlib/v2_toberemoved/transformer/transformer_test.v', // v2 transformer, UBSan-incompatible
220 'vlib/yaml/yaml_conformance_test.v', // upstream libyaml-style integer overflow flagged by UBSan
221]
222const skip_on_ubuntu_musl = [
223 'do_not_remove',
224 'vlib/arrays/parallel/parallel_test.v',
225 'vlib/builtin/js/array_test.js.v',
226 'vlib/db/pg_sqlite_consistency_test.v', // pg + sqlite dev headers pull in glibc-only sys/cdefs.h on musl-gcc
227 'vlib/db/sqlite/sqlite_test.v',
228 'vlib/db/sqlite/sqlite_orm_test.v',
229 'vlib/db/sqlite/sqlite_comptime_field_test.v',
230 'vlib/db/sqlite/sqlite_vfs_lowlevel_test.v',
231 'vlib/db/sqlite/parent_child_test.v',
232 'vlib/db/sqlite/sqlite_f32_test.v',
233 'vlib/gg/draw_rect_empty_test.v', // sokol.sapp needs X11/Xlib.h, not installed in the musl Docker image
234 'vlib/gg/text_rendering_test.v',
235 'vlib/orm/orm_test.v',
236 'vlib/orm/orm_sql_or_blocks_test.v',
237 'vlib/orm/orm_create_and_drop_test.v',
238 'vlib/orm/orm_dynamic_test.v',
239 'vlib/orm/orm_insert_test.v',
240 'vlib/orm/orm_insert_reserved_name_test.v',
241 'vlib/orm/orm_sum_type_insert_test.v',
242 'vlib/orm/orm_fn_calls_test.v',
243 'vlib/orm/orm_null_test.v',
244 'vlib/orm/orm_last_id_test.v',
245 'vlib/orm/orm_string_interpolation_in_where_test.v',
246 'vlib/orm/orm_interface_test.v',
247 'vlib/orm/orm_mut_db_test.v',
248 'vlib/orm/orm_result_test.v',
249 'vlib/orm/orm_custom_operators_test.v',
250 'vlib/orm/orm_fk_test.v',
251 'vlib/orm/orm_nested_struct_test.v',
252 'vlib/orm/orm_references_test.v',
253 'vlib/orm/orm_option_array_test.v',
254 'vlib/orm/orm_option_time_test.v',
255 'vlib/orm/orm_order_by_custom_field_test.v',
256 'vlib/orm/orm_serial_attribute_test.v',
257 'vlib/orm/orm_option_subselect_test.v',
258 'vlib/orm/orm_func_test.v',
259 'vlib/orm/orm_where_in_test.v',
260 'vlib/sokol/gfx/gfx_test.v', // sokol_app.h needs GL/gl.h, not installed in the musl Docker image
261 'vlib/v/gen/c/sql_assert_temp_var_test.v', // sqlite header dependency pulls in glibc sys/cdefs.h on musl-gcc
262 'vlib/v/tests/concurrency/shared_nested_lock_runtime_test.v', // nested shared lock times out on the musl image
263 'vlib/v/tests/orm_bulk_insert_update_test.v',
264 'vlib/v/tests/orm_enum_test.v',
265 'vlib/v/tests/orm_if_expr_value_test.v',
266 'vlib/v/tests/orm_sub_struct_test.v',
267 'vlib/v/tests/orm_sub_array_struct_test.v',
268 'vlib/v/tests/orm_joined_tables_select_test.v',
269 'vlib/v/tests/orm_stmt_wrong_return_checking_test.v',
270 'vlib/v/tests/orm_table_name_test.v',
271 'vlib/v/tests/orm_array_field_test.v',
272 'vlib/v/tests/orm_handle_error_for_select_from_not_created_table_test.v',
273 'vlib/v/tests/orm_create_several_tables_test.v',
274 'vlib/v/tests/orm_update_test.v',
275 'vlib/v/tests/orm_or_test.v',
276 'vlib/v/tests/shared_library_system_link_test.v', // musl libc.so is not a regular DSO; -lc shared link fails with hidden atexit
277 'vlib/v/tests/sql_statement_inside_fn_call_test.v',
278 'vlib/v/tests/websocket_logger_interface_should_compile_test.v',
279 'vlib/v/tests/fns/fn_literal_type_test.v',
280 'vlib/clipboard/clipboard_test.v',
281 'vlib/net/http/request_test.v',
282 'vlib/net/http/http_proxy_test.v', // pulls in libssl headers that include glibc-only sys/cdefs.h
283 'vlib/net/websocket/websocket_test.v',
284 'vlib/net/http/header_test.v',
285 'vlib/net/http/server_test.v',
286 'vlib/net/http/response_test.v',
287 'vlib/net/smtp/smtp_test.v',
288 'vlib/net/http/cookie_test.v',
289 'vlib/net/http/status_test.v',
290 'vlib/x/sessions/tests/db_store_test.v',
291 'vlib/veb/tests/veb_app_test.v',
292 'vlib/ncurses/ncurses_test.v',
293 'vlib/v/tests/fixed_array_update_c_struct_alias_test.v',
294 'vlib/x/crypto/mldsa/mldsa_test.v',
295]
296
297fn Config.init(vargs []string, targs []string) !Config {
298 mut cfg := Config{}
299 for arg in vargs {
300 match arg {
301 '-Werror', '-cstrict' { cfg.werror = true }
302 else {}
303 }
304
305 // Match -fsanitize prefix, since CI may pass combined values like
306 // `-fsanitize=address,pointer-compare,pointer-subtract`.
307 if arg.starts_with('-fsanitize=') {
308 vals := arg.all_after('=').split(',')
309 if 'memory' in vals {
310 cfg.sanitize_memory = true
311 }
312 if 'address' in vals {
313 cfg.sanitize_address = true
314 }
315 if 'undefined' in vals {
316 cfg.sanitize_undefined = true
317 }
318 }
319 }
320 if targs.len == 0 {
321 return cfg
322 }
323 mut tdirs := []string{}
324 mut errs := []string{}
325 for arg in targs {
326 match arg {
327 '-asan-compiler', '--asan-compiler' {
328 cfg.is_asan_compiler = true
329 }
330 '-msan-compiler', '--msan-compiler' {
331 cfg.is_msan_compiler = true
332 }
333 '-ubsan-compiler', '--ubsan-compiler' {
334 cfg.is_ubsan_compiler = true
335 }
336 else {
337 if arg.starts_with('-') {
338 errs << 'error: unknown flag `${arg}`'
339 continue
340 }
341 if !os.is_dir(os.join_path(vroot, arg)) {
342 errs << 'error: failed to find directory `${arg}`'
343 continue
344 }
345 tdirs << arg
346 }
347 }
348 }
349 if errs.len > 0 {
350 return error(errs.join_lines())
351 }
352 if tdirs.len > 0 {
353 cfg.test_dirs = tdirs
354 }
355 return cfg
356}
357
358fn main() {
359 unbuffer_stdout()
360 os.chdir(vroot)!
361 args_idx := os.args.index('test-self')
362 if args_idx < 0 {
363 eprintln('vtest-self: could not find `test-self` in os.args: ${os.args}')
364 exit(1)
365 }
366 vargs := os.args[1..args_idx]
367 targs := os.args#[args_idx + 1..]
368 cfg := Config.init(vargs, targs) or {
369 eprintln(err)
370 exit(1)
371 }
372 // dump(cfg)
373 title := 'testing: ${cfg.test_dirs.join(', ')}'
374 mut tpaths := map[string]bool{}
375 mut tpaths_ref := &tpaths
376 for dir in cfg.test_dirs {
377 os.walk(os.join_path(vroot, dir), fn [mut tpaths_ref] (p string) {
378 if p.ends_with('_test.v') || p.ends_with('_test.c.v')
379 || (testing.is_node_present && p.ends_with('_test.js.v')) {
380 unsafe {
381 tpaths_ref[p] = true
382 }
383 }
384 })
385 }
386 mut all_test_files := tpaths.keys()
387 if cfg.run_just_essential {
388 all_test_files = essential_list.map(os.join_path(vroot, it))
389 }
390 mut tsession := testing.new_test_session(vargs.join(' '), true)
391 tsession.exec_mode = .compile_and_run
392 tsession.files << all_test_files.filter(!it.contains('testdata' + os.path_separator))
393 // v2 and v3 have their own drivers and are still under heavy development,
394 // so their tests are excluded from `v test-self`.
395 for test_dir in temporarily_disabled_self_test_vlib_dirs {
396 dir_fragment := '${os.path_separator}vlib${os.path_separator}${test_dir}${os.path_separator}'
397 tsession.skip_files << tsession.files.filter(it.contains(dir_fragment))
398 }
399 if cfg.werror {
400 tsession.custom_defines << 'self_werror'
401 }
402 if cfg.sanitize_memory {
403 tsession.skip_files << skip_with_fsanitize_memory
404 tsession.custom_defines << 'self_sanitize_memory'
405 }
406 if cfg.sanitize_address {
407 tsession.skip_files << skip_with_fsanitize_address
408 tsession.custom_defines << 'self_sanitize_address'
409 }
410 if cfg.sanitize_undefined {
411 tsession.skip_files << skip_with_fsanitize_undefined
412 tsession.custom_defines << 'self_sanitize_undefined'
413 }
414 if cfg.is_asan_compiler {
415 tsession.custom_defines << 'self_asan_compiler'
416 }
417 if cfg.is_msan_compiler {
418 tsession.custom_defines << 'self_msan_compiler'
419 }
420 if cfg.is_ubsan_compiler {
421 tsession.custom_defines << 'self_ubsan_compiler'
422 }
423 if cfg.is_sandboxed_packaging {
424 tsession.custom_defines << 'self_sandboxed_packaging'
425 }
426 if cfg.is_ubuntu_musl_ci {
427 tsession.skip_files << skip_on_ubuntu_musl
428 tsession.custom_defines << 'self_ubuntu_musl_ci'
429 }
430 // dump(tsession.skip_files)
431 mut unavailable_files := tsession.files.filter(!os.exists(it))
432 unavailable_files << tsession.skip_files.filter(it != 'do_not_remove' && !os.exists(it))
433 if unavailable_files.len > 0 {
434 for f in unavailable_files {
435 eprintln('error: failed to find file: ${f}')
436 }
437 exit(1)
438 }
439 tsession.skip_files = tsession.skip_files.map(os.abs_path)
440 tsession.session_start(title)
441 tsession.test()
442 tsession.session_stop(title)
443 flush_stdout()
444 if tsession.benchmark.nfail > 0 {
445 eprintln('\nError: failed ${tsession.benchmark.nfail} times.\n')
446 exit(1)
447 }
448}
449