1 | import os |
2 | import flag |
3 | import term |
4 | import time |
5 | import v.parser |
6 | import v.ast |
7 | import v.pref |
8 | |
9 | const ( |
10 | vexe = os.real_path(os.getenv_opt('VEXE') or { @VEXE }) |
11 | vroot = os.dir(vexe) |
12 | support_color = term.can_show_color_on_stderr() && term.can_show_color_on_stdout() |
13 | ecode_timeout = 101 |
14 | ecode_memout = 102 |
15 | ecode_details = { |
16 | -1: 'worker executable not found' |
17 | 101: 'too slow' |
18 | 102: 'too memory hungry' |
19 | } |
20 | ) |
21 | |
22 | struct Context { |
23 | mut: |
24 | is_help bool |
25 | is_worker bool |
26 | is_verbose bool |
27 | is_silent bool // do not print any status/progress during processing, just failures. |
28 | is_linear bool // print linear progress log, without trying to do term cursor up + \r msg. Easier to use in a CI job |
29 | show_src bool // show the partial source, that cause the parser to panic/fault, when it happens. |
30 | timeout_ms int |
31 | myself string // path to this executable, so the supervisor can launch worker processes |
32 | all_paths []string // all files given to the supervisor process |
33 | path string // the current path, given to a worker process |
34 | cut_index int // the cut position in the source from context.path |
35 | max_index int // the maximum index (equivalent to the file content length) |
36 | // parser context in the worker processes: |
37 | table ast.Table |
38 | scope ast.Scope |
39 | pref &pref.Preferences = unsafe { nil } |
40 | period_ms int // print periodic progress |
41 | stop_print bool // stop printing the periodic progress |
42 | } |
43 | |
44 | fn main() { |
45 | mut context := process_cli_args() |
46 | if context.is_worker { |
47 | pid := os.getpid() |
48 | context.log('> worker ${pid:5} starts parsing at cut_index: ${context.cut_index:5} | ${context.path}') |
49 | // A worker's process job is to try to parse a single given file in context.path. |
50 | // It can crash/panic freely. |
51 | context.table = ast.new_table() |
52 | context.scope = &ast.Scope{ |
53 | parent: 0 |
54 | } |
55 | context.pref = &pref.Preferences{ |
56 | output_mode: .silent |
57 | } |
58 | mut source := os.read_file(context.path)! |
59 | source = source[..context.cut_index] |
60 | |
61 | spawn fn (ms int) { |
62 | time.sleep(ms * time.millisecond) |
63 | exit(ecode_timeout) |
64 | }(context.timeout_ms) |
65 | _ := parser.parse_text(source, context.path, context.table, .skip_comments, context.pref) |
66 | context.log('> worker ${pid:5} finished parsing ${context.path}') |
67 | exit(0) |
68 | } else { |
69 | // The process supervisor should NOT crash/panic, unlike the workers. |
70 | // It's job, is to: |
71 | // 1) start workers |
72 | // 2) accumulate results |
73 | // 3) produce a summary at the end |
74 | context.expand_all_paths() |
75 | mut fails := 0 |
76 | mut panics := 0 |
77 | sw := time.new_stopwatch() |
78 | for path in context.all_paths { |
79 | filesw := time.new_stopwatch() |
80 | context.start_printing() |
81 | new_fails, new_panics := context.process_whole_file_in_worker(path) |
82 | fails += new_fails |
83 | panics += new_panics |
84 | context.stop_printing() |
85 | context.info('File: ${path:-30} | new_fails: ${new_fails:5} | new_panics: ${new_panics:5} | Elapsed time: ${filesw.elapsed().milliseconds()}ms') |
86 | } |
87 | non_panics := fails - panics |
88 | context.info('Total files processed: ${context.all_paths.len:5} | Errors found: ${fails:5} | Panics: ${panics:5} | Non panics: ${non_panics:5} | Elapsed time: ${sw.elapsed().milliseconds()}ms') |
89 | if fails > 0 { |
90 | exit(1) |
91 | } |
92 | exit(0) |
93 | } |
94 | } |
95 | |
96 | fn process_cli_args() &Context { |
97 | mut context := &Context{ |
98 | pref: pref.new_preferences() |
99 | } |
100 | context.myself = os.executable() |
101 | mut fp := flag.new_flag_parser(os.args_after('test-parser')) |
102 | fp.application(os.file_name(context.myself)) |
103 | fp.version('0.0.1') |
104 | fp.description('Test the V parser, by parsing each .v file in each PATH,\n' + |
105 | 'as if it was typed character by character by the user.\n' + |
106 | 'A PATH can be either a folder, or a specific .v file.\n' + |
107 | 'Note: you *have to quote* the PATH, if it contains spaces/punctuation.') |
108 | fp.arguments_description('PATH1 PATH2 ...') |
109 | fp.skip_executable() |
110 | context.is_help = fp.bool('help', `h`, false, 'Show help/usage screen.') |
111 | context.is_verbose = fp.bool('verbose', `v`, false, 'Be more verbose.') |
112 | context.is_silent = fp.bool('silent', `S`, false, 'Do not print progress at all.') |
113 | context.is_linear = fp.bool('linear', `L`, false, 'Print linear progress log. Suitable for CI.') |
114 | context.show_src = fp.bool('show_source', `E`, false, 'Print the partial source code that caused a fault/panic in the parser.') |
115 | context.period_ms = fp.int('progress_ms', `s`, 500, 'print a status report periodically, the period is given in milliseconds.') |
116 | context.is_worker = fp.bool('worker', `w`, false, 'worker specific flag - is this a worker process, that can crash/panic.') |
117 | context.cut_index = fp.int('cut_index', `c`, 1, 'worker specific flag - cut index in the source file, everything before that will be parsed, the rest - ignored.') |
118 | context.timeout_ms = fp.int('timeout_ms', `t`, 250, 'worker specific flag - timeout in ms; a worker taking longer, will self terminate.') |
119 | context.path = fp.string('path', `p`, '', 'worker specific flag - path to the current source file, which will be parsed.') |
120 | // |
121 | if context.is_help { |
122 | println(fp.usage()) |
123 | exit(0) |
124 | } |
125 | context.all_paths = fp.finalize() or { |
126 | context.error(err.msg()) |
127 | exit(1) |
128 | } |
129 | if !context.is_worker && context.all_paths.len == 0 { |
130 | println(fp.usage()) |
131 | exit(0) |
132 | } |
133 | return context |
134 | } |
135 | |
136 | // //////////////// |
137 | fn bold(msg string) string { |
138 | if !support_color { |
139 | return msg |
140 | } |
141 | return term.bold(msg) |
142 | } |
143 | |
144 | fn red(msg string) string { |
145 | if !support_color { |
146 | return msg |
147 | } |
148 | return term.red(msg) |
149 | } |
150 | |
151 | fn yellow(msg string) string { |
152 | if !support_color { |
153 | return msg |
154 | } |
155 | return term.yellow(msg) |
156 | } |
157 | |
158 | fn (mut context Context) info(msg string) { |
159 | println(msg) |
160 | } |
161 | |
162 | fn (mut context Context) log(msg string) { |
163 | if context.is_verbose { |
164 | label := yellow('info') |
165 | ts := time.now().format_ss_micro() |
166 | eprintln('${label}: ${ts} | ${msg}') |
167 | } |
168 | } |
169 | |
170 | fn (mut context Context) error(msg string) { |
171 | label := red('error') |
172 | eprintln('${label}: ${msg}') |
173 | } |
174 | |
175 | fn (mut context Context) expand_all_paths() { |
176 | context.log('> context.all_paths before: ${context.all_paths}') |
177 | mut files := []string{} |
178 | for path in context.all_paths { |
179 | if os.is_dir(path) { |
180 | files << os.walk_ext(path, '.v') |
181 | files << os.walk_ext(path, '.vsh') |
182 | continue |
183 | } |
184 | if !path.ends_with('.v') && !path.ends_with('.vv') && !path.ends_with('.vsh') { |
185 | context.error('`v test-parser` can only be used on .v/.vv/.vsh files.\nOffending file: "${path}".') |
186 | continue |
187 | } |
188 | if !os.exists(path) { |
189 | context.error('"${path}" does not exist.') |
190 | continue |
191 | } |
192 | files << path |
193 | } |
194 | context.all_paths = files |
195 | context.log('> context.all_paths after: ${context.all_paths}') |
196 | } |
197 | |
198 | fn (mut context Context) process_whole_file_in_worker(path string) (int, int) { |
199 | context.path = path // needed for the progress bar |
200 | context.log('> context.process_whole_file_in_worker path: ${path}') |
201 | if !(os.is_file(path) && os.is_readable(path)) { |
202 | context.error('${path} is not readable') |
203 | return 1, 0 |
204 | } |
205 | source := os.read_file(path) or { '' } |
206 | if source == '' { |
207 | // an empty file is a valid .v file |
208 | return 0, 0 |
209 | } |
210 | len := source.len - 1 |
211 | mut fails := 0 |
212 | mut panics := 0 |
213 | context.max_index = len |
214 | for i in 0 .. len { |
215 | verbosity := if context.is_verbose { '-v' } else { '' } |
216 | context.cut_index = i // needed for the progress bar |
217 | cmd := '${os.quoted_path(context.myself)} ${verbosity} --worker --timeout_ms ${context.timeout_ms:5} --cut_index ${i:5} --path ${os.quoted_path(path)} ' |
218 | context.log(cmd) |
219 | mut res := os.execute(cmd) |
220 | context.log('worker exit_code: ${res.exit_code} | worker output:\n${res.output}') |
221 | if res.exit_code != 0 { |
222 | fails++ |
223 | mut is_panic := false |
224 | if res.output.contains('V panic:') { |
225 | is_panic = true |
226 | panics++ |
227 | } |
228 | part := source[..i] |
229 | line := part.count('\n') + 1 |
230 | last_line := part.all_after_last('\n') |
231 | col := last_line.len |
232 | err := if is_panic { |
233 | red('parser failure: panic') |
234 | } else { |
235 | red('parser failure: crash, ${ecode_details[res.exit_code]}') |
236 | } |
237 | path_to_line := bold('${path}:${line}:${col}:') |
238 | err_line := last_line.trim_left('\t') |
239 | println('${path_to_line} ${err}') |
240 | println('\t${line} | ${err_line}') |
241 | println('') |
242 | eprintln(res.output) |
243 | eprintln('>>> failed command: ${cmd}') |
244 | if context.show_src { |
245 | eprintln('>>> source so far:') |
246 | eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') |
247 | partial_source := source[..context.cut_index] |
248 | eprintln(partial_source) |
249 | eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') |
250 | } |
251 | } |
252 | } |
253 | return fails, panics |
254 | } |
255 | |
256 | fn (mut context Context) start_printing() { |
257 | context.stop_print = false |
258 | if !context.is_linear && !context.is_silent { |
259 | println('\n') |
260 | } |
261 | spawn context.print_periodic_status() |
262 | } |
263 | |
264 | fn (mut context Context) stop_printing() { |
265 | context.stop_print = true |
266 | time.sleep(time.millisecond * context.period_ms / 5) |
267 | } |
268 | |
269 | fn (mut context Context) print_status() { |
270 | if context.is_silent { |
271 | return |
272 | } |
273 | if context.cut_index == 1 && context.max_index == 0 { |
274 | return |
275 | } |
276 | msg := '> ${context.path:-30} | index: ${context.cut_index:5}/${context.max_index - 1:5}' |
277 | if context.is_linear { |
278 | eprintln(msg) |
279 | return |
280 | } |
281 | term.cursor_up(1) |
282 | eprint('\r ${msg}\n') |
283 | } |
284 | |
285 | fn (mut context Context) print_periodic_status() { |
286 | context.print_status() |
287 | mut printed_at_least_once := false |
288 | for !context.stop_print { |
289 | context.print_status() |
290 | for i := 0; i < 10 && !context.stop_print; i++ { |
291 | time.sleep(time.millisecond * context.period_ms / 10) |
292 | if context.cut_index > 50 && !printed_at_least_once { |
293 | context.print_status() |
294 | printed_at_least_once = true |
295 | } |
296 | } |
297 | } |
298 | context.print_status() |
299 | } |