v / cmd / tools
Raw file | 299 loc (282 sloc) | 8.69 KB | Latest commit hash 017ace6ea
1import os
2import flag
3import term
4import time
5import v.parser
6import v.ast
7import v.pref
8
9const (
10 vexe = os.real_path(os.getenv_opt('VEXE') or { @VEXE })
11 vroot = os.dir(vexe)
12 support_color = term.can_show_color_on_stderr() && term.can_show_color_on_stdout()
13 ecode_timeout = 101
14 ecode_memout = 102
15 ecode_details = {
16 -1: 'worker executable not found'
17 101: 'too slow'
18 102: 'too memory hungry'
19 }
20)
21
22struct Context {
23mut:
24 is_help bool
25 is_worker bool
26 is_verbose bool
27 is_silent bool // do not print any status/progress during processing, just failures.
28 is_linear bool // print linear progress log, without trying to do term cursor up + \r msg. Easier to use in a CI job
29 show_src bool // show the partial source, that cause the parser to panic/fault, when it happens.
30 timeout_ms int
31 myself string // path to this executable, so the supervisor can launch worker processes
32 all_paths []string // all files given to the supervisor process
33 path string // the current path, given to a worker process
34 cut_index int // the cut position in the source from context.path
35 max_index int // the maximum index (equivalent to the file content length)
36 // parser context in the worker processes:
37 table ast.Table
38 scope ast.Scope
39 pref &pref.Preferences = unsafe { nil }
40 period_ms int // print periodic progress
41 stop_print bool // stop printing the periodic progress
42}
43
44fn main() {
45 mut context := process_cli_args()
46 if context.is_worker {
47 pid := os.getpid()
48 context.log('> worker ${pid:5} starts parsing at cut_index: ${context.cut_index:5} | ${context.path}')
49 // A worker's process job is to try to parse a single given file in context.path.
50 // It can crash/panic freely.
51 context.table = ast.new_table()
52 context.scope = &ast.Scope{
53 parent: 0
54 }
55 context.pref = &pref.Preferences{
56 output_mode: .silent
57 }
58 mut source := os.read_file(context.path)!
59 source = source[..context.cut_index]
60
61 spawn fn (ms int) {
62 time.sleep(ms * time.millisecond)
63 exit(ecode_timeout)
64 }(context.timeout_ms)
65 _ := parser.parse_text(source, context.path, context.table, .skip_comments, context.pref)
66 context.log('> worker ${pid:5} finished parsing ${context.path}')
67 exit(0)
68 } else {
69 // The process supervisor should NOT crash/panic, unlike the workers.
70 // It's job, is to:
71 // 1) start workers
72 // 2) accumulate results
73 // 3) produce a summary at the end
74 context.expand_all_paths()
75 mut fails := 0
76 mut panics := 0
77 sw := time.new_stopwatch()
78 for path in context.all_paths {
79 filesw := time.new_stopwatch()
80 context.start_printing()
81 new_fails, new_panics := context.process_whole_file_in_worker(path)
82 fails += new_fails
83 panics += new_panics
84 context.stop_printing()
85 context.info('File: ${path:-30} | new_fails: ${new_fails:5} | new_panics: ${new_panics:5} | Elapsed time: ${filesw.elapsed().milliseconds()}ms')
86 }
87 non_panics := fails - panics
88 context.info('Total files processed: ${context.all_paths.len:5} | Errors found: ${fails:5} | Panics: ${panics:5} | Non panics: ${non_panics:5} | Elapsed time: ${sw.elapsed().milliseconds()}ms')
89 if fails > 0 {
90 exit(1)
91 }
92 exit(0)
93 }
94}
95
96fn process_cli_args() &Context {
97 mut context := &Context{
98 pref: pref.new_preferences()
99 }
100 context.myself = os.executable()
101 mut fp := flag.new_flag_parser(os.args_after('test-parser'))
102 fp.application(os.file_name(context.myself))
103 fp.version('0.0.1')
104 fp.description('Test the V parser, by parsing each .v file in each PATH,\n' +
105 'as if it was typed character by character by the user.\n' +
106 'A PATH can be either a folder, or a specific .v file.\n' +
107 'Note: you *have to quote* the PATH, if it contains spaces/punctuation.')
108 fp.arguments_description('PATH1 PATH2 ...')
109 fp.skip_executable()
110 context.is_help = fp.bool('help', `h`, false, 'Show help/usage screen.')
111 context.is_verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
112 context.is_silent = fp.bool('silent', `S`, false, 'Do not print progress at all.')
113 context.is_linear = fp.bool('linear', `L`, false, 'Print linear progress log. Suitable for CI.')
114 context.show_src = fp.bool('show_source', `E`, false, 'Print the partial source code that caused a fault/panic in the parser.')
115 context.period_ms = fp.int('progress_ms', `s`, 500, 'print a status report periodically, the period is given in milliseconds.')
116 context.is_worker = fp.bool('worker', `w`, false, 'worker specific flag - is this a worker process, that can crash/panic.')
117 context.cut_index = fp.int('cut_index', `c`, 1, 'worker specific flag - cut index in the source file, everything before that will be parsed, the rest - ignored.')
118 context.timeout_ms = fp.int('timeout_ms', `t`, 250, 'worker specific flag - timeout in ms; a worker taking longer, will self terminate.')
119 context.path = fp.string('path', `p`, '', 'worker specific flag - path to the current source file, which will be parsed.')
120 //
121 if context.is_help {
122 println(fp.usage())
123 exit(0)
124 }
125 context.all_paths = fp.finalize() or {
126 context.error(err.msg())
127 exit(1)
128 }
129 if !context.is_worker && context.all_paths.len == 0 {
130 println(fp.usage())
131 exit(0)
132 }
133 return context
134}
135
136// ////////////////
137fn bold(msg string) string {
138 if !support_color {
139 return msg
140 }
141 return term.bold(msg)
142}
143
144fn red(msg string) string {
145 if !support_color {
146 return msg
147 }
148 return term.red(msg)
149}
150
151fn yellow(msg string) string {
152 if !support_color {
153 return msg
154 }
155 return term.yellow(msg)
156}
157
158fn (mut context Context) info(msg string) {
159 println(msg)
160}
161
162fn (mut context Context) log(msg string) {
163 if context.is_verbose {
164 label := yellow('info')
165 ts := time.now().format_ss_micro()
166 eprintln('${label}: ${ts} | ${msg}')
167 }
168}
169
170fn (mut context Context) error(msg string) {
171 label := red('error')
172 eprintln('${label}: ${msg}')
173}
174
175fn (mut context Context) expand_all_paths() {
176 context.log('> context.all_paths before: ${context.all_paths}')
177 mut files := []string{}
178 for path in context.all_paths {
179 if os.is_dir(path) {
180 files << os.walk_ext(path, '.v')
181 files << os.walk_ext(path, '.vsh')
182 continue
183 }
184 if !path.ends_with('.v') && !path.ends_with('.vv') && !path.ends_with('.vsh') {
185 context.error('`v test-parser` can only be used on .v/.vv/.vsh files.\nOffending file: "${path}".')
186 continue
187 }
188 if !os.exists(path) {
189 context.error('"${path}" does not exist.')
190 continue
191 }
192 files << path
193 }
194 context.all_paths = files
195 context.log('> context.all_paths after: ${context.all_paths}')
196}
197
198fn (mut context Context) process_whole_file_in_worker(path string) (int, int) {
199 context.path = path // needed for the progress bar
200 context.log('> context.process_whole_file_in_worker path: ${path}')
201 if !(os.is_file(path) && os.is_readable(path)) {
202 context.error('${path} is not readable')
203 return 1, 0
204 }
205 source := os.read_file(path) or { '' }
206 if source == '' {
207 // an empty file is a valid .v file
208 return 0, 0
209 }
210 len := source.len - 1
211 mut fails := 0
212 mut panics := 0
213 context.max_index = len
214 for i in 0 .. len {
215 verbosity := if context.is_verbose { '-v' } else { '' }
216 context.cut_index = i // needed for the progress bar
217 cmd := '${os.quoted_path(context.myself)} ${verbosity} --worker --timeout_ms ${context.timeout_ms:5} --cut_index ${i:5} --path ${os.quoted_path(path)} '
218 context.log(cmd)
219 mut res := os.execute(cmd)
220 context.log('worker exit_code: ${res.exit_code} | worker output:\n${res.output}')
221 if res.exit_code != 0 {
222 fails++
223 mut is_panic := false
224 if res.output.contains('V panic:') {
225 is_panic = true
226 panics++
227 }
228 part := source[..i]
229 line := part.count('\n') + 1
230 last_line := part.all_after_last('\n')
231 col := last_line.len
232 err := if is_panic {
233 red('parser failure: panic')
234 } else {
235 red('parser failure: crash, ${ecode_details[res.exit_code]}')
236 }
237 path_to_line := bold('${path}:${line}:${col}:')
238 err_line := last_line.trim_left('\t')
239 println('${path_to_line} ${err}')
240 println('\t${line} | ${err_line}')
241 println('')
242 eprintln(res.output)
243 eprintln('>>> failed command: ${cmd}')
244 if context.show_src {
245 eprintln('>>> source so far:')
246 eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
247 partial_source := source[..context.cut_index]
248 eprintln(partial_source)
249 eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
250 }
251 }
252 }
253 return fails, panics
254}
255
256fn (mut context Context) start_printing() {
257 context.stop_print = false
258 if !context.is_linear && !context.is_silent {
259 println('\n')
260 }
261 spawn context.print_periodic_status()
262}
263
264fn (mut context Context) stop_printing() {
265 context.stop_print = true
266 time.sleep(time.millisecond * context.period_ms / 5)
267}
268
269fn (mut context Context) print_status() {
270 if context.is_silent {
271 return
272 }
273 if context.cut_index == 1 && context.max_index == 0 {
274 return
275 }
276 msg := '> ${context.path:-30} | index: ${context.cut_index:5}/${context.max_index - 1:5}'
277 if context.is_linear {
278 eprintln(msg)
279 return
280 }
281 term.cursor_up(1)
282 eprint('\r ${msg}\n')
283}
284
285fn (mut context Context) print_periodic_status() {
286 context.print_status()
287 mut printed_at_least_once := false
288 for !context.stop_print {
289 context.print_status()
290 for i := 0; i < 10 && !context.stop_print; i++ {
291 time.sleep(time.millisecond * context.period_ms / 10)
292 if context.cut_index > 50 && !printed_at_least_once {
293 context.print_status()
294 printed_at_least_once = true
295 }
296 }
297 }
298 context.print_status()
299}