vxx / vlib / v3 / scanner / scanner.v
631 lines · 616 sloc · 11.72 KB · eecdfebe02e3f976e0a993721335b50a9c2a56f4
Raw
1module scanner
2
3import v3.token
4import v3.pref
5
6// Mode lists mode values used by scanner.
7@[flag]
8pub enum Mode {
9 normal
10 scan_comments
11 skip_interpolation
12}
13
14// Scanner represents scanner data used by scanner.
15pub struct Scanner {
16 pref &pref.Preferences
17 mode Mode
18 skip_interpolation bool
19mut:
20 file &token.File = unsafe { nil }
21 insert_semi bool
22pub mut:
23 src string
24 offset int
25 pos int
26 lit string
27 in_str_incomplete bool
28 in_str_inter bool
29 str_inter_cbr_depth int
30 str_quote u8
31}
32
33// peek_byte supports peek byte handling for Scanner.
34@[direct_array_access; inline]
35fn (s &Scanner) peek_byte(n int) u8 {
36 idx := s.offset + n
37 if idx < 0 || idx >= s.src.len {
38 return 0
39 }
40 return s.src[idx]
41}
42
43// new_scanner supports new scanner handling for scanner.
44pub fn new_scanner(prefs &pref.Preferences, mode Mode) Scanner {
45 unsafe {
46 return Scanner{
47 pref: prefs
48 mode: mode
49 skip_interpolation: mode.has(.skip_interpolation)
50 }
51 }
52}
53
54// init supports init handling for Scanner.
55pub fn (mut s Scanner) init(file &token.File, src string) {
56 s.offset = 0
57 s.pos = 0
58 s.lit = ''
59 s.insert_semi = false
60 s.in_str_incomplete = false
61 s.in_str_inter = false
62 s.str_inter_cbr_depth = 0
63 s.str_quote = 0
64 s.file = unsafe { file }
65 s.src = src
66}
67
68@[inline]
69fn (s &Scanner) source_lit(start int, end int) string {
70 if end <= start {
71 return ''
72 }
73 unsafe {
74 return tos(s.src.str + start, end - start)
75 }
76}
77
78// scan_char_literal reads scan char literal input for scanner.
79fn (mut s Scanner) scan_char_literal(quote u8) token.Token {
80 s.offset++
81 for s.offset < s.src.len {
82 c2 := s.src[s.offset]
83 if c2 == quote {
84 break
85 }
86 if c2 == `\\` && s.offset + 1 < s.src.len {
87 s.offset += 2
88 continue
89 }
90 s.offset++
91 }
92 mut end := s.offset
93 if s.offset < s.src.len && s.src[s.offset] == quote {
94 end = s.offset
95 s.offset++
96 }
97 s.lit = s.source_lit(s.pos + 1, end)
98 s.insert_semi = true
99 return .char
100}
101
102// current_file returns current file data for Scanner.
103pub fn (s &Scanner) current_file() &token.File {
104 return unsafe { s.file }
105}
106
107// scan supports scan handling for Scanner.
108@[direct_array_access]
109pub fn (mut s Scanner) scan() token.Token {
110 if s.in_str_incomplete {
111 s.in_str_incomplete = false
112 s.pos = s.offset
113 s.string_literal(false, s.str_quote)
114 s.lit = s.source_lit(s.pos, s.offset)
115 return .string
116 }
117 start:
118 s.whitespace()
119 if s.offset == s.src.len {
120 s.lit = ''
121 if s.insert_semi {
122 s.insert_semi = false
123 return .semicolon
124 }
125 s.file.add_line(s.offset)
126 return .eof
127 }
128 c := s.src[s.offset]
129 s.pos = s.offset
130 s.lit = ''
131 preserve_insert_semi := s.insert_semi
132 s.insert_semi = false
133 if c == `\n` {
134 s.lit = ''
135 return .semicolon
136 } else if c == `/` {
137 c2 := s.peek_byte(1)
138 if c2 == `/` || c2 == `*` {
139 if preserve_insert_semi {
140 s.insert_semi = true
141 }
142 s.comment()
143 if !s.mode.has(.scan_comments) {
144 unsafe {
145 goto start
146 }
147 }
148 s.lit = s.source_lit(s.pos, s.offset)
149 return .comment
150 } else if c2 == `=` {
151 s.offset += 2
152 return .div_assign
153 }
154 s.offset++
155 return .div
156 } else if c >= `0` && c <= `9` {
157 s.number()
158 s.lit = s.source_lit(s.pos, s.offset)
159 s.insert_semi = true
160 return .number
161 } else if (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`) || c == `_` || c == `@` {
162 s.offset++
163 if c == `@` && s.peek_byte(0) == `[` {
164 s.offset++
165 return .attribute
166 }
167 for s.offset < s.src.len {
168 c3 := s.src[s.offset]
169 if c3.is_alnum() || c3 == `_` {
170 s.offset++
171 continue
172 }
173 break
174 }
175 s.lit = s.source_lit(s.pos, s.offset)
176 if s.lit == 'c' && s.offset < s.src.len && s.src[s.offset] == `'` {
177 s.pos = s.offset
178 tok := s.scan_char_literal(`'`)
179 s.lit = 'c:${s.lit}'
180 return tok
181 }
182 if s.lit == 'r' && s.offset < s.src.len
183 && (s.src[s.offset] == `'` || s.src[s.offset] == `"`) {
184 quote := s.src[s.offset]
185 s.offset++
186 if !s.in_str_inter {
187 s.str_quote = quote
188 }
189 s.string_literal(true, quote)
190 s.lit = s.source_lit(s.pos, s.offset)
191 s.insert_semi = true
192 return .string
193 }
194 tok := token.Token.from_string_tinyv(s.lit)
195 if tok in [.key_break, .key_continue, .key_nil, .key_none, .key_return, .key_false, .key_true,
196 .name] {
197 s.insert_semi = true
198 }
199 return tok
200 } else if c == `'` || c == `"` {
201 s.offset++
202 if !s.in_str_inter {
203 s.str_quote = c
204 }
205 s.string_literal(s.in_str_inter || (s.offset >= 2 && s.src[s.offset - 2] == `r`), c)
206 s.lit = s.source_lit(s.pos, s.offset)
207 s.insert_semi = true
208 return .string
209 } else if c == `\`` {
210 return s.scan_char_literal(c)
211 }
212 s.lit = ''
213 s.offset++
214 match c {
215 `.` {
216 c2 := s.peek_byte(0)
217 if c2 >= `0` && c2 <= `9` {
218 s.number()
219 s.lit = s.source_lit(s.pos, s.offset)
220 return .number
221 } else if c2 == `.` {
222 s.offset++
223 if s.peek_byte(0) == `.` {
224 s.offset++
225 return .ellipsis
226 }
227 return .dotdot
228 }
229 return .dot
230 }
231 `:` {
232 if s.peek_byte(0) == `=` {
233 s.offset++
234 return .decl_assign
235 }
236 return .colon
237 }
238 `!` {
239 c2 := s.peek_byte(0)
240 if c2 == `=` {
241 s.offset++
242 return .ne
243 } else if c2 == `i` {
244 c3 := s.peek_byte(1)
245 c4 := s.peek_byte(2)
246 c4_is_space := c4 == ` ` || c4 == `\t`
247 if c3 == `n` && c4_is_space {
248 s.offset += 2
249 return .not_in
250 } else if c3 == `s` && c4_is_space {
251 s.offset += 2
252 return .not_is
253 }
254 }
255 s.insert_semi = true
256 return .not
257 }
258 `=` {
259 if s.peek_byte(0) == `=` {
260 s.offset++
261 return .eq
262 }
263 return .assign
264 }
265 `+` {
266 c2 := s.peek_byte(0)
267 if c2 == `+` {
268 s.offset++
269 return .inc
270 } else if c2 == `=` {
271 s.offset++
272 return .plus_assign
273 }
274 return .plus
275 }
276 `-` {
277 c2 := s.peek_byte(0)
278 if c2 == `-` {
279 s.offset++
280 return .dec
281 } else if c2 == `=` {
282 s.offset++
283 return .minus_assign
284 }
285 return .minus
286 }
287 `%` {
288 if s.peek_byte(0) == `=` {
289 s.offset++
290 return .mod_assign
291 }
292 return .mod
293 }
294 `*` {
295 if s.peek_byte(0) == `=` {
296 s.offset++
297 return .mul_assign
298 }
299 return .mul
300 }
301 `^` {
302 if s.peek_byte(0) == `=` {
303 s.offset++
304 return .xor_assign
305 }
306 return .xor
307 }
308 `&` {
309 c2 := s.peek_byte(0)
310 if c2 == `&` {
311 if s.peek_byte(1) == `=` {
312 s.offset += 2
313 return .and_assign
314 }
315 s.offset++
316 return .and
317 } else if c2 == `=` {
318 s.offset++
319 return .and_assign
320 }
321 return .amp
322 }
323 `|` {
324 c2 := s.peek_byte(0)
325 if c2 == `|` {
326 if s.peek_byte(1) == `=` {
327 s.offset += 2
328 return .or_assign
329 }
330 s.offset++
331 return .logical_or
332 } else if c2 == `=` {
333 s.offset++
334 return .or_assign
335 }
336 return .pipe
337 }
338 `<` {
339 c2 := s.peek_byte(0)
340 if c2 == `<` {
341 s.offset++
342 if s.peek_byte(0) == `=` {
343 s.offset++
344 return .left_shift_assign
345 }
346 return .left_shift
347 } else if c2 == `=` {
348 s.offset++
349 return .le
350 } else if c2 == `-` {
351 s.offset++
352 return .arrow
353 }
354 return .lt
355 }
356 `>` {
357 c2 := s.peek_byte(0)
358 if c2 == `>` {
359 s.offset++
360 c3 := s.peek_byte(0)
361 if c3 == `>` {
362 s.offset++
363 if s.peek_byte(0) == `=` {
364 s.offset++
365 return .right_shift_unsigned_assign
366 }
367 return .right_shift_unsigned
368 } else if c3 == `=` {
369 s.offset++
370 return .right_shift_assign
371 }
372 return .right_shift
373 } else if c2 == `=` {
374 s.offset++
375 return .ge
376 }
377 return .gt
378 }
379 `#` {
380 if s.peek_byte(0) == `[` {
381 s.offset++
382 return .lsbr
383 }
384 start := s.offset
385 for s.offset < s.src.len && s.src[s.offset] != `\n` {
386 s.offset++
387 }
388 s.lit = s.source_lit(start, s.offset).trim_space()
389 s.insert_semi = true
390 return .hash
391 }
392 `~` {
393 return .bit_not
394 }
395 `,` {
396 return .comma
397 }
398 `$` {
399 if s.in_str_inter {
400 return .str_dollar
401 }
402 return .dollar
403 }
404 `{` {
405 if s.in_str_inter {
406 s.str_inter_cbr_depth++
407 }
408 return .lcbr
409 }
410 `}` {
411 if s.in_str_inter {
412 s.str_inter_cbr_depth--
413 if s.str_inter_cbr_depth == 0 {
414 s.in_str_incomplete = true
415 s.in_str_inter = false
416 }
417 }
418 s.insert_semi = true
419 return .rcbr
420 }
421 `(` {
422 return .lpar
423 }
424 `)` {
425 s.insert_semi = true
426 return .rpar
427 }
428 `[` {
429 return .lsbr
430 }
431 `]` {
432 s.insert_semi = true
433 return .rsbr
434 }
435 `;` {
436 return .semicolon
437 }
438 `?` {
439 s.insert_semi = true
440 return .question
441 }
442 else {
443 return .unknown
444 }
445 }
446}
447
448@[direct_array_access]
449fn (mut s Scanner) whitespace() {
450 for s.offset < s.src.len {
451 c := s.src[s.offset]
452 if c == ` ` || c == `\t` || c == `\r` {
453 s.offset++
454 continue
455 } else if c == `\n` {
456 if s.insert_semi {
457 return
458 }
459 s.offset++
460 s.file.add_line(s.offset)
461 continue
462 }
463 break
464 }
465}
466
467@[direct_array_access]
468fn (mut s Scanner) line() {
469 for s.offset < s.src.len {
470 if s.src[s.offset] == `\n` {
471 break
472 }
473 s.offset++
474 }
475}
476
477@[direct_array_access]
478fn (mut s Scanner) comment() {
479 s.offset++
480 if s.offset >= s.src.len {
481 return
482 }
483 c := s.src[s.offset]
484 if c == `/` {
485 s.line()
486 } else if c == `*` {
487 s.offset++
488 mut ml_comment_depth := 1
489 for s.offset < s.src.len {
490 c2 := s.src[s.offset]
491 c3 := s.peek_byte(1)
492 if c2 == `\n` {
493 s.offset++
494 s.file.add_line(s.offset)
495 } else if c2 == `/` && c3 == `*` {
496 s.offset += 2
497 ml_comment_depth++
498 } else if c2 == `*` && c3 == `/` {
499 s.offset += 2
500 ml_comment_depth--
501 if ml_comment_depth == 0 {
502 break
503 }
504 } else {
505 s.offset++
506 }
507 }
508 }
509}
510
511@[direct_array_access]
512fn (mut s Scanner) string_literal(scan_as_raw bool, c_quote u8) {
513 if scan_as_raw {
514 for s.offset < s.src.len {
515 c := s.src[s.offset]
516 if c == c_quote {
517 break
518 }
519 if c == `\n` {
520 s.offset++
521 s.file.add_line(s.offset)
522 continue
523 }
524 s.offset++
525 }
526 if s.offset < s.src.len {
527 s.offset++
528 }
529 return
530 }
531 for s.offset < s.src.len {
532 c := s.src[s.offset]
533 if c == `\\` {
534 s.offset += 2
535 continue
536 } else if c == `\n` {
537 s.offset++
538 s.file.add_line(s.offset)
539 continue
540 } else if c == `$` && s.peek_byte(1) == `{` {
541 s.in_str_inter = true
542 if s.skip_interpolation {
543 s.str_inter_cbr_depth++
544 s.offset += 2
545 continue
546 } else {
547 return
548 }
549 } else if s.skip_interpolation && s.in_str_inter {
550 if c == `{` {
551 s.str_inter_cbr_depth++
552 } else if c == `}` {
553 s.str_inter_cbr_depth--
554 if s.str_inter_cbr_depth == 0 {
555 s.in_str_inter = false
556 }
557 }
558 } else if c == c_quote && !s.in_str_inter {
559 s.offset++
560 break
561 }
562 s.offset++
563 }
564}
565
566@[direct_array_access]
567fn (mut s Scanner) number() {
568 if s.offset < s.src.len && s.src[s.offset] == `0` {
569 s.offset++
570 c := s.peek_byte(0)
571 if c == `b` || c == `B` {
572 s.offset++
573 for s.offset < s.src.len {
574 c2 := s.src[s.offset]
575 if c2 == `0` || c2 == `1` || c2 == `_` {
576 s.offset++
577 continue
578 }
579 return
580 }
581 return
582 } else if c == `x` || c == `X` {
583 s.offset++
584 for s.offset < s.src.len {
585 c2 := s.src[s.offset]
586 if (c2 >= `0` && c2 <= `9`) || (c2 >= `a` && c2 <= `f`)
587 || (c2 >= `A` && c2 <= `F`) || c2 == `_` {
588 s.offset++
589 continue
590 }
591 return
592 }
593 return
594 } else if c == `o` || c == `O` {
595 s.offset++
596 for s.offset < s.src.len {
597 c2 := s.src[s.offset]
598 if (c2 >= `0` && c2 <= `7`) || c2 == `_` {
599 s.offset++
600 continue
601 }
602 return
603 }
604 return
605 }
606 }
607 mut has_decimal := false
608 mut has_exponent := false
609 for s.offset < s.src.len {
610 c := s.src[s.offset]
611 if (c >= `0` && c <= `9`) || c == `_` {
612 s.offset++
613 continue
614 } else if !has_decimal && c == `.` {
615 next := s.peek_byte(1)
616 if next != `.` && next >= `0` && next <= `9` {
617 has_decimal = true
618 s.offset++
619 continue
620 }
621 } else if !has_exponent && (c == `e` || c == `E`) {
622 has_exponent = true
623 s.offset++
624 if s.offset < s.src.len && (s.src[s.offset] == `+` || s.src[s.offset] == `-`) {
625 s.offset++
626 }
627 continue
628 }
629 break
630 }
631}
632