vxx2 / vlib / v3 / scanner / scanner.v
627 lines · 612 sloc · 11.64 KB · 073a7c9674c038f990d84c2707a980d998445c5e
Raw
1module scanner
2
3import v3.token
4import v3.pref
5
6// Mode lists mode values used by scanner.
7@[flag]
8pub enum Mode {
9 normal
10 scan_comments
11 skip_interpolation
12}
13
14// Scanner represents scanner data used by scanner.
15pub struct Scanner {
16 pref &pref.Preferences
17 mode Mode
18 skip_interpolation bool
19mut:
20 file &token.File = unsafe { nil }
21 insert_semi bool
22pub mut:
23 src string
24 offset int
25 pos int
26 lit string
27 in_str_incomplete bool
28 in_str_inter bool
29 str_inter_cbr_depth int
30 str_quote u8
31}
32
33// peek_byte supports peek byte handling for Scanner.
34@[direct_array_access; inline]
35fn (s &Scanner) peek_byte(n int) u8 {
36 idx := s.offset + n
37 if idx < 0 || idx >= s.src.len {
38 return 0
39 }
40 return s.src[idx]
41}
42
43// new_scanner supports new scanner handling for scanner.
44pub fn new_scanner(prefs &pref.Preferences, mode Mode) Scanner {
45 unsafe {
46 return Scanner{
47 pref: prefs
48 mode: mode
49 skip_interpolation: mode.has(.skip_interpolation)
50 }
51 }
52}
53
54// init supports init handling for Scanner.
55pub fn (mut s Scanner) init(file &token.File, src string) {
56 s.offset = 0
57 s.pos = 0
58 s.lit = ''
59 s.insert_semi = false
60 s.in_str_incomplete = false
61 s.in_str_inter = false
62 s.str_inter_cbr_depth = 0
63 s.str_quote = 0
64 s.file = unsafe { file }
65 s.src = src
66}
67
68@[inline]
69fn (s &Scanner) source_lit(start int, end int) string {
70 if end <= start {
71 return ''
72 }
73 unsafe {
74 return tos(s.src.str + start, end - start)
75 }
76}
77
78// scan_char_literal reads scan char literal input for scanner.
79fn (mut s Scanner) scan_char_literal(quote u8) token.Token {
80 s.offset++
81 for s.offset < s.src.len {
82 c2 := s.src[s.offset]
83 if c2 == quote {
84 break
85 }
86 if c2 == `\\` && s.offset + 1 < s.src.len {
87 s.offset += 2
88 continue
89 }
90 s.offset++
91 }
92 mut end := s.offset
93 if s.offset < s.src.len && s.src[s.offset] == quote {
94 end = s.offset
95 s.offset++
96 }
97 s.lit = s.source_lit(s.pos + 1, end)
98 s.insert_semi = true
99 return .char
100}
101
102// current_file returns current file data for Scanner.
103pub fn (s &Scanner) current_file() &token.File {
104 return unsafe { s.file }
105}
106
107// scan supports scan handling for Scanner.
108@[direct_array_access]
109pub fn (mut s Scanner) scan() token.Token {
110 if s.in_str_incomplete {
111 s.in_str_incomplete = false
112 s.pos = s.offset
113 s.string_literal(false, s.str_quote)
114 s.lit = s.source_lit(s.pos, s.offset)
115 return .string
116 }
117 start:
118 s.whitespace()
119 if s.offset == s.src.len {
120 s.lit = ''
121 if s.insert_semi {
122 s.insert_semi = false
123 return .semicolon
124 }
125 return .eof
126 }
127 c := s.src[s.offset]
128 s.pos = s.offset
129 s.lit = ''
130 preserve_insert_semi := s.insert_semi
131 s.insert_semi = false
132 if c == `\n` {
133 s.lit = ''
134 return .semicolon
135 } else if c == `/` {
136 c2 := s.peek_byte(1)
137 if c2 == `/` || c2 == `*` {
138 if preserve_insert_semi {
139 s.insert_semi = true
140 }
141 s.comment()
142 if !s.mode.has(.scan_comments) {
143 unsafe {
144 goto start
145 }
146 }
147 s.lit = s.source_lit(s.pos, s.offset)
148 return .comment
149 } else if c2 == `=` {
150 s.offset += 2
151 return .div_assign
152 }
153 s.offset++
154 return .div
155 } else if c >= `0` && c <= `9` {
156 s.number()
157 s.lit = s.source_lit(s.pos, s.offset)
158 s.insert_semi = true
159 return .number
160 } else if (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`) || c == `_` || c == `@` {
161 s.offset++
162 if c == `@` && s.peek_byte(0) == `[` {
163 s.offset++
164 return .attribute
165 }
166 for s.offset < s.src.len {
167 c3 := s.src[s.offset]
168 if (c3 >= `a` && c3 <= `z`) || (c3 >= `A` && c3 <= `Z`)
169 || (c3 >= `0` && c3 <= `9`) || c3 == `_` {
170 s.offset++
171 continue
172 }
173 break
174 }
175 s.lit = s.source_lit(s.pos, s.offset)
176 if s.lit == 'c' && s.offset < s.src.len && s.src[s.offset] == `'` {
177 s.pos = s.offset
178 tok := s.scan_char_literal(`'`)
179 s.lit = 'c:${s.lit}'
180 return tok
181 }
182 if s.lit == 'r' && s.offset < s.src.len
183 && (s.src[s.offset] == `'` || s.src[s.offset] == `"`) {
184 quote := s.src[s.offset]
185 s.offset++
186 if !s.in_str_inter {
187 s.str_quote = quote
188 }
189 s.string_literal(true, quote)
190 s.lit = s.source_lit(s.pos, s.offset)
191 s.insert_semi = true
192 return .string
193 }
194 tok := token.Token.from_string_tinyv(s.lit)
195 if tok in [.key_break, .key_continue, .key_nil, .key_none, .key_return, .key_false, .key_true,
196 .name] {
197 s.insert_semi = true
198 }
199 return tok
200 } else if c == `'` || c == `"` {
201 s.offset++
202 if !s.in_str_inter {
203 s.str_quote = c
204 }
205 s.string_literal(s.in_str_inter || (s.offset >= 2 && s.src[s.offset - 2] == `r`), c)
206 s.lit = s.source_lit(s.pos, s.offset)
207 s.insert_semi = true
208 return .string
209 } else if c == `\`` {
210 return s.scan_char_literal(c)
211 }
212 s.lit = ''
213 s.offset++
214 match c {
215 `.` {
216 c2 := s.peek_byte(0)
217 if c2 >= `0` && c2 <= `9` {
218 s.number()
219 s.lit = s.source_lit(s.pos, s.offset)
220 return .number
221 } else if c2 == `.` {
222 s.offset++
223 if s.peek_byte(0) == `.` {
224 s.offset++
225 return .ellipsis
226 }
227 return .dotdot
228 }
229 return .dot
230 }
231 `:` {
232 if s.peek_byte(0) == `=` {
233 s.offset++
234 return .decl_assign
235 }
236 return .colon
237 }
238 `!` {
239 c2 := s.peek_byte(0)
240 if c2 == `=` {
241 s.offset++
242 return .ne
243 } else if c2 == `i` {
244 c3 := s.peek_byte(1)
245 c4 := s.peek_byte(2)
246 c4_is_space := c4 == ` ` || c4 == `\t`
247 if c3 == `n` && c4_is_space {
248 s.offset += 2
249 return .not_in
250 } else if c3 == `s` && c4_is_space {
251 s.offset += 2
252 return .not_is
253 }
254 }
255 s.insert_semi = true
256 return .not
257 }
258 `=` {
259 if s.peek_byte(0) == `=` {
260 s.offset++
261 return .eq
262 }
263 return .assign
264 }
265 `+` {
266 c2 := s.peek_byte(0)
267 if c2 == `+` {
268 s.offset++
269 return .inc
270 } else if c2 == `=` {
271 s.offset++
272 return .plus_assign
273 }
274 return .plus
275 }
276 `-` {
277 c2 := s.peek_byte(0)
278 if c2 == `-` {
279 s.offset++
280 return .dec
281 } else if c2 == `=` {
282 s.offset++
283 return .minus_assign
284 }
285 return .minus
286 }
287 `%` {
288 if s.peek_byte(0) == `=` {
289 s.offset++
290 return .mod_assign
291 }
292 return .mod
293 }
294 `*` {
295 if s.peek_byte(0) == `=` {
296 s.offset++
297 return .mul_assign
298 }
299 return .mul
300 }
301 `^` {
302 if s.peek_byte(0) == `=` {
303 s.offset++
304 return .xor_assign
305 }
306 return .xor
307 }
308 `&` {
309 c2 := s.peek_byte(0)
310 if c2 == `&` {
311 if s.peek_byte(1) == `=` {
312 s.offset += 2
313 return .and_assign
314 }
315 s.offset++
316 return .and
317 } else if c2 == `=` {
318 s.offset++
319 return .and_assign
320 }
321 return .amp
322 }
323 `|` {
324 c2 := s.peek_byte(0)
325 if c2 == `|` {
326 if s.peek_byte(1) == `=` {
327 s.offset += 2
328 return .or_assign
329 }
330 s.offset++
331 return .logical_or
332 } else if c2 == `=` {
333 s.offset++
334 return .or_assign
335 }
336 return .pipe
337 }
338 `<` {
339 c2 := s.peek_byte(0)
340 if c2 == `<` {
341 s.offset++
342 if s.peek_byte(0) == `=` {
343 s.offset++
344 return .left_shift_assign
345 }
346 return .left_shift
347 } else if c2 == `=` {
348 s.offset++
349 return .le
350 } else if c2 == `-` {
351 s.offset++
352 return .arrow
353 }
354 return .lt
355 }
356 `>` {
357 c2 := s.peek_byte(0)
358 if c2 == `>` {
359 s.offset++
360 c3 := s.peek_byte(0)
361 if c3 == `>` {
362 s.offset++
363 if s.peek_byte(0) == `=` {
364 s.offset++
365 return .right_shift_unsigned_assign
366 }
367 return .right_shift_unsigned
368 } else if c3 == `=` {
369 s.offset++
370 return .right_shift_assign
371 }
372 return .right_shift
373 } else if c2 == `=` {
374 s.offset++
375 return .ge
376 }
377 return .gt
378 }
379 `#` {
380 if s.peek_byte(0) == `[` {
381 s.offset++
382 return .lsbr
383 }
384 start := s.offset
385 for s.offset < s.src.len && s.src[s.offset] != `\n` {
386 s.offset++
387 }
388 s.lit = s.source_lit(start, s.offset).trim_space()
389 s.insert_semi = true
390 return .hash
391 }
392 `~` {
393 return .bit_not
394 }
395 `,` {
396 return .comma
397 }
398 `$` {
399 if s.in_str_inter {
400 return .str_dollar
401 }
402 return .dollar
403 }
404 `{` {
405 if s.in_str_inter {
406 s.str_inter_cbr_depth++
407 }
408 return .lcbr
409 }
410 `}` {
411 if s.in_str_inter {
412 s.str_inter_cbr_depth--
413 if s.str_inter_cbr_depth == 0 {
414 s.in_str_incomplete = true
415 s.in_str_inter = false
416 }
417 }
418 s.insert_semi = true
419 return .rcbr
420 }
421 `(` {
422 return .lpar
423 }
424 `)` {
425 s.insert_semi = true
426 return .rpar
427 }
428 `[` {
429 return .lsbr
430 }
431 `]` {
432 s.insert_semi = true
433 return .rsbr
434 }
435 `;` {
436 return .semicolon
437 }
438 `?` {
439 s.insert_semi = true
440 return .question
441 }
442 else {
443 return .unknown
444 }
445 }
446}
447
448@[direct_array_access]
449fn (mut s Scanner) whitespace() {
450 for s.offset < s.src.len {
451 c := s.src[s.offset]
452 if c == ` ` || c == `\t` || c == `\r` {
453 s.offset++
454 continue
455 } else if c == `\n` {
456 if s.insert_semi {
457 return
458 }
459 s.offset++
460 continue
461 }
462 break
463 }
464}
465
466@[direct_array_access]
467fn (mut s Scanner) line() {
468 for s.offset < s.src.len {
469 if s.src[s.offset] == `\n` {
470 break
471 }
472 s.offset++
473 }
474}
475
476@[direct_array_access]
477fn (mut s Scanner) comment() {
478 s.offset++
479 if s.offset >= s.src.len {
480 return
481 }
482 c := s.src[s.offset]
483 if c == `/` {
484 s.line()
485 } else if c == `*` {
486 s.offset++
487 mut ml_comment_depth := 1
488 for s.offset < s.src.len {
489 c2 := s.src[s.offset]
490 c3 := s.peek_byte(1)
491 if c2 == `\n` {
492 s.offset++
493 } else if c2 == `/` && c3 == `*` {
494 s.offset += 2
495 ml_comment_depth++
496 } else if c2 == `*` && c3 == `/` {
497 s.offset += 2
498 ml_comment_depth--
499 if ml_comment_depth == 0 {
500 break
501 }
502 } else {
503 s.offset++
504 }
505 }
506 }
507}
508
509@[direct_array_access]
510fn (mut s Scanner) string_literal(scan_as_raw bool, c_quote u8) {
511 if scan_as_raw {
512 for s.offset < s.src.len {
513 c := s.src[s.offset]
514 if c == c_quote {
515 break
516 }
517 if c == `\n` {
518 s.offset++
519 continue
520 }
521 s.offset++
522 }
523 if s.offset < s.src.len {
524 s.offset++
525 }
526 return
527 }
528 for s.offset < s.src.len {
529 c := s.src[s.offset]
530 if c == `\\` {
531 s.offset += 2
532 continue
533 } else if c == `\n` {
534 s.offset++
535 continue
536 } else if c == `$` && s.peek_byte(1) == `{` {
537 s.in_str_inter = true
538 if s.skip_interpolation {
539 s.str_inter_cbr_depth++
540 s.offset += 2
541 continue
542 } else {
543 return
544 }
545 } else if s.skip_interpolation && s.in_str_inter {
546 if c == `{` {
547 s.str_inter_cbr_depth++
548 } else if c == `}` {
549 s.str_inter_cbr_depth--
550 if s.str_inter_cbr_depth == 0 {
551 s.in_str_inter = false
552 }
553 }
554 } else if c == c_quote && !s.in_str_inter {
555 s.offset++
556 break
557 }
558 s.offset++
559 }
560}
561
562@[direct_array_access]
563fn (mut s Scanner) number() {
564 if s.offset < s.src.len && s.src[s.offset] == `0` {
565 s.offset++
566 c := s.peek_byte(0)
567 if c == `b` || c == `B` {
568 s.offset++
569 for s.offset < s.src.len {
570 c2 := s.src[s.offset]
571 if c2 == `0` || c2 == `1` || c2 == `_` {
572 s.offset++
573 continue
574 }
575 return
576 }
577 return
578 } else if c == `x` || c == `X` {
579 s.offset++
580 for s.offset < s.src.len {
581 c2 := s.src[s.offset]
582 if (c2 >= `0` && c2 <= `9`) || (c2 >= `a` && c2 <= `f`)
583 || (c2 >= `A` && c2 <= `F`) || c2 == `_` {
584 s.offset++
585 continue
586 }
587 return
588 }
589 return
590 } else if c == `o` || c == `O` {
591 s.offset++
592 for s.offset < s.src.len {
593 c2 := s.src[s.offset]
594 if (c2 >= `0` && c2 <= `7`) || c2 == `_` {
595 s.offset++
596 continue
597 }
598 return
599 }
600 return
601 }
602 }
603 mut has_decimal := false
604 mut has_exponent := false
605 for s.offset < s.src.len {
606 c := s.src[s.offset]
607 if (c >= `0` && c <= `9`) || c == `_` {
608 s.offset++
609 continue
610 } else if !has_decimal && c == `.` {
611 next := s.peek_byte(1)
612 if next != `.` && next >= `0` && next <= `9` {
613 has_decimal = true
614 s.offset++
615 continue
616 }
617 } else if !has_exponent && (c == `e` || c == `E`) {
618 has_exponent = true
619 s.offset++
620 if s.offset < s.src.len && (s.src[s.offset] == `+` || s.src[s.offset] == `-`) {
621 s.offset++
622 }
623 continue
624 }
625 break
626 }
627}
628