| 1 | module scanner |
| 2 | |
| 3 | import v3.token |
| 4 | import v3.pref |
| 5 | |
| 6 | // Mode lists mode values used by scanner. |
| 7 | @[flag] |
| 8 | pub enum Mode { |
| 9 | normal |
| 10 | scan_comments |
| 11 | skip_interpolation |
| 12 | } |
| 13 | |
| 14 | // Scanner represents scanner data used by scanner. |
| 15 | pub struct Scanner { |
| 16 | pref &pref.Preferences |
| 17 | mode Mode |
| 18 | skip_interpolation bool |
| 19 | mut: |
| 20 | file &token.File = unsafe { nil } |
| 21 | insert_semi bool |
| 22 | pub mut: |
| 23 | src string |
| 24 | offset int |
| 25 | pos int |
| 26 | lit string |
| 27 | in_str_incomplete bool |
| 28 | in_str_inter bool |
| 29 | str_inter_cbr_depth int |
| 30 | str_quote u8 |
| 31 | } |
| 32 | |
| 33 | // peek_byte supports peek byte handling for Scanner. |
| 34 | @[direct_array_access; inline] |
| 35 | fn (s &Scanner) peek_byte(n int) u8 { |
| 36 | idx := s.offset + n |
| 37 | if idx < 0 || idx >= s.src.len { |
| 38 | return 0 |
| 39 | } |
| 40 | return s.src[idx] |
| 41 | } |
| 42 | |
| 43 | // new_scanner supports new scanner handling for scanner. |
| 44 | pub fn new_scanner(prefs &pref.Preferences, mode Mode) Scanner { |
| 45 | unsafe { |
| 46 | return Scanner{ |
| 47 | pref: prefs |
| 48 | mode: mode |
| 49 | skip_interpolation: mode.has(.skip_interpolation) |
| 50 | } |
| 51 | } |
| 52 | } |
| 53 | |
| 54 | // init supports init handling for Scanner. |
| 55 | pub fn (mut s Scanner) init(file &token.File, src string) { |
| 56 | s.offset = 0 |
| 57 | s.pos = 0 |
| 58 | s.lit = '' |
| 59 | s.insert_semi = false |
| 60 | s.in_str_incomplete = false |
| 61 | s.in_str_inter = false |
| 62 | s.str_inter_cbr_depth = 0 |
| 63 | s.str_quote = 0 |
| 64 | s.file = unsafe { file } |
| 65 | s.src = src |
| 66 | } |
| 67 | |
| 68 | @[inline] |
| 69 | fn (s &Scanner) source_lit(start int, end int) string { |
| 70 | if end <= start { |
| 71 | return '' |
| 72 | } |
| 73 | unsafe { |
| 74 | return tos(s.src.str + start, end - start) |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | // scan_char_literal reads scan char literal input for scanner. |
| 79 | fn (mut s Scanner) scan_char_literal(quote u8) token.Token { |
| 80 | s.offset++ |
| 81 | for s.offset < s.src.len { |
| 82 | c2 := s.src[s.offset] |
| 83 | if c2 == quote { |
| 84 | break |
| 85 | } |
| 86 | if c2 == `\\` && s.offset + 1 < s.src.len { |
| 87 | s.offset += 2 |
| 88 | continue |
| 89 | } |
| 90 | s.offset++ |
| 91 | } |
| 92 | mut end := s.offset |
| 93 | if s.offset < s.src.len && s.src[s.offset] == quote { |
| 94 | end = s.offset |
| 95 | s.offset++ |
| 96 | } |
| 97 | s.lit = s.source_lit(s.pos + 1, end) |
| 98 | s.insert_semi = true |
| 99 | return .char |
| 100 | } |
| 101 | |
| 102 | // current_file returns current file data for Scanner. |
| 103 | pub fn (s &Scanner) current_file() &token.File { |
| 104 | return unsafe { s.file } |
| 105 | } |
| 106 | |
| 107 | // scan supports scan handling for Scanner. |
| 108 | @[direct_array_access] |
| 109 | pub fn (mut s Scanner) scan() token.Token { |
| 110 | if s.in_str_incomplete { |
| 111 | s.in_str_incomplete = false |
| 112 | s.pos = s.offset |
| 113 | s.string_literal(false, s.str_quote) |
| 114 | s.lit = s.source_lit(s.pos, s.offset) |
| 115 | return .string |
| 116 | } |
| 117 | start: |
| 118 | s.whitespace() |
| 119 | if s.offset == s.src.len { |
| 120 | s.lit = '' |
| 121 | if s.insert_semi { |
| 122 | s.insert_semi = false |
| 123 | return .semicolon |
| 124 | } |
| 125 | return .eof |
| 126 | } |
| 127 | c := s.src[s.offset] |
| 128 | s.pos = s.offset |
| 129 | s.lit = '' |
| 130 | preserve_insert_semi := s.insert_semi |
| 131 | s.insert_semi = false |
| 132 | if c == `\n` { |
| 133 | s.lit = '' |
| 134 | return .semicolon |
| 135 | } else if c == `/` { |
| 136 | c2 := s.peek_byte(1) |
| 137 | if c2 == `/` || c2 == `*` { |
| 138 | if preserve_insert_semi { |
| 139 | s.insert_semi = true |
| 140 | } |
| 141 | s.comment() |
| 142 | if !s.mode.has(.scan_comments) { |
| 143 | unsafe { |
| 144 | goto start |
| 145 | } |
| 146 | } |
| 147 | s.lit = s.source_lit(s.pos, s.offset) |
| 148 | return .comment |
| 149 | } else if c2 == `=` { |
| 150 | s.offset += 2 |
| 151 | return .div_assign |
| 152 | } |
| 153 | s.offset++ |
| 154 | return .div |
| 155 | } else if c >= `0` && c <= `9` { |
| 156 | s.number() |
| 157 | s.lit = s.source_lit(s.pos, s.offset) |
| 158 | s.insert_semi = true |
| 159 | return .number |
| 160 | } else if (c >= `a` && c <= `z`) || (c >= `A` && c <= `Z`) || c == `_` || c == `@` { |
| 161 | s.offset++ |
| 162 | if c == `@` && s.peek_byte(0) == `[` { |
| 163 | s.offset++ |
| 164 | return .attribute |
| 165 | } |
| 166 | for s.offset < s.src.len { |
| 167 | c3 := s.src[s.offset] |
| 168 | if (c3 >= `a` && c3 <= `z`) || (c3 >= `A` && c3 <= `Z`) |
| 169 | || (c3 >= `0` && c3 <= `9`) || c3 == `_` { |
| 170 | s.offset++ |
| 171 | continue |
| 172 | } |
| 173 | break |
| 174 | } |
| 175 | s.lit = s.source_lit(s.pos, s.offset) |
| 176 | if s.lit == 'c' && s.offset < s.src.len && s.src[s.offset] == `'` { |
| 177 | s.pos = s.offset |
| 178 | tok := s.scan_char_literal(`'`) |
| 179 | s.lit = 'c:${s.lit}' |
| 180 | return tok |
| 181 | } |
| 182 | if s.lit == 'r' && s.offset < s.src.len |
| 183 | && (s.src[s.offset] == `'` || s.src[s.offset] == `"`) { |
| 184 | quote := s.src[s.offset] |
| 185 | s.offset++ |
| 186 | if !s.in_str_inter { |
| 187 | s.str_quote = quote |
| 188 | } |
| 189 | s.string_literal(true, quote) |
| 190 | s.lit = s.source_lit(s.pos, s.offset) |
| 191 | s.insert_semi = true |
| 192 | return .string |
| 193 | } |
| 194 | tok := token.Token.from_string_tinyv(s.lit) |
| 195 | if tok in [.key_break, .key_continue, .key_nil, .key_none, .key_return, .key_false, .key_true, |
| 196 | .name] { |
| 197 | s.insert_semi = true |
| 198 | } |
| 199 | return tok |
| 200 | } else if c == `'` || c == `"` { |
| 201 | s.offset++ |
| 202 | if !s.in_str_inter { |
| 203 | s.str_quote = c |
| 204 | } |
| 205 | s.string_literal(s.in_str_inter || (s.offset >= 2 && s.src[s.offset - 2] == `r`), c) |
| 206 | s.lit = s.source_lit(s.pos, s.offset) |
| 207 | s.insert_semi = true |
| 208 | return .string |
| 209 | } else if c == `\`` { |
| 210 | return s.scan_char_literal(c) |
| 211 | } |
| 212 | s.lit = '' |
| 213 | s.offset++ |
| 214 | match c { |
| 215 | `.` { |
| 216 | c2 := s.peek_byte(0) |
| 217 | if c2 >= `0` && c2 <= `9` { |
| 218 | s.number() |
| 219 | s.lit = s.source_lit(s.pos, s.offset) |
| 220 | return .number |
| 221 | } else if c2 == `.` { |
| 222 | s.offset++ |
| 223 | if s.peek_byte(0) == `.` { |
| 224 | s.offset++ |
| 225 | return .ellipsis |
| 226 | } |
| 227 | return .dotdot |
| 228 | } |
| 229 | return .dot |
| 230 | } |
| 231 | `:` { |
| 232 | if s.peek_byte(0) == `=` { |
| 233 | s.offset++ |
| 234 | return .decl_assign |
| 235 | } |
| 236 | return .colon |
| 237 | } |
| 238 | `!` { |
| 239 | c2 := s.peek_byte(0) |
| 240 | if c2 == `=` { |
| 241 | s.offset++ |
| 242 | return .ne |
| 243 | } else if c2 == `i` { |
| 244 | c3 := s.peek_byte(1) |
| 245 | c4 := s.peek_byte(2) |
| 246 | c4_is_space := c4 == ` ` || c4 == `\t` |
| 247 | if c3 == `n` && c4_is_space { |
| 248 | s.offset += 2 |
| 249 | return .not_in |
| 250 | } else if c3 == `s` && c4_is_space { |
| 251 | s.offset += 2 |
| 252 | return .not_is |
| 253 | } |
| 254 | } |
| 255 | s.insert_semi = true |
| 256 | return .not |
| 257 | } |
| 258 | `=` { |
| 259 | if s.peek_byte(0) == `=` { |
| 260 | s.offset++ |
| 261 | return .eq |
| 262 | } |
| 263 | return .assign |
| 264 | } |
| 265 | `+` { |
| 266 | c2 := s.peek_byte(0) |
| 267 | if c2 == `+` { |
| 268 | s.offset++ |
| 269 | return .inc |
| 270 | } else if c2 == `=` { |
| 271 | s.offset++ |
| 272 | return .plus_assign |
| 273 | } |
| 274 | return .plus |
| 275 | } |
| 276 | `-` { |
| 277 | c2 := s.peek_byte(0) |
| 278 | if c2 == `-` { |
| 279 | s.offset++ |
| 280 | return .dec |
| 281 | } else if c2 == `=` { |
| 282 | s.offset++ |
| 283 | return .minus_assign |
| 284 | } |
| 285 | return .minus |
| 286 | } |
| 287 | `%` { |
| 288 | if s.peek_byte(0) == `=` { |
| 289 | s.offset++ |
| 290 | return .mod_assign |
| 291 | } |
| 292 | return .mod |
| 293 | } |
| 294 | `*` { |
| 295 | if s.peek_byte(0) == `=` { |
| 296 | s.offset++ |
| 297 | return .mul_assign |
| 298 | } |
| 299 | return .mul |
| 300 | } |
| 301 | `^` { |
| 302 | if s.peek_byte(0) == `=` { |
| 303 | s.offset++ |
| 304 | return .xor_assign |
| 305 | } |
| 306 | return .xor |
| 307 | } |
| 308 | `&` { |
| 309 | c2 := s.peek_byte(0) |
| 310 | if c2 == `&` { |
| 311 | if s.peek_byte(1) == `=` { |
| 312 | s.offset += 2 |
| 313 | return .and_assign |
| 314 | } |
| 315 | s.offset++ |
| 316 | return .and |
| 317 | } else if c2 == `=` { |
| 318 | s.offset++ |
| 319 | return .and_assign |
| 320 | } |
| 321 | return .amp |
| 322 | } |
| 323 | `|` { |
| 324 | c2 := s.peek_byte(0) |
| 325 | if c2 == `|` { |
| 326 | if s.peek_byte(1) == `=` { |
| 327 | s.offset += 2 |
| 328 | return .or_assign |
| 329 | } |
| 330 | s.offset++ |
| 331 | return .logical_or |
| 332 | } else if c2 == `=` { |
| 333 | s.offset++ |
| 334 | return .or_assign |
| 335 | } |
| 336 | return .pipe |
| 337 | } |
| 338 | `<` { |
| 339 | c2 := s.peek_byte(0) |
| 340 | if c2 == `<` { |
| 341 | s.offset++ |
| 342 | if s.peek_byte(0) == `=` { |
| 343 | s.offset++ |
| 344 | return .left_shift_assign |
| 345 | } |
| 346 | return .left_shift |
| 347 | } else if c2 == `=` { |
| 348 | s.offset++ |
| 349 | return .le |
| 350 | } else if c2 == `-` { |
| 351 | s.offset++ |
| 352 | return .arrow |
| 353 | } |
| 354 | return .lt |
| 355 | } |
| 356 | `>` { |
| 357 | c2 := s.peek_byte(0) |
| 358 | if c2 == `>` { |
| 359 | s.offset++ |
| 360 | c3 := s.peek_byte(0) |
| 361 | if c3 == `>` { |
| 362 | s.offset++ |
| 363 | if s.peek_byte(0) == `=` { |
| 364 | s.offset++ |
| 365 | return .right_shift_unsigned_assign |
| 366 | } |
| 367 | return .right_shift_unsigned |
| 368 | } else if c3 == `=` { |
| 369 | s.offset++ |
| 370 | return .right_shift_assign |
| 371 | } |
| 372 | return .right_shift |
| 373 | } else if c2 == `=` { |
| 374 | s.offset++ |
| 375 | return .ge |
| 376 | } |
| 377 | return .gt |
| 378 | } |
| 379 | `#` { |
| 380 | if s.peek_byte(0) == `[` { |
| 381 | s.offset++ |
| 382 | return .lsbr |
| 383 | } |
| 384 | start := s.offset |
| 385 | for s.offset < s.src.len && s.src[s.offset] != `\n` { |
| 386 | s.offset++ |
| 387 | } |
| 388 | s.lit = s.source_lit(start, s.offset).trim_space() |
| 389 | s.insert_semi = true |
| 390 | return .hash |
| 391 | } |
| 392 | `~` { |
| 393 | return .bit_not |
| 394 | } |
| 395 | `,` { |
| 396 | return .comma |
| 397 | } |
| 398 | `$` { |
| 399 | if s.in_str_inter { |
| 400 | return .str_dollar |
| 401 | } |
| 402 | return .dollar |
| 403 | } |
| 404 | `{` { |
| 405 | if s.in_str_inter { |
| 406 | s.str_inter_cbr_depth++ |
| 407 | } |
| 408 | return .lcbr |
| 409 | } |
| 410 | `}` { |
| 411 | if s.in_str_inter { |
| 412 | s.str_inter_cbr_depth-- |
| 413 | if s.str_inter_cbr_depth == 0 { |
| 414 | s.in_str_incomplete = true |
| 415 | s.in_str_inter = false |
| 416 | } |
| 417 | } |
| 418 | s.insert_semi = true |
| 419 | return .rcbr |
| 420 | } |
| 421 | `(` { |
| 422 | return .lpar |
| 423 | } |
| 424 | `)` { |
| 425 | s.insert_semi = true |
| 426 | return .rpar |
| 427 | } |
| 428 | `[` { |
| 429 | return .lsbr |
| 430 | } |
| 431 | `]` { |
| 432 | s.insert_semi = true |
| 433 | return .rsbr |
| 434 | } |
| 435 | `;` { |
| 436 | return .semicolon |
| 437 | } |
| 438 | `?` { |
| 439 | s.insert_semi = true |
| 440 | return .question |
| 441 | } |
| 442 | else { |
| 443 | return .unknown |
| 444 | } |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | @[direct_array_access] |
| 449 | fn (mut s Scanner) whitespace() { |
| 450 | for s.offset < s.src.len { |
| 451 | c := s.src[s.offset] |
| 452 | if c == ` ` || c == `\t` || c == `\r` { |
| 453 | s.offset++ |
| 454 | continue |
| 455 | } else if c == `\n` { |
| 456 | if s.insert_semi { |
| 457 | return |
| 458 | } |
| 459 | s.offset++ |
| 460 | continue |
| 461 | } |
| 462 | break |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | @[direct_array_access] |
| 467 | fn (mut s Scanner) line() { |
| 468 | for s.offset < s.src.len { |
| 469 | if s.src[s.offset] == `\n` { |
| 470 | break |
| 471 | } |
| 472 | s.offset++ |
| 473 | } |
| 474 | } |
| 475 | |
| 476 | @[direct_array_access] |
| 477 | fn (mut s Scanner) comment() { |
| 478 | s.offset++ |
| 479 | if s.offset >= s.src.len { |
| 480 | return |
| 481 | } |
| 482 | c := s.src[s.offset] |
| 483 | if c == `/` { |
| 484 | s.line() |
| 485 | } else if c == `*` { |
| 486 | s.offset++ |
| 487 | mut ml_comment_depth := 1 |
| 488 | for s.offset < s.src.len { |
| 489 | c2 := s.src[s.offset] |
| 490 | c3 := s.peek_byte(1) |
| 491 | if c2 == `\n` { |
| 492 | s.offset++ |
| 493 | } else if c2 == `/` && c3 == `*` { |
| 494 | s.offset += 2 |
| 495 | ml_comment_depth++ |
| 496 | } else if c2 == `*` && c3 == `/` { |
| 497 | s.offset += 2 |
| 498 | ml_comment_depth-- |
| 499 | if ml_comment_depth == 0 { |
| 500 | break |
| 501 | } |
| 502 | } else { |
| 503 | s.offset++ |
| 504 | } |
| 505 | } |
| 506 | } |
| 507 | } |
| 508 | |
| 509 | @[direct_array_access] |
| 510 | fn (mut s Scanner) string_literal(scan_as_raw bool, c_quote u8) { |
| 511 | if scan_as_raw { |
| 512 | for s.offset < s.src.len { |
| 513 | c := s.src[s.offset] |
| 514 | if c == c_quote { |
| 515 | break |
| 516 | } |
| 517 | if c == `\n` { |
| 518 | s.offset++ |
| 519 | continue |
| 520 | } |
| 521 | s.offset++ |
| 522 | } |
| 523 | if s.offset < s.src.len { |
| 524 | s.offset++ |
| 525 | } |
| 526 | return |
| 527 | } |
| 528 | for s.offset < s.src.len { |
| 529 | c := s.src[s.offset] |
| 530 | if c == `\\` { |
| 531 | s.offset += 2 |
| 532 | continue |
| 533 | } else if c == `\n` { |
| 534 | s.offset++ |
| 535 | continue |
| 536 | } else if c == `$` && s.peek_byte(1) == `{` { |
| 537 | s.in_str_inter = true |
| 538 | if s.skip_interpolation { |
| 539 | s.str_inter_cbr_depth++ |
| 540 | s.offset += 2 |
| 541 | continue |
| 542 | } else { |
| 543 | return |
| 544 | } |
| 545 | } else if s.skip_interpolation && s.in_str_inter { |
| 546 | if c == `{` { |
| 547 | s.str_inter_cbr_depth++ |
| 548 | } else if c == `}` { |
| 549 | s.str_inter_cbr_depth-- |
| 550 | if s.str_inter_cbr_depth == 0 { |
| 551 | s.in_str_inter = false |
| 552 | } |
| 553 | } |
| 554 | } else if c == c_quote && !s.in_str_inter { |
| 555 | s.offset++ |
| 556 | break |
| 557 | } |
| 558 | s.offset++ |
| 559 | } |
| 560 | } |
| 561 | |
| 562 | @[direct_array_access] |
| 563 | fn (mut s Scanner) number() { |
| 564 | if s.offset < s.src.len && s.src[s.offset] == `0` { |
| 565 | s.offset++ |
| 566 | c := s.peek_byte(0) |
| 567 | if c == `b` || c == `B` { |
| 568 | s.offset++ |
| 569 | for s.offset < s.src.len { |
| 570 | c2 := s.src[s.offset] |
| 571 | if c2 == `0` || c2 == `1` || c2 == `_` { |
| 572 | s.offset++ |
| 573 | continue |
| 574 | } |
| 575 | return |
| 576 | } |
| 577 | return |
| 578 | } else if c == `x` || c == `X` { |
| 579 | s.offset++ |
| 580 | for s.offset < s.src.len { |
| 581 | c2 := s.src[s.offset] |
| 582 | if (c2 >= `0` && c2 <= `9`) || (c2 >= `a` && c2 <= `f`) |
| 583 | || (c2 >= `A` && c2 <= `F`) || c2 == `_` { |
| 584 | s.offset++ |
| 585 | continue |
| 586 | } |
| 587 | return |
| 588 | } |
| 589 | return |
| 590 | } else if c == `o` || c == `O` { |
| 591 | s.offset++ |
| 592 | for s.offset < s.src.len { |
| 593 | c2 := s.src[s.offset] |
| 594 | if (c2 >= `0` && c2 <= `7`) || c2 == `_` { |
| 595 | s.offset++ |
| 596 | continue |
| 597 | } |
| 598 | return |
| 599 | } |
| 600 | return |
| 601 | } |
| 602 | } |
| 603 | mut has_decimal := false |
| 604 | mut has_exponent := false |
| 605 | for s.offset < s.src.len { |
| 606 | c := s.src[s.offset] |
| 607 | if (c >= `0` && c <= `9`) || c == `_` { |
| 608 | s.offset++ |
| 609 | continue |
| 610 | } else if !has_decimal && c == `.` { |
| 611 | next := s.peek_byte(1) |
| 612 | if next != `.` && next >= `0` && next <= `9` { |
| 613 | has_decimal = true |
| 614 | s.offset++ |
| 615 | continue |
| 616 | } |
| 617 | } else if !has_exponent && (c == `e` || c == `E`) { |
| 618 | has_exponent = true |
| 619 | s.offset++ |
| 620 | if s.offset < s.src.len && (s.src[s.offset] == `+` || s.src[s.offset] == `-`) { |
| 621 | s.offset++ |
| 622 | } |
| 623 | continue |
| 624 | } |
| 625 | break |
| 626 | } |
| 627 | } |
| 628 | |