1 module ctpg.dsl.typed.visitors; 2 3 import std.conv : to; 4 5 import ctpg.dsl.typed.token : Token, TokenType; 6 import ctpg.dsl.typed.node : Node; 7 8 import compile_time_unittest : enableCompileTimeUnittest; 9 mixin enableCompileTimeUnittest; 10 11 12 Node applySkipLiteral(Node node, bool isSkipLiteral = true) 13 { 14 final switch(node.token.type) 15 { 16 case TokenType.DOLLAR: 17 case TokenType.RANGE: 18 case TokenType.STRING: 19 if(isSkipLiteral) 20 { 21 Node skipNode; 22 skipNode.token.type = TokenType.SKIP; 23 skipNode.children ~= node; 24 25 node = skipNode; 26 } 27 break; 28 29 case TokenType.SKIP_LITERAL_TRUE: 30 node = node.children[0].applySkipLiteral(true); 31 break; 32 33 case TokenType.SKIP_LITERAL_FALSE: 34 node = node.children[0].applySkipLiteral(false); 35 break; 36 37 38 case TokenType.DEFINITIONS: 39 foreach(ref child; node.children) 40 { 41 switch(child.token.type) 42 { 43 case TokenType.GLOBAL_SKIP_LITERAL_TRUE: 44 isSkipLiteral = true; 45 break; 46 47 case TokenType.GLOBAL_SKIP_LITERAL_FALSE: 48 isSkipLiteral = false; 49 break; 50 51 default: 52 child = child.applySkipLiteral(isSkipLiteral); 53 } 54 } 55 break; 56 57 case TokenType.DEFINITION: 58 case TokenType.SLASH: 59 case TokenType.LEFT_QUESTION: 60 case TokenType.LEFT_SHIFT: 61 case TokenType.SEQUENCE: 62 case TokenType.QUESTION: 63 case TokenType.PLUS: 64 case TokenType.ASTERISK: 65 case TokenType.ASCIICIRCUM: 66 case TokenType.ANPERSAND: 67 case TokenType.EXCLAM: 68 69 case TokenType.SKIP: 70 case TokenType.MEMOIZE: 71 72 case TokenType.MEMOIZE_SKIP_TRUE: 73 case TokenType.MEMOIZE_SKIP_FALSE: 74 case TokenType.MEMOIZE_LITERAL_TRUE: 75 case TokenType.MEMOIZE_LITERAL_FALSE: 76 case TokenType.MEMOIZE_NONTERMINAL_TRUE: 77 case TokenType.MEMOIZE_NONTERMINAL_FALSE: 78 foreach(ref child; node.children) 79 { 80 child = child.applySkipLiteral(isSkipLiteral); 81 } 82 break; 83 84 case TokenType.SKIP_WITH: 85 case TokenType.SET_SKIP: 86 node.children[1] = node.children[1].applySkipLiteral(isSkipLiteral); 87 break; 88 89 case TokenType.GLOBAL_SKIP_LITERAL_TRUE: 90 case TokenType.GLOBAL_SKIP_LITERAL_FALSE: 91 assert(false); 92 93 case TokenType.UNDEFINED: 94 case TokenType.CONVERTER: 95 case TokenType.ID: 96 case TokenType.TEMPLATE_INSTANCE: 97 case TokenType.NONTERMINAL: 98 case TokenType.RANGE_ONE_CHAR: 99 case TokenType.RANGE_CHAR_RANGE: 100 101 case TokenType.GLOBAL_SET_SKIP: 102 103 case TokenType.GLOBAL_MEMOIZE_SKIP_TRUE: 104 case TokenType.GLOBAL_MEMOIZE_SKIP_FALSE: 105 case TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE: 106 case TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE: 107 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 108 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 109 } 110 111 return node; 112 } 113 114 debug(ctpg) unittest 115 { 116 assert 117 ( 118 Node(Token(TokenType.DEFINITIONS), 119 [ 120 Node(Token(TokenType.DEFINITION), 121 [ 122 Node(Token(TokenType.TEMPLATE_INSTANCE)), 123 Node(Token(TokenType.ID)), 124 Node(Token(TokenType.DOLLAR)) 125 ]) 126 ]).applySkipLiteral() 127 128 == 129 130 Node(Token(TokenType.DEFINITIONS), 131 [ 132 Node(Token(TokenType.DEFINITION), 133 [ 134 Node(Token(TokenType.TEMPLATE_INSTANCE)), 135 Node(Token(TokenType.ID)), 136 Node(Token(TokenType.SKIP), 137 [ 138 Node(Token(TokenType.DOLLAR)) 139 ]) 140 ]) 141 ]) 142 ); 143 144 assert 145 ( 146 Node(Token(TokenType.DEFINITIONS), 147 [ 148 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_FALSE)), 149 Node(Token(TokenType.DEFINITION), 150 [ 151 Node(Token(TokenType.TEMPLATE_INSTANCE)), 152 Node(Token(TokenType.ID)), 153 Node(Token(TokenType.DOLLAR)) 154 ]) 155 ]).applySkipLiteral() 156 157 == 158 159 Node(Token(TokenType.DEFINITIONS), 160 [ 161 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_FALSE)), 162 Node(Token(TokenType.DEFINITION), 163 [ 164 Node(Token(TokenType.TEMPLATE_INSTANCE)), 165 Node(Token(TokenType.ID)), 166 Node(Token(TokenType.DOLLAR)) 167 ]) 168 ]) 169 ); 170 171 assert 172 ( 173 Node(Token(TokenType.DEFINITIONS), 174 [ 175 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_FALSE)), 176 Node(Token(TokenType.DEFINITION), 177 [ 178 Node(Token(TokenType.TEMPLATE_INSTANCE)), 179 Node(Token(TokenType.ID)), 180 Node(Token(TokenType.SKIP_LITERAL_TRUE), 181 [ 182 Node(Token(TokenType.DOLLAR)) 183 ]) 184 ]) 185 ]).applySkipLiteral() 186 187 == 188 189 Node(Token(TokenType.DEFINITIONS), 190 [ 191 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_FALSE)), 192 Node(Token(TokenType.DEFINITION), 193 [ 194 Node(Token(TokenType.TEMPLATE_INSTANCE)), 195 Node(Token(TokenType.ID)), 196 Node(Token(TokenType.SKIP), 197 [ 198 Node(Token(TokenType.DOLLAR)) 199 ]) 200 ]) 201 ]) 202 ); 203 204 assert 205 ( 206 Node(Token(TokenType.DEFINITIONS), 207 [ 208 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_FALSE)), 209 Node(Token(TokenType.DEFINITION), 210 [ 211 Node(Token(TokenType.TEMPLATE_INSTANCE)), 212 Node(Token(TokenType.ID)), 213 Node(Token(TokenType.DOLLAR)) 214 ]), 215 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_TRUE)), 216 Node(Token(TokenType.DEFINITION), 217 [ 218 Node(Token(TokenType.TEMPLATE_INSTANCE)), 219 Node(Token(TokenType.ID)), 220 Node(Token(TokenType.DOLLAR)) 221 ]) 222 ]).applySkipLiteral() 223 224 == 225 226 Node(Token(TokenType.DEFINITIONS), 227 [ 228 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_FALSE)), 229 Node(Token(TokenType.DEFINITION), 230 [ 231 Node(Token(TokenType.TEMPLATE_INSTANCE)), 232 Node(Token(TokenType.ID)), 233 Node(Token(TokenType.DOLLAR)) 234 ]), 235 Node(Token(TokenType.GLOBAL_SKIP_LITERAL_TRUE)), 236 Node(Token(TokenType.DEFINITION), 237 [ 238 Node(Token(TokenType.TEMPLATE_INSTANCE)), 239 Node(Token(TokenType.ID)), 240 Node(Token(TokenType.SKIP), 241 [ 242 Node(Token(TokenType.DOLLAR)) 243 ]) 244 ]) 245 ]) 246 ); 247 248 assert 249 ( 250 Node(Token(TokenType.DEFINITIONS), 251 [ 252 Node(Token(TokenType.DEFINITION), 253 [ 254 Node(Token(TokenType.TEMPLATE_INSTANCE)), 255 Node(Token(TokenType.ID)), 256 Node(Token(TokenType.SET_SKIP), 257 [ 258 Node(Token(TokenType.DOLLAR)), 259 Node(Token(TokenType.DOLLAR)) 260 ]) 261 ]) 262 ]).applySkipLiteral() 263 264 == 265 266 Node(Token(TokenType.DEFINITIONS), 267 [ 268 Node(Token(TokenType.DEFINITION), 269 [ 270 Node(Token(TokenType.TEMPLATE_INSTANCE)), 271 Node(Token(TokenType.ID)), 272 Node(Token(TokenType.SET_SKIP), 273 [ 274 Node(Token(TokenType.DOLLAR)), 275 Node(Token(TokenType.SKIP), 276 [ 277 Node(Token(TokenType.DOLLAR)) 278 ]) 279 ]) 280 ]) 281 ]) 282 ); 283 284 assert 285 ( 286 Node(Token(TokenType.DEFINITIONS), 287 [ 288 Node(Token(TokenType.DEFINITION), 289 [ 290 Node(Token(TokenType.TEMPLATE_INSTANCE)), 291 Node(Token(TokenType.ID)), 292 Node(Token(TokenType.SKIP_WITH), 293 [ 294 Node(Token(TokenType.DOLLAR)), 295 Node(Token(TokenType.DOLLAR)) 296 ]) 297 ]) 298 ]).applySkipLiteral() 299 300 == 301 302 Node(Token(TokenType.DEFINITIONS), 303 [ 304 Node(Token(TokenType.DEFINITION), 305 [ 306 Node(Token(TokenType.TEMPLATE_INSTANCE)), 307 Node(Token(TokenType.ID)), 308 Node(Token(TokenType.SKIP_WITH), 309 [ 310 Node(Token(TokenType.DOLLAR)), 311 Node(Token(TokenType.SKIP), 312 [ 313 Node(Token(TokenType.DOLLAR)) 314 ]) 315 ]) 316 ]) 317 ]) 318 ); 319 } 320 321 322 // MEMOIZE_SKIP_TRUEな場所にある 323 // SET_SKIPやSKIP_WITHで指定されてるスキップパーサをMEMOIZEする 324 Node applyMemoizeSkip(Node node, bool isMemoizeSkip = true) 325 { 326 final switch(node.token.type) 327 { 328 case TokenType.SET_SKIP: 329 case TokenType.SKIP_WITH: 330 node.children[1] = node.children[1].applyMemoizeSkip(isMemoizeSkip); 331 goto case; 332 333 case TokenType.GLOBAL_SET_SKIP: 334 if(isMemoizeSkip) 335 { 336 Node memoizeNode; 337 memoizeNode.token.type = TokenType.MEMOIZE; 338 memoizeNode.children ~= node.children[0]; 339 node.children[0] = memoizeNode; 340 } 341 break; 342 343 case TokenType.MEMOIZE_SKIP_TRUE: 344 node = node.children[0].applyMemoizeSkip(true); 345 break; 346 347 case TokenType.MEMOIZE_SKIP_FALSE: 348 node = node.children[0].applyMemoizeSkip(false); 349 break; 350 351 case TokenType.DEFINITIONS: 352 foreach(ref child; node.children) 353 { 354 switch(child.token.type) 355 { 356 case TokenType.GLOBAL_MEMOIZE_SKIP_TRUE: 357 isMemoizeSkip = true; 358 break; 359 360 case TokenType.GLOBAL_MEMOIZE_SKIP_FALSE: 361 isMemoizeSkip = false; 362 break; 363 364 default: 365 child = child.applyMemoizeSkip(isMemoizeSkip); 366 } 367 } 368 break; 369 370 case TokenType.DEFINITION: 371 case TokenType.SLASH: 372 case TokenType.LEFT_QUESTION: 373 case TokenType.LEFT_SHIFT: 374 case TokenType.SEQUENCE: 375 case TokenType.QUESTION: 376 case TokenType.PLUS: 377 case TokenType.ASTERISK: 378 case TokenType.ASCIICIRCUM: 379 case TokenType.ANPERSAND: 380 case TokenType.EXCLAM: 381 382 case TokenType.SKIP: 383 case TokenType.MEMOIZE: 384 385 case TokenType.SKIP_LITERAL_TRUE: 386 case TokenType.SKIP_LITERAL_FALSE: 387 case TokenType.MEMOIZE_LITERAL_TRUE: 388 case TokenType.MEMOIZE_LITERAL_FALSE: 389 case TokenType.MEMOIZE_NONTERMINAL_TRUE: 390 case TokenType.MEMOIZE_NONTERMINAL_FALSE: 391 foreach(ref child; node.children) 392 { 393 child = child.applyMemoizeSkip(isMemoizeSkip); 394 } 395 break; 396 397 case TokenType.GLOBAL_MEMOIZE_SKIP_TRUE: 398 case TokenType.GLOBAL_MEMOIZE_SKIP_FALSE: 399 assert(false); 400 401 case TokenType.UNDEFINED: 402 case TokenType.CONVERTER: 403 case TokenType.ID: 404 case TokenType.DOLLAR: 405 case TokenType.TEMPLATE_INSTANCE: 406 case TokenType.NONTERMINAL: 407 case TokenType.RANGE_ONE_CHAR: 408 case TokenType.RANGE_CHAR_RANGE: 409 case TokenType.RANGE: 410 case TokenType.STRING: 411 412 case TokenType.GLOBAL_SKIP_LITERAL_TRUE: 413 case TokenType.GLOBAL_SKIP_LITERAL_FALSE: 414 case TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE: 415 case TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE: 416 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 417 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 418 } 419 420 return node; 421 } 422 423 debug(ctpg) unittest 424 { 425 assert 426 ( 427 Node(Token(TokenType.DEFINITIONS), 428 [ 429 Node(Token(TokenType.DEFINITION), 430 [ 431 Node(Token(TokenType.TEMPLATE_INSTANCE)), 432 Node(Token(TokenType.ID)), 433 Node(Token(TokenType.SET_SKIP), 434 [ 435 Node(Token(TokenType.DOLLAR)), 436 Node(Token(TokenType.DOLLAR)) 437 ]) 438 ]) 439 ]).applyMemoizeSkip() 440 441 == 442 443 Node(Token(TokenType.DEFINITIONS), 444 [ 445 Node(Token(TokenType.DEFINITION), 446 [ 447 Node(Token(TokenType.TEMPLATE_INSTANCE)), 448 Node(Token(TokenType.ID)), 449 Node(Token(TokenType.SET_SKIP), 450 [ 451 Node(Token(TokenType.MEMOIZE), 452 [ 453 Node(Token(TokenType.DOLLAR)) 454 ]), 455 Node(Token(TokenType.DOLLAR)) 456 ]) 457 ]) 458 ]) 459 ); 460 461 assert 462 ( 463 Node(Token(TokenType.DEFINITIONS), 464 [ 465 Node(Token(TokenType.DEFINITION), 466 [ 467 Node(Token(TokenType.TEMPLATE_INSTANCE)), 468 Node(Token(TokenType.ID)), 469 Node(Token(TokenType.SET_SKIP), 470 [ 471 Node(Token(TokenType.DOLLAR)), 472 Node(Token(TokenType.SET_SKIP), 473 [ 474 Node(Token(TokenType.DOLLAR)), 475 Node(Token(TokenType.DOLLAR)) 476 ]) 477 ]) 478 ]) 479 ]).applyMemoizeSkip() 480 481 == 482 483 Node(Token(TokenType.DEFINITIONS), 484 [ 485 Node(Token(TokenType.DEFINITION), 486 [ 487 Node(Token(TokenType.TEMPLATE_INSTANCE)), 488 Node(Token(TokenType.ID)), 489 Node(Token(TokenType.SET_SKIP), 490 [ 491 Node(Token(TokenType.MEMOIZE), 492 [ 493 Node(Token(TokenType.DOLLAR)) 494 ]), 495 Node(Token(TokenType.SET_SKIP), 496 [ 497 Node(Token(TokenType.MEMOIZE), 498 [ 499 Node(Token(TokenType.DOLLAR)) 500 ]), 501 Node(Token(TokenType.DOLLAR)) 502 ]) 503 ]) 504 ]) 505 ]) 506 ); 507 508 assert 509 ( 510 Node(Token(TokenType.DEFINITIONS), 511 [ 512 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_FALSE)), 513 Node(Token(TokenType.DEFINITION), 514 [ 515 Node(Token(TokenType.TEMPLATE_INSTANCE)), 516 Node(Token(TokenType.ID)), 517 Node(Token(TokenType.SET_SKIP), 518 [ 519 Node(Token(TokenType.DOLLAR)), 520 Node(Token(TokenType.DOLLAR)) 521 ]) 522 ]) 523 ]).applyMemoizeSkip() 524 525 == 526 527 Node(Token(TokenType.DEFINITIONS), 528 [ 529 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_FALSE)), 530 Node(Token(TokenType.DEFINITION), 531 [ 532 Node(Token(TokenType.TEMPLATE_INSTANCE)), 533 Node(Token(TokenType.ID)), 534 Node(Token(TokenType.SET_SKIP), 535 [ 536 Node(Token(TokenType.DOLLAR)), 537 Node(Token(TokenType.DOLLAR)) 538 ]) 539 ]) 540 ]) 541 ); 542 543 assert 544 ( 545 Node(Token(TokenType.DEFINITIONS), 546 [ 547 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_FALSE)), 548 Node(Token(TokenType.DEFINITION), 549 [ 550 Node(Token(TokenType.TEMPLATE_INSTANCE)), 551 Node(Token(TokenType.ID)), 552 Node(Token(TokenType.MEMOIZE_SKIP_TRUE), 553 [ 554 Node(Token(TokenType.SET_SKIP), 555 [ 556 Node(Token(TokenType.DOLLAR)), 557 Node(Token(TokenType.DOLLAR)) 558 ]) 559 ]) 560 ]) 561 ]).applyMemoizeSkip() 562 563 == 564 565 Node(Token(TokenType.DEFINITIONS), 566 [ 567 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_FALSE)), 568 Node(Token(TokenType.DEFINITION), 569 [ 570 Node(Token(TokenType.TEMPLATE_INSTANCE)), 571 Node(Token(TokenType.ID)), 572 Node(Token(TokenType.SET_SKIP), 573 [ 574 Node(Token(TokenType.MEMOIZE), 575 [ 576 Node(Token(TokenType.DOLLAR)) 577 ]), 578 Node(Token(TokenType.DOLLAR)) 579 ]) 580 ]) 581 ]) 582 ); 583 584 assert 585 ( 586 Node(Token(TokenType.DEFINITIONS), 587 [ 588 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_FALSE)), 589 Node(Token(TokenType.DEFINITION), 590 [ 591 Node(Token(TokenType.TEMPLATE_INSTANCE)), 592 Node(Token(TokenType.ID)), 593 Node(Token(TokenType.SET_SKIP), 594 [ 595 Node(Token(TokenType.DOLLAR)), 596 Node(Token(TokenType.DOLLAR)) 597 ]) 598 ]), 599 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_TRUE)), 600 Node(Token(TokenType.DEFINITION), 601 [ 602 Node(Token(TokenType.TEMPLATE_INSTANCE)), 603 Node(Token(TokenType.ID)), 604 Node(Token(TokenType.SET_SKIP), 605 [ 606 Node(Token(TokenType.DOLLAR)), 607 Node(Token(TokenType.DOLLAR)) 608 ]) 609 ]), 610 ]).applyMemoizeSkip() 611 612 == 613 614 Node(Token(TokenType.DEFINITIONS), 615 [ 616 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_FALSE)), 617 Node(Token(TokenType.DEFINITION), 618 [ 619 Node(Token(TokenType.TEMPLATE_INSTANCE)), 620 Node(Token(TokenType.ID)), 621 Node(Token(TokenType.SET_SKIP), 622 [ 623 Node(Token(TokenType.DOLLAR)), 624 Node(Token(TokenType.DOLLAR)) 625 ]) 626 ]), 627 Node(Token(TokenType.GLOBAL_MEMOIZE_SKIP_TRUE)), 628 Node(Token(TokenType.DEFINITION), 629 [ 630 Node(Token(TokenType.TEMPLATE_INSTANCE)), 631 Node(Token(TokenType.ID)), 632 Node(Token(TokenType.SET_SKIP), 633 [ 634 Node(Token(TokenType.MEMOIZE), 635 [ 636 Node(Token(TokenType.DOLLAR)) 637 ]), 638 Node(Token(TokenType.DOLLAR)) 639 ]) 640 ]), 641 ]) 642 ); 643 } 644 645 646 // MEMOIZE_LITERAL_TRUEな場所にLITERALがあったらMEMOIZEの下に入れる 647 Node applyMemoizeLiteral(Node node, bool isMemoizeLiteral = true) 648 { 649 final switch(node.token.type) 650 { 651 case TokenType.DOLLAR: 652 case TokenType.RANGE: 653 case TokenType.STRING: 654 if(isMemoizeLiteral) 655 { 656 Node memoizeNode; 657 memoizeNode.token.type = TokenType.MEMOIZE; 658 memoizeNode.children ~= node; 659 660 node = memoizeNode; 661 } 662 break; 663 664 case TokenType.MEMOIZE_LITERAL_TRUE: 665 node = node.children[0].applyMemoizeLiteral(true); 666 break; 667 668 case TokenType.MEMOIZE_LITERAL_FALSE: 669 node = node.children[0].applyMemoizeLiteral(false); 670 break; 671 672 case TokenType.DEFINITIONS: 673 foreach(ref child; node.children) 674 { 675 switch(child.token.type) 676 { 677 case TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE: 678 isMemoizeLiteral = true; 679 break; 680 681 case TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE: 682 isMemoizeLiteral = false; 683 break; 684 685 default: 686 child = child.applyMemoizeLiteral(isMemoizeLiteral); 687 } 688 } 689 break; 690 691 case TokenType.DEFINITION: 692 case TokenType.SLASH: 693 case TokenType.LEFT_QUESTION: 694 case TokenType.LEFT_SHIFT: 695 case TokenType.SEQUENCE: 696 case TokenType.QUESTION: 697 case TokenType.PLUS: 698 case TokenType.ASTERISK: 699 case TokenType.ASCIICIRCUM: 700 case TokenType.ANPERSAND: 701 case TokenType.EXCLAM: 702 703 case TokenType.SKIP: 704 case TokenType.SKIP_WITH: 705 case TokenType.SET_SKIP: 706 case TokenType.MEMOIZE: 707 708 case TokenType.SKIP_LITERAL_TRUE: 709 case TokenType.SKIP_LITERAL_FALSE: 710 case TokenType.MEMOIZE_SKIP_TRUE: 711 case TokenType.MEMOIZE_SKIP_FALSE: 712 case TokenType.MEMOIZE_NONTERMINAL_TRUE: 713 case TokenType.MEMOIZE_NONTERMINAL_FALSE: 714 715 case TokenType.GLOBAL_SET_SKIP: 716 foreach(ref child; node.children) 717 { 718 child = child.applyMemoizeLiteral(isMemoizeLiteral); 719 } 720 break; 721 722 case TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE: 723 case TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE: 724 assert(false); 725 726 case TokenType.UNDEFINED: 727 case TokenType.CONVERTER: 728 case TokenType.ID: 729 case TokenType.TEMPLATE_INSTANCE: 730 case TokenType.NONTERMINAL: 731 case TokenType.RANGE_ONE_CHAR: 732 case TokenType.RANGE_CHAR_RANGE: 733 734 case TokenType.GLOBAL_SKIP_LITERAL_TRUE: 735 case TokenType.GLOBAL_SKIP_LITERAL_FALSE: 736 case TokenType.GLOBAL_MEMOIZE_SKIP_TRUE: 737 case TokenType.GLOBAL_MEMOIZE_SKIP_FALSE: 738 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 739 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 740 } 741 742 return node; 743 } 744 745 debug(ctpg) unittest 746 { 747 assert 748 ( 749 Node(Token(TokenType.DEFINITIONS), 750 [ 751 Node(Token(TokenType.DEFINITION), 752 [ 753 Node(Token(TokenType.TEMPLATE_INSTANCE)), 754 Node(Token(TokenType.ID)), 755 Node(Token(TokenType.DOLLAR)) 756 ]) 757 ]).applyMemoizeLiteral() 758 759 == 760 761 Node(Token(TokenType.DEFINITIONS), 762 [ 763 Node(Token(TokenType.DEFINITION), 764 [ 765 Node(Token(TokenType.TEMPLATE_INSTANCE)), 766 Node(Token(TokenType.ID)), 767 Node(Token(TokenType.MEMOIZE), 768 [ 769 Node(Token(TokenType.DOLLAR)) 770 ]) 771 ]) 772 ]) 773 ); 774 775 assert 776 ( 777 Node(Token(TokenType.DEFINITIONS), 778 [ 779 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 780 Node(Token(TokenType.DEFINITION), 781 [ 782 Node(Token(TokenType.TEMPLATE_INSTANCE)), 783 Node(Token(TokenType.ID)), 784 Node(Token(TokenType.DOLLAR)) 785 ]) 786 ]).applyMemoizeLiteral() 787 788 == 789 790 Node(Token(TokenType.DEFINITIONS), 791 [ 792 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 793 Node(Token(TokenType.DEFINITION), 794 [ 795 Node(Token(TokenType.TEMPLATE_INSTANCE)), 796 Node(Token(TokenType.ID)), 797 Node(Token(TokenType.DOLLAR)) 798 ]) 799 ]) 800 ); 801 802 assert 803 ( 804 Node(Token(TokenType.DEFINITIONS), 805 [ 806 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 807 Node(Token(TokenType.DEFINITION), 808 [ 809 Node(Token(TokenType.TEMPLATE_INSTANCE)), 810 Node(Token(TokenType.ID)), 811 Node(Token(TokenType.MEMOIZE_LITERAL_TRUE), 812 [ 813 Node(Token(TokenType.DOLLAR)) 814 ]) 815 ]) 816 ]).applyMemoizeLiteral() 817 818 == 819 820 Node(Token(TokenType.DEFINITIONS), 821 [ 822 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 823 Node(Token(TokenType.DEFINITION), 824 [ 825 Node(Token(TokenType.TEMPLATE_INSTANCE)), 826 Node(Token(TokenType.ID)), 827 Node(Token(TokenType.MEMOIZE), 828 [ 829 Node(Token(TokenType.DOLLAR)) 830 ]) 831 ]) 832 ]) 833 ); 834 835 assert 836 ( 837 Node(Token(TokenType.DEFINITIONS), 838 [ 839 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 840 Node(Token(TokenType.DEFINITION), 841 [ 842 Node(Token(TokenType.TEMPLATE_INSTANCE)), 843 Node(Token(TokenType.ID)), 844 Node(Token(TokenType.DOLLAR)) 845 ]), 846 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE)), 847 Node(Token(TokenType.DEFINITION), 848 [ 849 Node(Token(TokenType.TEMPLATE_INSTANCE)), 850 Node(Token(TokenType.ID)), 851 Node(Token(TokenType.DOLLAR)) 852 ]) 853 ]).applyMemoizeLiteral() 854 855 == 856 857 Node(Token(TokenType.DEFINITIONS), 858 [ 859 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 860 Node(Token(TokenType.DEFINITION), 861 [ 862 Node(Token(TokenType.TEMPLATE_INSTANCE)), 863 Node(Token(TokenType.ID)), 864 Node(Token(TokenType.DOLLAR)) 865 ]), 866 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE)), 867 Node(Token(TokenType.DEFINITION), 868 [ 869 Node(Token(TokenType.TEMPLATE_INSTANCE)), 870 Node(Token(TokenType.ID)), 871 Node(Token(TokenType.MEMOIZE), 872 [ 873 Node(Token(TokenType.DOLLAR)) 874 ]) 875 ]) 876 ]) 877 ); 878 879 assert 880 ( 881 Node(Token(TokenType.DEFINITIONS), 882 [ 883 Node(Token(TokenType.GLOBAL_SET_SKIP), 884 [ 885 Node(Token(TokenType.DOLLAR)) 886 ]), 887 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 888 Node(Token(TokenType.DEFINITION), 889 [ 890 Node(Token(TokenType.TEMPLATE_INSTANCE)), 891 Node(Token(TokenType.ID)), 892 Node(Token(TokenType.DOLLAR)) 893 ]) 894 ]).applyMemoizeLiteral() 895 896 == 897 898 Node(Token(TokenType.DEFINITIONS), 899 [ 900 Node(Token(TokenType.GLOBAL_SET_SKIP), 901 [ 902 Node(Token(TokenType.MEMOIZE), 903 [ 904 Node(Token(TokenType.DOLLAR)) 905 ]) 906 ]), 907 Node(Token(TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE)), 908 Node(Token(TokenType.DEFINITION), 909 [ 910 Node(Token(TokenType.TEMPLATE_INSTANCE)), 911 Node(Token(TokenType.ID)), 912 Node(Token(TokenType.DOLLAR)) 913 ]) 914 ]) 915 ); 916 } 917 918 919 // MEMOIZE_NONTERMINAL_TRUEな場所にNONTERMINALがあったらMEMOIZEの下に入れる 920 Node applyMemoizeNonterminal(Node node, bool isMemoizeNonterminal = true) 921 { 922 final switch(node.token.type) 923 { 924 case TokenType.NONTERMINAL: 925 if(isMemoizeNonterminal) 926 { 927 Node memoizeNode; 928 929 memoizeNode.token.type = TokenType.MEMOIZE; 930 memoizeNode.children ~= node; 931 932 node = memoizeNode; 933 } 934 break; 935 936 case TokenType.MEMOIZE_NONTERMINAL_TRUE: 937 node = node.children[0].applyMemoizeNonterminal(true); 938 break; 939 940 case TokenType.MEMOIZE_NONTERMINAL_FALSE: 941 node = node.children[0].applyMemoizeNonterminal(false); 942 break; 943 944 case TokenType.DEFINITIONS: 945 foreach(ref child; node.children) 946 { 947 switch(child.token.type) 948 { 949 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 950 isMemoizeNonterminal = true; 951 break; 952 953 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 954 isMemoizeNonterminal = false; 955 break; 956 957 default: 958 child = child.applyMemoizeNonterminal(isMemoizeNonterminal); 959 } 960 } 961 break; 962 963 case TokenType.DEFINITION: 964 case TokenType.SLASH: 965 case TokenType.LEFT_QUESTION: 966 case TokenType.LEFT_SHIFT: 967 case TokenType.SEQUENCE: 968 case TokenType.QUESTION: 969 case TokenType.PLUS: 970 case TokenType.ASTERISK: 971 case TokenType.ASCIICIRCUM: 972 case TokenType.ANPERSAND: 973 case TokenType.EXCLAM: 974 975 case TokenType.SKIP: 976 case TokenType.SKIP_WITH: 977 case TokenType.SET_SKIP: 978 case TokenType.MEMOIZE: 979 980 case TokenType.SKIP_LITERAL_TRUE: 981 case TokenType.SKIP_LITERAL_FALSE: 982 case TokenType.MEMOIZE_SKIP_TRUE: 983 case TokenType.MEMOIZE_SKIP_FALSE: 984 case TokenType.MEMOIZE_LITERAL_TRUE: 985 case TokenType.MEMOIZE_LITERAL_FALSE: 986 987 case TokenType.GLOBAL_SET_SKIP: 988 foreach(ref child; node.children) 989 { 990 child = child.applyMemoizeNonterminal(isMemoizeNonterminal); 991 } 992 break; 993 994 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 995 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 996 assert(false); 997 998 case TokenType.UNDEFINED: 999 case TokenType.CONVERTER: 1000 case TokenType.ID: 1001 case TokenType.DOLLAR: 1002 case TokenType.TEMPLATE_INSTANCE: 1003 case TokenType.RANGE_ONE_CHAR: 1004 case TokenType.RANGE_CHAR_RANGE: 1005 case TokenType.RANGE: 1006 case TokenType.STRING: 1007 1008 case TokenType.GLOBAL_SKIP_LITERAL_TRUE: 1009 case TokenType.GLOBAL_SKIP_LITERAL_FALSE: 1010 case TokenType.GLOBAL_MEMOIZE_SKIP_TRUE: 1011 case TokenType.GLOBAL_MEMOIZE_SKIP_FALSE: 1012 case TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE: 1013 case TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE: 1014 } 1015 1016 return node; 1017 } 1018 1019 debug(ctpg) unittest 1020 { 1021 assert 1022 ( 1023 Node(Token(TokenType.DEFINITIONS), 1024 [ 1025 Node(Token(TokenType.DEFINITION), 1026 [ 1027 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1028 Node(Token(TokenType.ID)), 1029 Node(Token(TokenType.NONTERMINAL)) 1030 ]) 1031 ]).applyMemoizeNonterminal() 1032 1033 == 1034 1035 Node(Token(TokenType.DEFINITIONS), 1036 [ 1037 Node(Token(TokenType.DEFINITION), 1038 [ 1039 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1040 Node(Token(TokenType.ID)), 1041 Node(Token(TokenType.MEMOIZE), 1042 [ 1043 Node(Token(TokenType.NONTERMINAL)) 1044 ]) 1045 ]) 1046 ]) 1047 ); 1048 1049 assert 1050 ( 1051 Node(Token(TokenType.DEFINITIONS), 1052 [ 1053 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1054 Node(Token(TokenType.DEFINITION), 1055 [ 1056 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1057 Node(Token(TokenType.ID)), 1058 Node(Token(TokenType.NONTERMINAL)) 1059 ]) 1060 ]).applyMemoizeNonterminal() 1061 1062 == 1063 1064 Node(Token(TokenType.DEFINITIONS), 1065 [ 1066 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1067 Node(Token(TokenType.DEFINITION), 1068 [ 1069 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1070 Node(Token(TokenType.ID)), 1071 Node(Token(TokenType.NONTERMINAL)) 1072 ]) 1073 ]) 1074 ); 1075 1076 assert 1077 ( 1078 Node(Token(TokenType.DEFINITIONS), 1079 [ 1080 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1081 Node(Token(TokenType.DEFINITION), 1082 [ 1083 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1084 Node(Token(TokenType.ID)), 1085 Node(Token(TokenType.MEMOIZE_NONTERMINAL_TRUE), 1086 [ 1087 Node(Token(TokenType.NONTERMINAL)) 1088 ]) 1089 ]) 1090 ]).applyMemoizeNonterminal() 1091 1092 == 1093 1094 Node(Token(TokenType.DEFINITIONS), 1095 [ 1096 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1097 Node(Token(TokenType.DEFINITION), 1098 [ 1099 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1100 Node(Token(TokenType.ID)), 1101 Node(Token(TokenType.MEMOIZE), 1102 [ 1103 Node(Token(TokenType.NONTERMINAL)) 1104 ]) 1105 ]) 1106 ]) 1107 ); 1108 1109 assert 1110 ( 1111 Node(Token(TokenType.DEFINITIONS), 1112 [ 1113 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1114 Node(Token(TokenType.DEFINITION), 1115 [ 1116 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1117 Node(Token(TokenType.ID)), 1118 Node(Token(TokenType.NONTERMINAL)) 1119 ]), 1120 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE)), 1121 Node(Token(TokenType.DEFINITION), 1122 [ 1123 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1124 Node(Token(TokenType.ID)), 1125 Node(Token(TokenType.NONTERMINAL)) 1126 ]) 1127 ]).applyMemoizeNonterminal() 1128 1129 == 1130 1131 Node(Token(TokenType.DEFINITIONS), 1132 [ 1133 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1134 Node(Token(TokenType.DEFINITION), 1135 [ 1136 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1137 Node(Token(TokenType.ID)), 1138 Node(Token(TokenType.NONTERMINAL)) 1139 ]), 1140 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE)), 1141 Node(Token(TokenType.DEFINITION), 1142 [ 1143 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1144 Node(Token(TokenType.ID)), 1145 Node(Token(TokenType.MEMOIZE), 1146 [ 1147 Node(Token(TokenType.NONTERMINAL)) 1148 ]) 1149 ]) 1150 ]) 1151 ); 1152 1153 assert 1154 ( 1155 Node(Token(TokenType.DEFINITIONS), 1156 [ 1157 Node(Token(TokenType.GLOBAL_SET_SKIP), 1158 [ 1159 Node(Token(TokenType.NONTERMINAL)) 1160 ]), 1161 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1162 Node(Token(TokenType.DEFINITION), 1163 [ 1164 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1165 Node(Token(TokenType.ID)), 1166 Node(Token(TokenType.NONTERMINAL)) 1167 ]) 1168 ]).applyMemoizeNonterminal() 1169 1170 == 1171 1172 Node(Token(TokenType.DEFINITIONS), 1173 [ 1174 Node(Token(TokenType.GLOBAL_SET_SKIP), 1175 [ 1176 Node(Token(TokenType.MEMOIZE), 1177 [ 1178 Node(Token(TokenType.NONTERMINAL)) 1179 ]) 1180 ]), 1181 Node(Token(TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE)), 1182 Node(Token(TokenType.DEFINITION), 1183 [ 1184 Node(Token(TokenType.TEMPLATE_INSTANCE)), 1185 Node(Token(TokenType.ID)), 1186 Node(Token(TokenType.NONTERMINAL)) 1187 ]) 1188 ]) 1189 ); 1190 } 1191 1192 1193 // SET_SKIPで指定されたスキップパーサでSKIPをSKIP_WITHに置換する 1194 Node expandSkip(Node node, Node skip = Node.init) 1195 { 1196 final switch(node.token.type) 1197 { 1198 case TokenType.SKIP: 1199 if(skip.token.type != TokenType.UNDEFINED) 1200 { 1201 Node skipWithNode; 1202 1203 skipWithNode.token.type = TokenType.SKIP_WITH; 1204 skipWithNode.children ~= skip; 1205 skipWithNode.children ~= node.children[0]; 1206 1207 node = skipWithNode; 1208 } 1209 else 1210 { 1211 node = node.children[0]; 1212 } 1213 break; 1214 1215 case TokenType.SET_SKIP: 1216 node = node.children[1].expandSkip(node.children[0]); 1217 break; 1218 1219 case TokenType.SKIP_WITH: 1220 node.children[1] = node.children[1].expandSkip(skip); 1221 break; 1222 1223 case TokenType.DEFINITIONS: 1224 foreach(ref child; node.children) 1225 { 1226 switch(child.token.type) 1227 { 1228 case TokenType.GLOBAL_SET_SKIP: 1229 skip = child.children[0]; 1230 break; 1231 1232 default: 1233 child = child.expandSkip(skip); 1234 } 1235 } 1236 break; 1237 1238 case TokenType.DEFINITION: 1239 case TokenType.SLASH: 1240 case TokenType.LEFT_QUESTION: 1241 case TokenType.LEFT_SHIFT: 1242 case TokenType.SEQUENCE: 1243 case TokenType.QUESTION: 1244 case TokenType.PLUS: 1245 case TokenType.ASTERISK: 1246 case TokenType.ASCIICIRCUM: 1247 case TokenType.ANPERSAND: 1248 case TokenType.EXCLAM: 1249 1250 case TokenType.MEMOIZE: 1251 1252 case TokenType.SKIP_LITERAL_TRUE: 1253 case TokenType.SKIP_LITERAL_FALSE: 1254 case TokenType.MEMOIZE_SKIP_TRUE: 1255 case TokenType.MEMOIZE_SKIP_FALSE: 1256 case TokenType.MEMOIZE_LITERAL_TRUE: 1257 case TokenType.MEMOIZE_LITERAL_FALSE: 1258 case TokenType.MEMOIZE_NONTERMINAL_TRUE: 1259 case TokenType.MEMOIZE_NONTERMINAL_FALSE: 1260 foreach(ref child; node.children) 1261 { 1262 child = child.expandSkip(skip); 1263 } 1264 break; 1265 1266 case TokenType.UNDEFINED: 1267 case TokenType.GLOBAL_SET_SKIP: 1268 assert(false); 1269 1270 case TokenType.CONVERTER: 1271 case TokenType.ID: 1272 case TokenType.DOLLAR: 1273 case TokenType.TEMPLATE_INSTANCE: 1274 case TokenType.NONTERMINAL: 1275 case TokenType.RANGE_ONE_CHAR: 1276 case TokenType.RANGE_CHAR_RANGE: 1277 case TokenType.RANGE: 1278 case TokenType.STRING: 1279 1280 case TokenType.GLOBAL_SKIP_LITERAL_TRUE: 1281 case TokenType.GLOBAL_SKIP_LITERAL_FALSE: 1282 case TokenType.GLOBAL_MEMOIZE_SKIP_TRUE: 1283 case TokenType.GLOBAL_MEMOIZE_SKIP_FALSE: 1284 case TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE: 1285 case TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE: 1286 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 1287 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 1288 } 1289 1290 return node; 1291 } 1292 1293 debug(ctpg) unittest 1294 { 1295 with(TokenType) 1296 { 1297 // GLOBAL_SET_SKIPが動く 1298 assert ( 1299 Node(Token(DEFINITIONS), 1300 [ 1301 Node(Token(GLOBAL_SET_SKIP), 1302 [ 1303 Node(Token(STRING)) 1304 ]), 1305 Node(Token(DEFINITION), 1306 [ 1307 Node(Token(TEMPLATE_INSTANCE)), 1308 Node(Token(ID)), 1309 Node(Token(SKIP), 1310 [ 1311 Node(Token(DOLLAR)) 1312 ]) 1313 ]) 1314 ]).expandSkip() 1315 1316 == 1317 1318 Node(Token(DEFINITIONS), 1319 [ 1320 Node(Token(GLOBAL_SET_SKIP), 1321 [ 1322 Node(Token(STRING)) 1323 ]), 1324 Node(Token(DEFINITION), 1325 [ 1326 Node(Token(TEMPLATE_INSTANCE)), 1327 Node(Token(ID)), 1328 Node(Token(SKIP_WITH), 1329 [ 1330 Node(Token(STRING)), 1331 Node(Token(DOLLAR)) 1332 ]) 1333 ]) 1334 ]) 1335 ); 1336 1337 // 最も近いGLOBAL_SET_SKIPが優先される 1338 assert ( 1339 Node(Token(DEFINITIONS), 1340 [ 1341 Node(Token(GLOBAL_SET_SKIP), 1342 [ 1343 Node(Token(STRING, "1")) 1344 ]), 1345 Node(Token(DEFINITION), 1346 [ 1347 Node(Token(TEMPLATE_INSTANCE)), 1348 Node(Token(ID)), 1349 Node(Token(SKIP), 1350 [ 1351 Node(Token(DOLLAR)) 1352 ]) 1353 ]), 1354 Node(Token(GLOBAL_SET_SKIP), 1355 [ 1356 Node(Token(STRING, "2")) 1357 ]), 1358 Node(Token(DEFINITION), 1359 [ 1360 Node(Token(TEMPLATE_INSTANCE)), 1361 Node(Token(ID)), 1362 Node(Token(SKIP), 1363 [ 1364 Node(Token(DOLLAR)) 1365 ]) 1366 ]) 1367 ]).expandSkip() 1368 1369 == 1370 1371 Node(Token(DEFINITIONS), 1372 [ 1373 Node(Token(GLOBAL_SET_SKIP), 1374 [ 1375 Node(Token(STRING, "1")) 1376 ]), 1377 Node(Token(DEFINITION), 1378 [ 1379 Node(Token(TEMPLATE_INSTANCE)), 1380 Node(Token(ID)), 1381 Node(Token(SKIP_WITH), 1382 [ 1383 Node(Token(STRING, "1")), 1384 Node(Token(DOLLAR)) 1385 ]) 1386 ]), 1387 Node(Token(GLOBAL_SET_SKIP), 1388 [ 1389 Node(Token(STRING, "2")) 1390 ]), 1391 Node(Token(DEFINITION), 1392 [ 1393 Node(Token(TEMPLATE_INSTANCE)), 1394 Node(Token(ID)), 1395 Node(Token(SKIP_WITH), 1396 [ 1397 Node(Token(STRING, "2")), 1398 Node(Token(DOLLAR)) 1399 ]) 1400 ]) 1401 ]) 1402 ); 1403 1404 // SET_SKIPが動く 1405 assert ( 1406 Node(Token(DEFINITIONS), 1407 [ 1408 Node(Token(DEFINITION), 1409 [ 1410 Node(Token(TEMPLATE_INSTANCE)), 1411 Node(Token(ID)), 1412 Node(Token(SET_SKIP), 1413 [ 1414 Node(Token(STRING)), 1415 Node(Token(SKIP), 1416 [ 1417 Node(Token(DOLLAR)) 1418 ]) 1419 ]) 1420 ]) 1421 ]).expandSkip() 1422 1423 == 1424 1425 Node(Token(DEFINITIONS), 1426 [ 1427 Node(Token(DEFINITION), 1428 [ 1429 Node(Token(TEMPLATE_INSTANCE)), 1430 Node(Token(ID)), 1431 Node(Token(SKIP_WITH), 1432 [ 1433 Node(Token(STRING)), 1434 Node(Token(DOLLAR)) 1435 ]) 1436 ]) 1437 ]) 1438 ); 1439 1440 // 最も近いSET_SKIPが優先される 1441 assert ( 1442 Node(Token(DEFINITIONS), 1443 [ 1444 Node(Token(DEFINITION), 1445 [ 1446 Node(Token(TEMPLATE_INSTANCE)), 1447 Node(Token(ID)), 1448 Node(Token(SET_SKIP), 1449 [ 1450 Node(Token(STRING, "1")), 1451 Node(Token(SEQUENCE), 1452 [ 1453 Node(Token(SKIP), 1454 [ 1455 Node(Token(DOLLAR)) 1456 ]), 1457 Node(Token(SET_SKIP), 1458 [ 1459 Node(Token(STRING, "2")), 1460 Node(Token(SKIP), 1461 [ 1462 Node(Token(DOLLAR)) 1463 ]) 1464 ]) 1465 ]), 1466 ]) 1467 ]) 1468 ]).expandSkip() 1469 1470 == 1471 1472 Node(Token(DEFINITIONS), 1473 [ 1474 Node(Token(DEFINITION), 1475 [ 1476 Node(Token(TEMPLATE_INSTANCE)), 1477 Node(Token(ID)), 1478 Node(Token(SEQUENCE), 1479 [ 1480 Node(Token(SKIP_WITH), 1481 [ 1482 Node(Token(STRING, "1")), 1483 Node(Token(DOLLAR)) 1484 ]), 1485 Node(Token(SKIP_WITH), 1486 [ 1487 Node(Token(STRING, "2")), 1488 Node(Token(DOLLAR)) 1489 ]) 1490 ]) 1491 ]) 1492 ]) 1493 ); 1494 1495 // GLOBAL_SET_SKIPよりもSET_SKIPが優先される 1496 assert ( 1497 Node(Token(DEFINITIONS), 1498 [ 1499 Node(Token(GLOBAL_SET_SKIP), 1500 [ 1501 Node(Token(STRING, "global")) 1502 ]), 1503 Node(Token(DEFINITION), 1504 [ 1505 Node(Token(TEMPLATE_INSTANCE)), 1506 Node(Token(ID)), 1507 Node(Token(SET_SKIP), 1508 [ 1509 Node(Token(STRING, "local")), 1510 Node(Token(SKIP), 1511 [ 1512 Node(Token(DOLLAR)) 1513 ]) 1514 ]) 1515 ]) 1516 ]).expandSkip() 1517 1518 == 1519 1520 Node(Token(DEFINITIONS), 1521 [ 1522 Node(Token(GLOBAL_SET_SKIP), 1523 [ 1524 Node(Token(STRING, "global")) 1525 ]), 1526 Node(Token(DEFINITION), 1527 [ 1528 Node(Token(TEMPLATE_INSTANCE)), 1529 Node(Token(ID)), 1530 Node(Token(SKIP_WITH), 1531 [ 1532 Node(Token(STRING, "local")), 1533 Node(Token(DOLLAR)) 1534 ]) 1535 ]) 1536 ]) 1537 ); 1538 } 1539 } 1540 1541 1542 // MEMOIZEの重複を消す 1543 Node removeMemoizeDuplicates(Node node) 1544 { 1545 final switch(node.token.type) with(TokenType) 1546 { 1547 case MEMOIZE: 1548 if(node.children[0].token.type == MEMOIZE) 1549 { 1550 node = node.children[0].removeMemoizeDuplicates(); 1551 } 1552 break; 1553 1554 case DEFINITIONS: 1555 case DEFINITION: 1556 case SLASH: 1557 case LEFT_QUESTION: 1558 case LEFT_SHIFT: 1559 case SEQUENCE: 1560 case QUESTION: 1561 case PLUS: 1562 case ASTERISK: 1563 case ASCIICIRCUM: 1564 case ANPERSAND: 1565 case EXCLAM: 1566 1567 case SKIP: 1568 case SET_SKIP: 1569 case SKIP_WITH: 1570 1571 case SKIP_LITERAL_TRUE: 1572 case SKIP_LITERAL_FALSE: 1573 case MEMOIZE_SKIP_TRUE: 1574 case MEMOIZE_SKIP_FALSE: 1575 case MEMOIZE_LITERAL_TRUE: 1576 case MEMOIZE_LITERAL_FALSE: 1577 case MEMOIZE_NONTERMINAL_TRUE: 1578 case MEMOIZE_NONTERMINAL_FALSE: 1579 1580 case TokenType.GLOBAL_SET_SKIP: 1581 foreach(ref child; node.children) 1582 { 1583 child = child.removeMemoizeDuplicates(); 1584 } 1585 break; 1586 1587 case UNDEFINED: 1588 case CONVERTER: 1589 case ID: 1590 case DOLLAR: 1591 case TEMPLATE_INSTANCE: 1592 case NONTERMINAL: 1593 case RANGE_ONE_CHAR: 1594 case RANGE_CHAR_RANGE: 1595 case RANGE: 1596 case STRING: 1597 1598 case GLOBAL_SKIP_LITERAL_TRUE: 1599 case GLOBAL_SKIP_LITERAL_FALSE: 1600 case GLOBAL_MEMOIZE_SKIP_TRUE: 1601 case GLOBAL_MEMOIZE_SKIP_FALSE: 1602 case GLOBAL_MEMOIZE_LITERAL_TRUE: 1603 case GLOBAL_MEMOIZE_LITERAL_FALSE: 1604 case GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 1605 case GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 1606 } 1607 return node; 1608 } 1609 1610 debug(ctpg) unittest 1611 { 1612 with(TokenType) 1613 { 1614 // MEMOIZEの重複が消される 1615 assert ( 1616 Node(Token(MEMOIZE), 1617 [ 1618 Node(Token(MEMOIZE), 1619 [ 1620 Node(Token(STRING)) 1621 ]) 1622 ]).removeMemoizeDuplicates() 1623 1624 == 1625 1626 Node(Token(MEMOIZE), 1627 [ 1628 Node(Token(STRING)) 1629 ]) 1630 ); 1631 1632 // スキップパーサ内のMEMOIZEが消される 1633 assert ( 1634 Node(Token(SKIP_WITH), 1635 [ 1636 Node(Token(MEMOIZE), 1637 [ 1638 Node(Token(MEMOIZE), 1639 [ 1640 Node(Token(STRING)) 1641 ]) 1642 ]), 1643 Node(Token(STRING)) 1644 ]).removeMemoizeDuplicates() 1645 1646 == 1647 1648 Node(Token(SKIP_WITH), 1649 [ 1650 Node(Token(MEMOIZE), 1651 [ 1652 Node(Token(STRING)) 1653 ]), 1654 Node(Token(STRING)) 1655 ]) 1656 ); 1657 } 1658 } 1659 1660 1661 // SKIP_WITHの重複を消す 1662 Node removeSkipWithDuplicates(Node node) 1663 { 1664 final switch(node.token.type) with(TokenType) 1665 { 1666 case SKIP_WITH: 1667 if(node.children[1].token.type == SKIP_WITH && node.children[0] == node.children[1].children[0]) 1668 { 1669 node = node.children[1].removeSkipWithDuplicates(); 1670 } 1671 break; 1672 1673 case DEFINITIONS: 1674 case DEFINITION: 1675 case SLASH: 1676 case LEFT_QUESTION: 1677 case LEFT_SHIFT: 1678 case SEQUENCE: 1679 case QUESTION: 1680 case PLUS: 1681 case ASTERISK: 1682 case ASCIICIRCUM: 1683 case ANPERSAND: 1684 case EXCLAM: 1685 1686 case SKIP: 1687 case SET_SKIP: 1688 case MEMOIZE: 1689 1690 case SKIP_LITERAL_TRUE: 1691 case SKIP_LITERAL_FALSE: 1692 case MEMOIZE_SKIP_TRUE: 1693 case MEMOIZE_SKIP_FALSE: 1694 case MEMOIZE_LITERAL_TRUE: 1695 case MEMOIZE_LITERAL_FALSE: 1696 case MEMOIZE_NONTERMINAL_TRUE: 1697 case MEMOIZE_NONTERMINAL_FALSE: 1698 1699 case TokenType.GLOBAL_SET_SKIP: 1700 foreach(ref child; node.children) 1701 { 1702 child = child.removeSkipWithDuplicates(); 1703 } 1704 break; 1705 1706 case UNDEFINED: 1707 case CONVERTER: 1708 case ID: 1709 case DOLLAR: 1710 case TEMPLATE_INSTANCE: 1711 case NONTERMINAL: 1712 case RANGE_ONE_CHAR: 1713 case RANGE_CHAR_RANGE: 1714 case RANGE: 1715 case STRING: 1716 1717 case GLOBAL_SKIP_LITERAL_TRUE: 1718 case GLOBAL_SKIP_LITERAL_FALSE: 1719 case GLOBAL_MEMOIZE_SKIP_TRUE: 1720 case GLOBAL_MEMOIZE_SKIP_FALSE: 1721 case GLOBAL_MEMOIZE_LITERAL_TRUE: 1722 case GLOBAL_MEMOIZE_LITERAL_FALSE: 1723 case GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 1724 case GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 1725 } 1726 return node; 1727 } 1728 1729 debug(ctpg) unittest 1730 { 1731 with(TokenType) 1732 { 1733 // スキップパーサが等しいSKIP_WITHの重複が消される 1734 assert ( 1735 Node(Token(SKIP_WITH), 1736 [ 1737 Node(Token(STRING, "a")), 1738 Node(Token(SKIP_WITH), 1739 [ 1740 Node(Token(STRING, "a")), 1741 Node(Token(DOLLAR)) 1742 ]) 1743 ]).removeSkipWithDuplicates() 1744 1745 == 1746 1747 Node(Token(SKIP_WITH), 1748 [ 1749 Node(Token(STRING, "a")), 1750 Node(Token(DOLLAR)) 1751 ]) 1752 ); 1753 1754 // スキップパーサが等しくないSKIP_WITHの重複は消されない 1755 assert ( 1756 Node(Token(SKIP_WITH), 1757 [ 1758 Node(Token(STRING, "a")), 1759 Node(Token(SKIP_WITH), 1760 [ 1761 Node(Token(STRING, "b")), 1762 Node(Token(DOLLAR)) 1763 ]) 1764 ]).removeSkipWithDuplicates() 1765 1766 == 1767 1768 Node(Token(SKIP_WITH), 1769 [ 1770 Node(Token(STRING, "a")), 1771 Node(Token(SKIP_WITH), 1772 [ 1773 Node(Token(STRING, "b")), 1774 Node(Token(DOLLAR)) 1775 ]) 1776 ]) 1777 ); 1778 } 1779 } 1780 1781 1782 // NodeからmixinされるD言語のコードを生成する 1783 string generate(Node node) 1784 { 1785 string code; 1786 1787 final switch(node.token.type) 1788 { 1789 case TokenType.DEFINITIONS: 1790 foreach(child; node.children) 1791 { 1792 if(child.token.type == TokenType.DEFINITION) 1793 { 1794 code ~= child.generate(); 1795 } 1796 } 1797 break; 1798 1799 case TokenType.DEFINITION: 1800 code = 1801 "template " ~ node.children[1].token.text ~ "()" 1802 "{" 1803 "template build(alias kind, SrcType)" 1804 "{" 1805 "static if(kind.hasValue)" 1806 "{" 1807 "#line " ~ node.children[0].line.to!string() ~ " \"" ~ node.children[0].file ~ "\"\n" 1808 "alias Result = ctpg.parse_result.ParseResult!(kind, SrcType, " ~ node.children[0].token.text ~ ");" 1809 "}" 1810 "else" 1811 "{" 1812 "alias Result = ctpg.parse_result.ParseResult!(kind, SrcType);" 1813 "}" 1814 "static Result parse(ctpg.input.Input!SrcType input, in ctpg.caller.Caller caller)" 1815 "{" 1816 "static if(!ctpg.is_wrapper.isImplicitlyConvertible!(typeof(" ~ node.children[2].generate() ~ ".build!(kind, SrcType).parse(input, caller)), Result))" 1817 "{" 1818 "pragma(msg, `" ~ node.children[1].file ~ "(" ~ node.children[1].line.to!string() ~ "): '" ~ node.children[1].token.text ~ "' should return '" ~ node.children[0].token.text ~ "', not '` ~ ctpg.parse_result.getParseResultType!(" ~ node.children[2].generate() ~ ".build!(kind, SrcType)).stringof ~ `'`);" 1819 "static assert(false);" 1820 "}" 1821 "return " ~ node.children[2].generate() ~ ".build!(kind, SrcType).parse(input, caller);" 1822 "}" 1823 "}" 1824 "}"; 1825 break; 1826 1827 case TokenType.SLASH: 1828 foreach(child; node.children) 1829 { 1830 code ~= child.generate() ~ ","; 1831 } 1832 code = "ctpg.combinators.choiceWithFileLine!(`" ~ node.file ~ "`," ~ node.line.to!string() ~ "," ~ code ~ ")"; 1833 break; 1834 1835 case TokenType.LEFT_SHIFT: 1836 code = "ctpg.combinators.convert!(" ~ node.children[0].generate() ~ ",#line " ~ node.children[1].line.to!string() ~ "\n" ~ node.children[1].token.text ~ "," ~ node.line.to!string() ~ ",`" ~ node.file ~ "`)"; 1837 break; 1838 1839 case TokenType.LEFT_QUESTION: 1840 code = "ctpg.combinators.check!(" ~ node.children[0].generate() ~ ",#line " ~ node.children[1].line.to!string() ~ "\n" ~ node.children[1].token.text ~ "," ~ node.line.to!string() ~ ",`" ~ node.file ~ "`)"; 1841 break; 1842 1843 case TokenType.SEQUENCE: 1844 foreach(child; node.children) 1845 { 1846 code ~= child.generate() ~ ","; 1847 } 1848 code = "ctpg.combinators.untuple!(ctpg.combinators.sequence!(" ~ code ~ "))"; 1849 break; 1850 1851 case TokenType.QUESTION: 1852 code = "ctpg.combinators.option!(" ~ node.children[0].generate() ~ ")"; 1853 break; 1854 1855 case TokenType.ASTERISK: 1856 if(node.children.length == 2) 1857 { 1858 code = "ctpg.combinators.more0!(" ~ node.children[0].generate() ~ "," ~ node.children[1].generate() ~ ")"; 1859 } 1860 else 1861 { 1862 code = "ctpg.combinators.more0!(" ~ node.children[0].generate() ~ ")"; 1863 } 1864 break; 1865 1866 case TokenType.PLUS: 1867 if(node.children.length == 2) 1868 { 1869 code = "ctpg.combinators.more1!(" ~ node.children[0].generate() ~ "," ~ node.children[1].generate() ~ ")"; 1870 } 1871 else 1872 { 1873 code = "ctpg.combinators.more1!(" ~ node.children[0].generate() ~ ")"; 1874 } 1875 break; 1876 1877 case TokenType.EXCLAM: 1878 code = "ctpg.combinators.none!(" ~ node.children[0].generate() ~ ")"; 1879 break; 1880 1881 case TokenType.ANPERSAND: 1882 code = "ctpg.combinators.andPred!(" ~ node.children[0].generate() ~ ")"; 1883 break; 1884 1885 case TokenType.ASCIICIRCUM: 1886 code = "ctpg.combinators.notPred!(" ~ node.children[0].generate() ~ ")"; 1887 break; 1888 1889 case TokenType.SKIP_WITH: 1890 code = "ctpg.combinators.skip!(" ~ node.children[0].generate() ~ "," ~ node.children[1].generate() ~ ")"; 1891 break; 1892 1893 case TokenType.MEMOIZE: 1894 code = "ctpg.combinators.memoize!(" ~ node.children[0].generate() ~ ")"; 1895 break; 1896 1897 case TokenType.DOLLAR: 1898 code = "ctpg.parsers.eof!()"; 1899 break; 1900 1901 case TokenType.RANGE: 1902 if(node.children.length > 1) 1903 { 1904 code ~= "ctpg.combinators.choice!("; 1905 } 1906 1907 foreach(i, child; node.children) 1908 { 1909 if(i) 1910 { 1911 code ~= ","; 1912 } 1913 1914 switch(child.token.type) 1915 { 1916 case TokenType.RANGE_CHAR_RANGE: 1917 code ~= "ctpg.parsers.charRange!('" ~ child.children[0].token.text ~ "','" ~ child.children[1].token.text ~ "')"; 1918 break; 1919 1920 case TokenType.RANGE_ONE_CHAR: 1921 code ~= "ctpg.parsers.charRange!('" ~ child.token.text ~ "','" ~ child.token.text ~ "')"; 1922 break; 1923 1924 default: assert(false); 1925 } 1926 } 1927 1928 if(node.children.length > 1) 1929 { 1930 code ~= ")"; 1931 } 1932 break; 1933 1934 case TokenType.STRING: 1935 code = "ctpg.parsers.string_!(" ~ node.token.text ~ ")"; 1936 break; 1937 1938 case TokenType.NONTERMINAL: 1939 code = "#line " ~ node.line.to!string() ~ " \"" ~ node.file ~ "\"\n" ~ node.token.text ~ "!()"; 1940 break; 1941 1942 case TokenType.UNDEFINED: 1943 case TokenType.CONVERTER: 1944 case TokenType.ID: 1945 case TokenType.TEMPLATE_INSTANCE: 1946 case TokenType.RANGE_ONE_CHAR: 1947 case TokenType.RANGE_CHAR_RANGE: 1948 1949 case TokenType.SKIP: 1950 case TokenType.SET_SKIP: 1951 1952 case TokenType.MEMOIZE_SKIP_TRUE: 1953 case TokenType.MEMOIZE_SKIP_FALSE: 1954 case TokenType.SKIP_LITERAL_TRUE: 1955 case TokenType.SKIP_LITERAL_FALSE: 1956 case TokenType.MEMOIZE_LITERAL_TRUE: 1957 case TokenType.MEMOIZE_LITERAL_FALSE: 1958 case TokenType.MEMOIZE_NONTERMINAL_TRUE: 1959 case TokenType.MEMOIZE_NONTERMINAL_FALSE: 1960 1961 case TokenType.GLOBAL_SET_SKIP: 1962 1963 case TokenType.GLOBAL_SKIP_LITERAL_TRUE: 1964 case TokenType.GLOBAL_SKIP_LITERAL_FALSE: 1965 case TokenType.GLOBAL_MEMOIZE_SKIP_TRUE: 1966 case TokenType.GLOBAL_MEMOIZE_SKIP_FALSE: 1967 case TokenType.GLOBAL_MEMOIZE_LITERAL_TRUE: 1968 case TokenType.GLOBAL_MEMOIZE_LITERAL_FALSE: 1969 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_TRUE: 1970 case TokenType.GLOBAL_MEMOIZE_NONTERMINAL_FALSE: 1971 assert(false); 1972 } 1973 1974 return code; 1975 } 1976 1977 debug(ctpg) unittest 1978 { 1979 with(TokenType) 1980 { 1981 assert ( 1982 Node(Token(RANGE), [ 1983 Node(Token(RANGE_ONE_CHAR, "a")) 1984 ]).generate() 1985 1986 == 1987 1988 "ctpg.parsers.charRange!('a','a')" 1989 ); 1990 1991 assert ( 1992 Node(Token(RANGE), [ 1993 Node(Token(RANGE_CHAR_RANGE), [ 1994 Node(Token(RANGE_ONE_CHAR, "a")), 1995 Node(Token(RANGE_ONE_CHAR, "z")) 1996 ]) 1997 ]).generate() 1998 1999 == 2000 2001 "ctpg.parsers.charRange!('a','z')" 2002 ); 2003 2004 assert ( 2005 Node(Token(RANGE), [ 2006 Node(Token(RANGE_CHAR_RANGE), [ 2007 Node(Token(RANGE_ONE_CHAR, "a")), 2008 Node(Token(RANGE_ONE_CHAR, "z")) 2009 ]), 2010 Node(Token(RANGE_ONE_CHAR, "_")) 2011 ]).generate() 2012 2013 == 2014 2015 "ctpg.combinators.choice!" 2016 "(" 2017 "ctpg.parsers.charRange!('a','z')," 2018 "ctpg.parsers.charRange!('_','_')" 2019 ")" 2020 ); 2021 } 2022 }