Appending tokens through the lexer's field grew the slice as it filled, which made deeply nested formulas lex slower than before the lexer was split into methods. Starting with room for a token every two bytes halves the allocations of parsing and brings BenchmarkNested/paren-10000 from 2.18 ms (1.93 ms before the split) to 1.78 ms.
- commit
- e5a0be842c52d33d3b22c4ad852a7cc764600106
- parent
- 4cdc7667799e817d1715b49d5f738d9d3a29e6da
- author
- Dami <dami.etoile@gmail.com>
| M | internal/ | 4 | +++- |
@@ -55,7 +55,9 @@ type lexer struct {
5555 }5656 5757 func lex(src string) ([]token, error) {58- lx := lexer{src: src}58+ // Most formulas have a token for every two or three bytes; growing59+ // the slice as it fills cost more than lexing in deep formulas.60+ lx := lexer{src: src, toks: make([]token, 0, len(src)/2+1)}5961 for lx.i < len(src) {6062 if err := lx.next(); err != nil {6163 return nil, err