diff --git a/examples/mml-exp3.treerack b/examples/mml-exp3.treerack new file mode 100644 index 0000000..dadd761 --- /dev/null +++ b/examples/mml-exp3.treerack @@ -0,0 +1,271 @@ +ws:ws = " " | "\b" | "\f" | "\r" | "\t" | "\v"; +wsc:ws = comment; +nl:alias = "\n"; +wsep:alias = ws | nl; + +line-comment-content:nows = [^\n]*; +line-comment:alias:nows = "//" line-comment-content; +block-comment-content:nows = ([^*] | "*" [^/])*; +block-comment:alias:nows = "/*" block-comment-content "*/"; +comment-part:alias = line-comment | block-comment; +comment:alias = comment-part (nl? comment-part)*; + +decimal-digit:alias = [0-9]; +octal-digit:alias = [0-7]; +hexa-digit:alias = [0-9a-fA-F]; + +decimal:alias:nows = [1-9] decimal-digit*; +octal:alias:nows = "0" octal-digit*; +hexa:alias:nows = "0" [xX] hexa-digit+; +int = decimal | octal | hexa; + +exponent:alias:nows = [eE] [+\-]? decimal-digit+; +float:nows = decimal-digit+ "." decimal-digit* exponent? + | "." decimal-digit+ exponent? + | decimal-digit+ exponent; + +string:nows = "\"" ([^\\"] | "\\" .)* "\""; + +true = "true"; +false = "false"; +bool:alias = true | false; + +symbol:nows = [a-zA-Z_][a-zA-Z_0-9]*; + +spread-expression = primary-expression "..."; +list-sep:alias = (nl | ",")+; +list-item:alias = expression | spread-expression; +expression-list:alias = list-item (list-sep list-item)*; + +list-fact:alias = "[" list-sep? expression-list? list-sep? "]"; +list = list-fact; +mutable-list = "~" nl* list-fact; + +expression-key = "[" nl* expression nl* "]"; +entry = (symbol | string | expression-key) nl* ":" nl* expression; +entry-list:alias = (entry | spread-expression) (list-sep (entry | spread-expression))*; +struct-fact:alias = "{" list-sep? entry-list? list-sep? "}"; +struct = struct-fact; +mutable-struct = "~" nl* struct-fact; + +parameter-list:alias = symbol (list-sep symbol)*; +collect-parameter = "..." nl* symbol; +return = "return" | "return" nl* expression; +block = "{" sep? statement-list? sep? "}"; +function-fact:alias = "(" list-sep? + (parameter-list + | parameter-list list-sep collect-parameter + | collect-parameter)? + list-sep? ")" nl* + (expression | simple-statement | block); +function = "fn" nl* function-fact; +effect = "fn" nl* "~" nl* function-fact; + +range-from = expression; +range-to = expression; +range:alias = range-from? nl* ":" nl* range-to?; + +symbol-index = "." nl* symbol; +expression-index = "[" nl* expression nl* "]"; +range-index = "[" nl* range nl* "]"; +index:alias = symbol-index | expression-index | range-index; +index-list:alias = index (nl* index)?; +indexer = primary-expression nl* index-list; + +application = primary-expression "(" list-sep? expression-list? list-sep? ")"; + +expression-group:alias = "(" nl* expression nl* ")"; +primary-expression:alias = int + | float + | string + | bool + | symbol + | list + | mutable-list + | struct + | mutable-struct + | function + | effect + | indexer + | application + | receive + | expression-group; + +binary-not = "^"; +binary-and = "&"; +binary-or = "|"; +xor = "^"; +and-not = "&^"; +lshift = "<<"; +rshift = ">>"; + +plus = "+"; +minus = "-"; +mul = "*"; +div = "/"; +mod = "%"; +add = "+"; +sub = "-"; + +logical-not = "!"; +eq = "=="; +not-eq = "!="; +less = "<"; +less-or-eq = "<="; +greater = ">"; +greater-or-eq = ">="; +logical-and = "&&"; +logical-or = "||"; + +chain:alias = "->"; + +unary-operator:alias = plus | minus | binary-not | logical-not; +unary-expression = unary-operator primary-expression; + +binary-op0:alias = binary-and | and-not | lshift | rshift | mul | div | mod; +binary-op1:alias = binary-or | xor | add | sub; +binary-op2:alias = eq | not-eq | less | less-or-eq | greater | greater-or-eq; +binary-op3:alias = logical-and; +binary-op4:alias = logical-or; + +operand0:alias = primary-expression | unary-expression; +operand1:alias = operand0 | binary0; +operand2:alias = operand1 | binary1; +operand3:alias = operand2 | binary2; +operand4:alias = operand3 | binary3; +operand5:alias = operand4 | binary4; + +binary0 = operand0 (nl* binary-op0 nl* operand0)+; +binary1 = operand1 (nl* binary-op1 nl* operand1)+; +binary2 = operand2 (nl* binary-op2 nl* operand2)+; +binary3 = operand3 (nl* binary-op3 nl* operand3)+; +binary4 = operand4 (nl* binary-op4 nl* operand4)+; + +binary-expression:alias = binary0 + | binary1 + | binary2 + | binary3 + | binary4; + +ternary-expression = expression nl* "?" nl* expression nl* ":" nl* expression; + +chainingOperand:alias = primary-expression + | unary-expression + | binary-expression + | ternary-expression; +chaining = chainingOperand (nl* chain nl* chainingOperand)+; + +expression:alias = primary-expression + | unary-expression + | binary-expression + | ternary-expression + | chaining; + +if = "if" nl* expression nl* block + (nl* "else" nl* "if" nl* expression nl* block)* + (nl* "else" nl* block)?; + +// TODO: empty switch not parsed +default = "default" nl* ":"; +default-line:alias = default ";"* statement?; +case = "case" nl* expression nl* ":"; +case-line:alias = case ";"* statement?; +switch = "switch" nl* expression? nl* "{" sep? + ((case-line | default-line) (sep (case-line | default-line | statement))*)? + sep? "}"; + +send = "send" nl* primary-expression nl* primary-expression; +receive = "receive" nl* primary-expression; +receive-definition = symbol nl* receive; +communication:alias = send | receive | receive-definition; +select-case = "case" nl* communication nl* ":"; +select-case-line:alias = select-case ";"* statement?; +select = "select" nl* "{" sep? + ((select-case-line | default-line) + (sep (select-case-line | default-line | statement))*)? + sep? "}"; + +go = "go" nl* application; +defer = "defer" nl* application; + +break = "break"; +continue = "continue"; +loop-control:alias = break | continue; +range-over = symbol nl* "in" nl* (expression | range) | range; +loop-expression:alias = expression | range-over; +loop = "for" ((nl* loop-expression)? nl* block | nl* block); + +assignment = (symbol | indexer) nl* "=" nl* expression; + +value-capture-fact:alias = symbol (nl* "=")? nl* expression; +value-capture = value-capture-fact; +mutable-capture = "~" nl* value-capture-fact; +value-definition = "let" nl* (value-capture | mutable-capture); +mixed-capture-list:alias = (value-capture | mutable-capture) (list-sep (value-capture | mutable-capture))*; +value-capture-list:alias = value-capture (list-sep value-capture)*; +value-definition-group = "let" nl* "(" list-sep? mixed-capture-list? list-sep? ")"; +mutable-definition-group = "let" nl* "~" nl* "(" list-sep? value-capture-list? list-sep? ")"; + +function-definition-fact:alias = symbol nl* function-fact; +function-capture = function-definition-fact; +effect-capture = "~" nl* function-definition-fact; +function-definition = "fn" nl* (function-capture | effect-capture); +function-capture-list:alias = function-capture (list-sep function-capture)*; +mixed-function-capture-list:alias = (function-capture | effect-capture) + (list-sep (function-capture | effect-capture))*; +function-definition-group = "fn" nl* "(" list-sep? + mixed-function-capture-list? + list-sep? ")"; +effect-definition-group = "fn" nl* "~" nl* "(" list-sep? + function-capture-list? + list-sep? ")"; + +definition:alias = value-definition + | value-definition-group + | mutable-definition-group + | function-definition + | function-definition-group + | effect-definition-group; + +// TODO: +// - use effect + +use-inline = "."; +use-fact = string + | (symbol | use-inline) (nl* "=")? nl* string; +use-fact-list:alias = use-fact (list-sep use-fact)*; +use-statement:alias = "use" nl* use-fact; +use-statement-group:alias = "use" nl* "(" list-sep? + use-fact-list? + list-sep? ")"; +use = use-statement | use-statement-group; + +export = "export" nl* definition; + +simple-statement:alias = send + | go + | defer + | assignment + | simple-statement-group; +simple-statement-group:alias = "(" nl* simple-statement nl* ")"; +statement:alias = simple-statement + | return + | if + | switch + | select + | loop + | loop-control + | definition + | use + | export + | application // TODO: sendSomething() and sendSomething + | chaining + | statement-group; +statement-group:alias = "(" nl* statement nl* ")"; + +sep:alias = (";" | nl)+; +statement-list:alias = statement (sep statement)*; + +shebang-command = [^\n]*; +shebang = "#!" shebang-command "\n"; +mml:root = shebang? sep? statement-list? sep?; diff --git a/head.go b/head.go index 93fdc11..d33d511 100644 --- a/head.go +++ b/head.go @@ -1,4 +1,4 @@ package treerack // generated with scripts/createhead.go -const headCode = "import (\n\t\"strconv\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n\t\"errors\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() CommitType {\n\treturn Alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]*Node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount\tint\n\t\tparsed\t\tbool\n\t)\n\titemIndex := 0\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&Alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []*Node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]*Node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&Whitespace == 0 && p.commitType()&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n}\n\nfunc newContext(r io.RuneReader) *context {\n\treturn &context{reader: r, results: &results{}, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif err == io.EOF {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = ErrInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\treturn &ParseError{Offset: c.failOffset, Line: line, Column: col, Definition: definition}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp)\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root)\n}\n\ntype Node struct {\n\tName\t\tstring\n\tNodes\t\t[]*Node\n\tFrom, To\tint\n\ttokens\t\t[]rune\n}\n\nfunc (n *Node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n *Node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n *Node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype CommitType int\n\nconst (\n\tNone\tCommitType\t= 0\n\tAlias\tCommitType\t= 1 << iota\n\tWhitespace\n\tNoWhitespace\n\tFailPass\n\tRoot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype ParseError struct {\n\tInput\t\tstring\n\tOffset\t\tint\n\tLine\t\tint\n\tColumn\t\tint\n\tDefinition\tstring\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() CommitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]*Node, bool)\n}\n\nvar ErrInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *ParseError) Error() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition)\n}\nfunc parseInput(r io.Reader, p parser, b builder) (*Node, error) {\n\tc := newContext(bufio.NewReader(r))\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn nil, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*ParseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn nil, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" +const headCode = "import (\n\t\"strconv\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n\t\"errors\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() CommitType {\n\treturn Alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]*Node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount\tint\n\t\tparsed\t\tbool\n\t)\n\titemIndex := 0\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&Alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []*Node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]*Node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&Whitespace == 0 && p.commitType()&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n}\n\nfunc newContext(r io.RuneReader) *context {\n\treturn &context{reader: r, results: &results{}, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif err == io.EOF {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = ErrInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\treturn &ParseError{Offset: c.failOffset, Line: line, Column: col, Definition: definition}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp)\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root)\n}\n\ntype Node struct {\n\tName\t\tstring\n\tNodes\t\t[]*Node\n\tFrom, To\tint\n\ttokens\t\t[]rune\n}\n\nfunc (n *Node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n *Node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n *Node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype CommitType int\n\nconst (\n\tNone\tCommitType\t= 0\n\tAlias\tCommitType\t= 1 << iota\n\tWhitespace\n\tNoWhitespace\n\tFailPass\n\tRoot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype ParseError struct {\n\tInput\t\tstring\n\tOffset\t\tint\n\tLine\t\tint\n\tColumn\t\tint\n\tDefinition\tstring\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() CommitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]*Node, bool)\n}\n\nvar ErrInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *ParseError) Error() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition)\n}\nfunc parseInput(r io.Reader, p parser, b builder) (*Node, error) {\n\tc := newContext(bufio.NewReader(r))\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn nil, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*ParseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn nil, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" diff --git a/mmlexp3_test.go b/mmlexp3_test.go new file mode 100644 index 0000000..b6f028e --- /dev/null +++ b/mmlexp3_test.go @@ -0,0 +1,45 @@ +package treerack + +import ( + "testing" +) + +func TestMMLExp3(t *testing.T) { + s, err := openSyntaxFile("examples/mml-exp3.treerack") + if err != nil { + t.Error(err) + return + } + + t.Run("indexer", func(t *testing.T) { + runTestsSyntax(t, s, []testItem{{ + title: "assignment", + text: "fn f() a.b = c", + ignorePosition: true, + nodes: []*Node{{ + Name: "function-definition", + Nodes: []*Node{{ + Name: "function-capture", + Nodes: []*Node{{ + Name: "symbol", + }, { + Name: "assignment", + Nodes: []*Node{{ + Name: "indexer", + Nodes: []*Node{{ + Name: "symbol", + }, { + Name: "symbol-index", + Nodes: []*Node{{ + Name: "symbol", + }}, + }}, + }, { + Name: "symbol", + }}, + }}, + }}, + }}, + }}) + }) +} diff --git a/self/self.go b/self/self.go index 991ed08..cb88c8d 100644 --- a/self/self.go +++ b/self/self.go @@ -137,6 +137,17 @@ func (p *sequenceParser) parse(c *context) { currentCount = 0 continue } + c.offset = from + if c.fromResults(p) { + if to > c.failOffset { + c.failOffset = -1 + c.failingParser = nil + } + if !p.allChars { + c.results.unmarkPending(from, p.id) + } + return + } if c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 { c.failingParser = p } diff --git a/sequence.go b/sequence.go index 69181c2..bfdc9d1 100644 --- a/sequence.go +++ b/sequence.go @@ -52,6 +52,20 @@ func (p *sequenceParser) parse(c *context) { continue } + c.offset = from + if c.fromResults(p) { + if to > c.failOffset { + c.failOffset = -1 + c.failingParser = nil + } + + if !p.allChars { + c.results.unmarkPending(from, p.id) + } + + return + } + if c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 &&