From a009c1918d18e3de4e3499ac2fd8833a09d24cba Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Sat, 6 Jun 2026 21:48:50 +0200 Subject: [PATCH] only use tracing when explicitly enabled --- cmd/treerack/check.go | 5 +++- cmd/treerack/checksyntax.go | 5 +++- cmd/treerack/docreflect.gen.go | 21 ++++++++------ cmd/treerack/format.go | 40 ++++++++++++++++---------- cmd/treerack/generate.go | 5 +++- cmd/treerack/readme.md | 12 ++++++-- cmd/treerack/show.go | 5 +++- context.go | 52 ++++++++++++++++++---------------- head.gen.go | 2 +- headexported.gen.go | 2 +- internal/self/self.gen.go | 48 ++++++++++++++++--------------- parse_test.go | 2 +- syntax.go | 8 ++++-- syntaxhead.go | 4 +-- 14 files changed, 127 insertions(+), 84 deletions(-) diff --git a/cmd/treerack/check.go b/cmd/treerack/check.go index d837df4..bd9bfcc 100644 --- a/cmd/treerack/check.go +++ b/cmd/treerack/check.go @@ -20,6 +20,9 @@ type checkOptions struct { // InputString specifies the input content as an inline string. InputString *string + + // MaxTraceLength enables tracing when set to a positive integer. + MaxTraceLength int } // check parses input content against the provided syntax definition and fails if the input does not match. @@ -38,7 +41,7 @@ func check(o checkOptions, stdin io.Reader, args ...string) error { } defer finalizeInput() - s := &treerack.Syntax{} + s := &treerack.Syntax{MaxTraceLength: o.MaxTraceLength} if err := s.ReadSyntax(syntax); err != nil { if terr := treerack.Trace(os.Stderr, err); terr != nil { err = errors.Join(err, terr) diff --git a/cmd/treerack/checksyntax.go b/cmd/treerack/checksyntax.go index 1051fee..a121659 100644 --- a/cmd/treerack/checksyntax.go +++ b/cmd/treerack/checksyntax.go @@ -14,6 +14,9 @@ type checkSyntaxOptions struct { // SyntaxString specifies the syntax as an inline string. SyntaxString *string + + // MaxTraceLength enables tracing when set to a positive integer. + MaxTraceLength int } // checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a @@ -25,7 +28,7 @@ func checkSyntax(o checkSyntaxOptions, stdin io.Reader, args ...string) error { } defer finalize() - s := &treerack.Syntax{} + s := &treerack.Syntax{MaxTraceLength: o.MaxTraceLength} if err := s.ReadSyntax(syntax); err != nil { if terr := treerack.Trace(os.Stderr, err); terr != nil { err = errors.Join(err, terr) diff --git a/cmd/treerack/docreflect.gen.go b/cmd/treerack/docreflect.gen.go index 525f0c2..ce8ae1b 100644 --- a/cmd/treerack/docreflect.gen.go +++ b/cmd/treerack/docreflect.gen.go @@ -12,28 +12,32 @@ func init() { docreflect.Register("main.checkOptions", "") docreflect.Register("main.checkOptions.Input", "Input specifies the filename of the input content to be validated.\n") docreflect.Register("main.checkOptions.InputString", "InputString specifies the input content as an inline string.\n") + docreflect.Register("main.checkOptions.MaxTraceLength", "MaxTraceLength enables tracing when set to a positive integer.\n") docreflect.Register("main.checkOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.checkOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") docreflect.Register("main.checkSyntax", "checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a\npositional argument), an inline string, or piped from standard input.\n\nfunc(o, stdin, args)") docreflect.Register("main.checkSyntaxOptions", "") + docreflect.Register("main.checkSyntaxOptions.MaxTraceLength", "MaxTraceLength enables tracing when set to a positive integer.\n") docreflect.Register("main.checkSyntaxOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.checkSyntaxOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") docreflect.Register("main.errInvalidFilename", "") docreflect.Register("main.errMultipleInputs", "") docreflect.Register("main.errNoInput", "") docreflect.Register("main.format", "format input syntax. Accepts syntax from one or more files, inline syntax string or stdin. Use the --in-place\noption, when formatting files in place, or print the formatted syntax to stdout.\n\nfunc(o, stdin, stdout, syntax)") - docreflect.Register("main.formatFile", "\nfunc(name, inPlace, out)") - docreflect.Register("main.formatFiles", "\nfunc(files, inPlace, out)") - docreflect.Register("main.formatInline", "\nfunc(syntax, out)") + docreflect.Register("main.formatFile", "\nfunc(name, inPlace, maxTraceLength, out)") + docreflect.Register("main.formatFiles", "\nfunc(files, inPlace, maxTraceLength, out)") + docreflect.Register("main.formatInline", "\nfunc(syntax, out, maxTraceLength)") docreflect.Register("main.formatOptions", "") - docreflect.Register("main.formatOptions.InPlace", "") - docreflect.Register("main.formatOptions.Syntax", "") - docreflect.Register("main.formatOptions.SyntaxString", "") - docreflect.Register("main.formatStdin", "\nfunc(in, out)") - docreflect.Register("main.formatSyntax", "\nfunc(in, out)") + docreflect.Register("main.formatOptions.InPlace", "InPlace specifies if an input file should be formatted in-place instead of printing the formatted\nresults to the standard output.\n") + docreflect.Register("main.formatOptions.MaxTraceLength", "MaxTraceLength enables tracing when set to a positive integer.\n") + docreflect.Register("main.formatOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") + docreflect.Register("main.formatOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") + docreflect.Register("main.formatStdin", "\nfunc(in, out, maxTraceLength)") + docreflect.Register("main.formatSyntax", "\nfunc(in, out, maxTraceLength)") docreflect.Register("main.generate", "generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded\nin an application.\n\nThe syntax may be provided via a file path (using an option or a positional argument), an\ninline string, or piped from standard input.\n\nfunc(o, stdin, stdout, args)") docreflect.Register("main.generateOptions", "") docreflect.Register("main.generateOptions.Export", "Export determines whether the generated parse function is exported (visible outside its package).\n") + docreflect.Register("main.generateOptions.MaxTraceLength", "MaxTraceLength enables tracing when set to a positive integer.\n") docreflect.Register("main.generateOptions.PackageName", "PackageName specifies the package name for the generated code. Defaults to main.\n") docreflect.Register("main.generateOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.generateOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") @@ -53,6 +57,7 @@ func init() { docreflect.Register("main.showOptions.Indent", "Indent specifies a custom indentation string for the output.\n") docreflect.Register("main.showOptions.Input", "Input specifies the filename of the input content to be validated.\n") docreflect.Register("main.showOptions.InputString", "InputString specifies the input content as an inline string.\n") + docreflect.Register("main.showOptions.MaxTraceLength", "MaxTraceLength enables tracing when set to a positive integer.\n") docreflect.Register("main.showOptions.Pretty", "Pretty enables indented, human-readable output.\n") docreflect.Register("main.showOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.showOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") diff --git a/cmd/treerack/format.go b/cmd/treerack/format.go index be6970c..bf924d7 100644 --- a/cmd/treerack/format.go +++ b/cmd/treerack/format.go @@ -10,13 +10,23 @@ import ( ) type formatOptions struct { - InPlace bool + + // InPlace specifies if an input file should be formatted in-place instead of printing the formatted + // results to the standard output. + InPlace bool + + // SyntaxString specifies the syntax as an inline string. SyntaxString *string - Syntax []string + + // Syntax specifies the filename of the syntax definition file. + Syntax []string + + // MaxTraceLength enables tracing when set to a positive integer. + MaxTraceLength int } -func formatSyntax(in io.Reader, out io.Writer) error { - s := new(treerack.Syntax) +func formatSyntax(in io.Reader, out io.Writer, maxTraceLength int) error { + s := treerack.Syntax{MaxTraceLength: maxTraceLength} if err := s.ReadSyntax(in); err != nil { if terr := treerack.Trace(os.Stderr, err); err != nil { err = errors.Join(err, terr) @@ -36,7 +46,7 @@ func formatSyntax(in io.Reader, out io.Writer) error { return nil } -func formatFile(name string, inPlace bool, out io.Writer) error { +func formatFile(name string, inPlace bool, maxTraceLength int, out io.Writer) error { var ( inBytes []byte buf *bytes.Buffer @@ -48,7 +58,7 @@ func formatFile(name string, inPlace bool, out io.Writer) error { } buf = bytes.NewBuffer(inBytes) - if err = formatSyntax(buf, buf); err != nil { + if err = formatSyntax(buf, buf, maxTraceLength); err != nil { return err } @@ -65,9 +75,9 @@ func formatFile(name string, inPlace bool, out io.Writer) error { return os.WriteFile(name, buf.Bytes(), 0644) } -func formatFiles(files []string, inPlace bool, out io.Writer) error { +func formatFiles(files []string, inPlace bool, maxTraceLength int, out io.Writer) error { for _, f := range files { - if err := formatFile(f, inPlace, out); err != nil { + if err := formatFile(f, inPlace, maxTraceLength, out); err != nil { return err } } @@ -75,13 +85,13 @@ func formatFiles(files []string, inPlace bool, out io.Writer) error { return nil } -func formatInline(syntax string, out io.Writer) error { +func formatInline(syntax string, out io.Writer, maxTraceLength int) error { buf := bytes.NewBufferString(syntax) - return formatSyntax(buf, out) + return formatSyntax(buf, out, maxTraceLength) } -func formatStdin(in io.Reader, out io.Writer) error { - return formatSyntax(in, out) +func formatStdin(in io.Reader, out io.Writer, maxTraceLength int) error { + return formatSyntax(in, out, maxTraceLength) } // format input syntax. Accepts syntax from one or more files, inline syntax string or stdin. Use the --in-place @@ -109,12 +119,12 @@ func format(o formatOptions, stdin io.Reader, stdout io.Writer, syntax ...string } if len(files) > 0 { - return formatFiles(files, o.InPlace, stdout) + return formatFiles(files, o.InPlace, o.MaxTraceLength, stdout) } if o.SyntaxString != nil { - return formatInline(*o.SyntaxString, stdout) + return formatInline(*o.SyntaxString, stdout, o.MaxTraceLength) } - return formatSyntax(stdin, stdout) + return formatSyntax(stdin, stdout, o.MaxTraceLength) } diff --git a/cmd/treerack/generate.go b/cmd/treerack/generate.go index fbc45b1..2eb94c0 100644 --- a/cmd/treerack/generate.go +++ b/cmd/treerack/generate.go @@ -20,6 +20,9 @@ type generateOptions struct { // Export determines whether the generated parse function is exported (visible outside its package). Export bool + + // MaxTraceLength enables tracing when set to a positive integer. + MaxTraceLength int } // generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded @@ -34,7 +37,7 @@ func generate(o generateOptions, stdin io.Reader, stdout io.Writer, args ...stri } defer finalizeSyntax() - s := &treerack.Syntax{} + s := &treerack.Syntax{MaxTraceLength: o.MaxTraceLength} if err := s.ReadSyntax(syntax); err != nil { if terr := treerack.Trace(os.Stderr, err); err != nil { err = errors.Join(err, terr) diff --git a/cmd/treerack/readme.md b/cmd/treerack/readme.md index 2760c9c..b84f8fe 100644 --- a/cmd/treerack/readme.md +++ b/cmd/treerack/readme.md @@ -32,6 +32,7 @@ argument), an inline string, or piped from standard input. #### Options: +- --max-trace-length int: enables tracing when set to a positive integer. - --syntax string: specifies the filename of the syntax definition file. - --syntax-string string: specifies the syntax as an inline string. - --help: Show help. @@ -57,6 +58,7 @@ positional argument filename, an inline string option, or piped from standard in - --input string: specifies the filename of the input content to be validated. - --input-string string: specifies the input content as an inline string. +- --max-trace-length int: enables tracing when set to a positive integer. - --syntax string: specifies the filename of the syntax definition file. - --syntax-string string: specifies the syntax as an inline string. - --help: Show help. @@ -83,6 +85,7 @@ filename option, a positional argument filename, an inline string option, or pip - --indent string: specifies a custom indentation string for the output. - --input string: specifies the filename of the input content to be validated. - --input-string string: specifies the input content as an inline string. +- --max-trace-length int: enables tracing when set to a positive integer. - --pretty bool: enables indented, human-readable output. - --syntax string: specifies the filename of the syntax definition file. - --syntax-string string: specifies the syntax as an inline string. @@ -110,6 +113,7 @@ piped from standard input. #### Options: - --export bool: determines whether the generated parse function is exported (visible outside its package). +- --max-trace-length int: enables tracing when set to a positive integer. - --package-name string: specifies the package name for the generated code. Defaults to main. - --syntax string: specifies the filename of the syntax definition file. - --syntax-string string: specifies the syntax as an inline string. @@ -131,9 +135,11 @@ when formatting files in place, or print the formatted syntax to stdout. #### Options: -- --in-place bool: -- --syntax string \[\*\]: -- --syntax-string string: +- --in-place bool: specifies if an input file should be formatted in-place instead of printing the formatted + results to the standard output. +- --max-trace-length int: enables tracing when set to a positive integer. +- --syntax string \[\*\]: specifies the filename of the syntax definition file. +- --syntax-string string: specifies the syntax as an inline string. - --help: Show help. ### treerack version diff --git a/cmd/treerack/show.go b/cmd/treerack/show.go index 405a80c..8b84b6f 100644 --- a/cmd/treerack/show.go +++ b/cmd/treerack/show.go @@ -27,6 +27,9 @@ type showOptions struct { // Indent specifies a custom indentation string for the output. Indent string + + // MaxTraceLength enables tracing when set to a positive integer. + MaxTraceLength int } type node struct { @@ -72,7 +75,7 @@ func show(o showOptions, stdin io.Reader, stdout io.Writer, args ...string) erro } defer finalizeInput() - s := &treerack.Syntax{} + s := &treerack.Syntax{MaxTraceLength: o.MaxTraceLength} if err := s.ReadSyntax(syntax); err != nil { if terr := treerack.Trace(os.Stderr, err); err != nil { err = errors.Join(err, terr) diff --git a/context.go b/context.go index eec6806..a65510f 100644 --- a/context.go +++ b/context.go @@ -7,33 +7,33 @@ import ( "unicode" ) -const maxTraceEntries = 36 - type context struct { - reader io.RuneReader - keywords []parser - offset int - readOffset int - consumed int - offsetLimit int - failOffset int - failingParser parser - readErr error - eof bool - results *results - tokens []rune - matchLast bool - level int - tr []TraceEntry + reader io.RuneReader + keywords []parser + offset int + readOffset int + consumed int + offsetLimit int + failOffset int + failingParser parser + readErr error + eof bool + results *results + tokens []rune + matchLast bool + level int + tr []TraceEntry + maxTraceLength int } -func newContext(r io.RuneReader, keywords []parser) *context { +func newContext(r io.RuneReader, keywords []parser, maxTraceLength int) *context { return &context{ - reader: r, - keywords: keywords, - results: &results{}, - offsetLimit: -1, - failOffset: -1, + reader: r, + keywords: keywords, + results: &results{}, + offsetLimit: -1, + failOffset: -1, + maxTraceLength: maxTraceLength, } } @@ -199,11 +199,15 @@ func (c *context) finalizeParse(root parser) error { } func (c *context) trace(p parser, from, to int, event TraceEvent, reason ...string) { + if c.maxTraceLength <= 0 { + return + } + if p.commitType()&userDefined == 0 || p.commitType()&FailPass != 0 { return } - if len(c.tr) == maxTraceEntries { + if len(c.tr) == c.maxTraceLength { c.tr = c.tr[1:] } diff --git a/head.gen.go b/head.gen.go index ccce581..1ac8c31 100644 --- a/head.gen.go +++ b/head.gen.go @@ -1,4 +1,4 @@ package treerack // generated with script/createhead.go -const headCode = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() commitType {\n\treturn alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount, itemIndex\tint\n\t\tparsed\t\t\tbool\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"no match\")\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.trace(p, from, to, Success)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.trace(p, from, to, Success)\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&whitespace == 0 && p.commitType()&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.trace(p, from, to, Fail, \"no match\")\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nconst maxTraceEntries = 36\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n\tlevel\t\tint\n\ttr\t\t[]TraceEntry\n}\n\nfunc newContext(r io.RuneReader, keywords []parser) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = errInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\tif offset < 0 {\n\t\treturn 0, 0\n\t}\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser, unexpectedInput bool, root int) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\tueLine, ueCol := -1, -1\n\tif unexpectedInput {\n\t\tto, _, _ := c.results.longestResult(0, root)\n\t\tueLine, ueCol = findLine(c.tokens, to)\n\t}\n\tfor i := range c.tr {\n\t\tc.tr[i].FromLine, c.tr[i].FromCol = findLine(c.tokens, c.tr[i].From)\n\t\tc.tr[i].ToLine, c.tr[i].ToCol = findLine(c.tokens, c.tr[i].To)\n\t}\n\treturn &parseError{inputContent: c.tokens, Offset: c.failOffset, Line: line, Column: col, Definition: definition, Trace: c.tr, UnexpectedInputLine: ueLine, UnexpectedInputCol: ueCol}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp, found && match && to < c.readOffset, root.nodeID())\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root, false, root.nodeID())\n}\nfunc (c *context) trace(p parser, from, to int, event TraceEvent, reason ...string) {\n\tif p.commitType()&userDefined == 0 || p.commitType()&failPass != 0 {\n\t\treturn\n\t}\n\tif len(c.tr) == maxTraceEntries {\n\t\tc.tr = c.tr[1:]\n\t}\n\tswitch event {\n\tcase Success, Fail:\n\t\tc.level--\n\t}\n\tc.tr = append(c.tr, TraceEntry{Level: c.level, Parser: p.nodeName(), From: from, To: to, Event: event, Reason: strings.Join(reason, \"; \")})\n\tswitch event {\n\tcase Enter:\n\t\tc.level++\n\t}\n}\n\ntype node struct {\n\tName\tstring\n\tNodes\t[]node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype commitType int\n\nconst (\n\tnone\tcommitType\t= 0\n\talias\tcommitType\t= 1 << iota\n\twhitespace\n\tnoWhitespace\n\tkeyword\n\tnoKeyword\n\tfailPass\n\tNoFailPass\n\troot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype formatOptions struct {\n\tmode\t\tformatFlags\n\ttargetWidth\tint\n}\ntype TraceEvent int\n\nconst (\n\tEnter\tTraceEvent\t= iota\n\tSuccess\n\tFail\n)\n\ntype TraceEntry struct {\n\tLevel\t\tint\n\tParser\t\tstring\n\tFrom\t\tint\n\tTo\t\tint\n\tFromLine\tint\n\tFromCol\t\tint\n\tToLine\t\tint\n\tToCol\t\tint\n\tEvent\t\tTraceEvent\n\tReason\t\tstring\n}\ntype parseError struct {\n\tinputContent\t\t[]rune\n\tInput\t\t\tstring\n\tOffset\t\t\tint\n\tLine\t\t\tint\n\tColumn\t\t\tint\n\tDefinition\t\tstring\n\tTrace\t\t\t[]TraceEntry\n\tUnexpectedInputLine\tint\n\tUnexpectedInputCol\tint\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() commitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]node, bool)\n}\n\nvar errInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *parseError) InputContent(from, to int) []rune {\n\tif to < 0 {\n\t\tto = len(pe.inputContent)\n\t}\n\tif to <= from {\n\t\treturn nil\n\t}\n\tc := make([]rune, to-from)\n\tcopy(c, pe.inputContent[from:])\n\treturn c\n}\nfunc (pe *parseError) Error() string {\n\tif pe.UnexpectedInputLine >= 0 && pe.UnexpectedInputCol >= 0 {\n\t\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, unexpected input at %d:%d\", pe.Input, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1)\n\t}\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s, at %d:%d\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition, pe.Line+1, pe.Column+1)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser) (node, error) {\n\tc := newContext(bufio.NewReader(r), kw)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn node{}, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*parseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn node{}, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" +const headCode = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() commitType {\n\treturn alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount, itemIndex\tint\n\t\tparsed\t\t\tbool\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"no match\")\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.trace(p, from, to, Success)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.trace(p, from, to, Success)\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&whitespace == 0 && p.commitType()&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.trace(p, from, to, Fail, \"no match\")\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n\tlevel\t\tint\n\ttr\t\t[]TraceEntry\n\tmaxTraceLength\tint\n}\n\nfunc newContext(r io.RuneReader, keywords []parser, maxTraceLength int) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1, maxTraceLength: maxTraceLength}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = errInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\tif offset < 0 {\n\t\treturn 0, 0\n\t}\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser, unexpectedInput bool, root int) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\tueLine, ueCol := -1, -1\n\tif unexpectedInput {\n\t\tto, _, _ := c.results.longestResult(0, root)\n\t\tueLine, ueCol = findLine(c.tokens, to)\n\t}\n\tfor i := range c.tr {\n\t\tc.tr[i].FromLine, c.tr[i].FromCol = findLine(c.tokens, c.tr[i].From)\n\t\tc.tr[i].ToLine, c.tr[i].ToCol = findLine(c.tokens, c.tr[i].To)\n\t}\n\treturn &parseError{inputContent: c.tokens, Offset: c.failOffset, Line: line, Column: col, Definition: definition, Trace: c.tr, UnexpectedInputLine: ueLine, UnexpectedInputCol: ueCol}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp, found && match && to < c.readOffset, root.nodeID())\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root, false, root.nodeID())\n}\nfunc (c *context) trace(p parser, from, to int, event TraceEvent, reason ...string) {\n\tif c.maxTraceLength <= 0 {\n\t\treturn\n\t}\n\tif p.commitType()&userDefined == 0 || p.commitType()&failPass != 0 {\n\t\treturn\n\t}\n\tif len(c.tr) == c.maxTraceLength {\n\t\tc.tr = c.tr[1:]\n\t}\n\tswitch event {\n\tcase Success, Fail:\n\t\tc.level--\n\t}\n\tc.tr = append(c.tr, TraceEntry{Level: c.level, Parser: p.nodeName(), From: from, To: to, Event: event, Reason: strings.Join(reason, \"; \")})\n\tswitch event {\n\tcase Enter:\n\t\tc.level++\n\t}\n}\n\ntype node struct {\n\tName\tstring\n\tNodes\t[]node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype commitType int\n\nconst (\n\tnone\tcommitType\t= 0\n\talias\tcommitType\t= 1 << iota\n\twhitespace\n\tnoWhitespace\n\tkeyword\n\tnoKeyword\n\tfailPass\n\tNoFailPass\n\troot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype formatOptions struct {\n\tmode\t\tformatFlags\n\ttargetWidth\tint\n}\ntype TraceEvent int\n\nconst (\n\tEnter\tTraceEvent\t= iota\n\tSuccess\n\tFail\n)\n\ntype TraceEntry struct {\n\tLevel\t\tint\n\tParser\t\tstring\n\tFrom\t\tint\n\tTo\t\tint\n\tFromLine\tint\n\tFromCol\t\tint\n\tToLine\t\tint\n\tToCol\t\tint\n\tEvent\t\tTraceEvent\n\tReason\t\tstring\n}\ntype parseError struct {\n\tinputContent\t\t[]rune\n\tInput\t\t\tstring\n\tOffset\t\t\tint\n\tLine\t\t\tint\n\tColumn\t\t\tint\n\tDefinition\t\tstring\n\tTrace\t\t\t[]TraceEntry\n\tUnexpectedInputLine\tint\n\tUnexpectedInputCol\tint\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() commitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]node, bool)\n}\n\nvar errInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *parseError) InputContent(from, to int) []rune {\n\tif to < 0 {\n\t\tto = len(pe.inputContent)\n\t}\n\tif to <= from {\n\t\treturn nil\n\t}\n\tc := make([]rune, to-from)\n\tcopy(c, pe.inputContent[from:])\n\treturn c\n}\nfunc (pe *parseError) Error() string {\n\tif pe.UnexpectedInputLine >= 0 && pe.UnexpectedInputCol >= 0 {\n\t\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, unexpected input at %d:%d\", pe.Input, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1)\n\t}\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s, at %d:%d\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition, pe.Line+1, pe.Column+1)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser, maxTraceLength int) (node, error) {\n\tc := newContext(bufio.NewReader(r), kw, maxTraceLength)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn node{}, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*parseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn node{}, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" diff --git a/headexported.gen.go b/headexported.gen.go index 22ed479..f22fe15 100644 --- a/headexported.gen.go +++ b/headexported.gen.go @@ -1,4 +1,4 @@ package treerack // generated with script/createhead.go -const headCodeExported = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() CommitType {\n\treturn Alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]Node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount, itemIndex\tint\n\t\tparsed\t\t\tbool\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"no match\")\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.trace(p, from, to, Success)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&Alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []Node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]Node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []Node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.trace(p, from, to, Success)\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&Whitespace == 0 && p.commitType()&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.trace(p, from, to, Fail, \"no match\")\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []Node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nconst maxTraceEntries = 36\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n\tlevel\t\tint\n\ttr\t\t[]TraceEntry\n}\n\nfunc newContext(r io.RuneReader, keywords []parser) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = ErrInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\tif offset < 0 {\n\t\treturn 0, 0\n\t}\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser, unexpectedInput bool, root int) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\tueLine, ueCol := -1, -1\n\tif unexpectedInput {\n\t\tto, _, _ := c.results.longestResult(0, root)\n\t\tueLine, ueCol = findLine(c.tokens, to)\n\t}\n\tfor i := range c.tr {\n\t\tc.tr[i].FromLine, c.tr[i].FromCol = findLine(c.tokens, c.tr[i].From)\n\t\tc.tr[i].ToLine, c.tr[i].ToCol = findLine(c.tokens, c.tr[i].To)\n\t}\n\treturn &ParseError{inputContent: c.tokens, Offset: c.failOffset, Line: line, Column: col, Definition: definition, Trace: c.tr, UnexpectedInputLine: ueLine, UnexpectedInputCol: ueCol}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp, found && match && to < c.readOffset, root.nodeID())\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root, false, root.nodeID())\n}\nfunc (c *context) trace(p parser, from, to int, event TraceEvent, reason ...string) {\n\tif p.commitType()&userDefined == 0 || p.commitType()&FailPass != 0 {\n\t\treturn\n\t}\n\tif len(c.tr) == maxTraceEntries {\n\t\tc.tr = c.tr[1:]\n\t}\n\tswitch event {\n\tcase Success, Fail:\n\t\tc.level--\n\t}\n\tc.tr = append(c.tr, TraceEntry{Level: c.level, Parser: p.nodeName(), From: from, To: to, Event: event, Reason: strings.Join(reason, \"; \")})\n\tswitch event {\n\tcase Enter:\n\t\tc.level++\n\t}\n}\n\ntype Node struct {\n\tName\tstring\n\tNodes\t[]Node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n Node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n Node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n Node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype CommitType int\n\nconst (\n\tNone\tCommitType\t= 0\n\tAlias\tCommitType\t= 1 << iota\n\tWhitespace\n\tNoWhitespace\n\tKeyword\n\tNoKeyword\n\tFailPass\n\tNoFailPass\n\tRoot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype formatOptions struct {\n\tmode\t\tformatFlags\n\ttargetWidth\tint\n}\ntype TraceEvent int\n\nconst (\n\tEnter\tTraceEvent\t= iota\n\tSuccess\n\tFail\n)\n\ntype TraceEntry struct {\n\tLevel\t\tint\n\tParser\t\tstring\n\tFrom\t\tint\n\tTo\t\tint\n\tFromLine\tint\n\tFromCol\t\tint\n\tToLine\t\tint\n\tToCol\t\tint\n\tEvent\t\tTraceEvent\n\tReason\t\tstring\n}\ntype ParseError struct {\n\tinputContent\t\t[]rune\n\tInput\t\t\tstring\n\tOffset\t\t\tint\n\tLine\t\t\tint\n\tColumn\t\t\tint\n\tDefinition\t\tstring\n\tTrace\t\t\t[]TraceEntry\n\tUnexpectedInputLine\tint\n\tUnexpectedInputCol\tint\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() CommitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]Node, bool)\n}\n\nvar ErrInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *ParseError) InputContent(from, to int) []rune {\n\tif to < 0 {\n\t\tto = len(pe.inputContent)\n\t}\n\tif to <= from {\n\t\treturn nil\n\t}\n\tc := make([]rune, to-from)\n\tcopy(c, pe.inputContent[from:])\n\treturn c\n}\nfunc (pe *ParseError) Error() string {\n\tif pe.UnexpectedInputLine >= 0 && pe.UnexpectedInputCol >= 0 {\n\t\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, unexpected input at %d:%d\", pe.Input, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1)\n\t}\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s, at %d:%d\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition, pe.Line+1, pe.Column+1)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser) (Node, error) {\n\tc := newContext(bufio.NewReader(r), kw)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn Node{}, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*ParseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn Node{}, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" +const headCodeExported = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() CommitType {\n\treturn Alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]Node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount, itemIndex\tint\n\t\tparsed\t\t\tbool\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"no match\")\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.trace(p, from, to, Success)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&Alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []Node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]Node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []Node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tc.trace(p, c.offset, c.offset, Enter)\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.trace(p, c.offset, c.offset, Fail, \"same position recursion\")\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.trace(p, from, to, Fail, \"illegal keyword\")\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.trace(p, from, to, Success)\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&Whitespace == 0 && p.commitType()&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.trace(p, from, to, Fail, \"no match\")\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []Node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n\tlevel\t\tint\n\ttr\t\t[]TraceEntry\n\tmaxTraceLength\tint\n}\n\nfunc newContext(r io.RuneReader, keywords []parser, maxTraceLength int) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1, maxTraceLength: maxTraceLength}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = ErrInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\tif offset < 0 {\n\t\treturn 0, 0\n\t}\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser, unexpectedInput bool, root int) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\tueLine, ueCol := -1, -1\n\tif unexpectedInput {\n\t\tto, _, _ := c.results.longestResult(0, root)\n\t\tueLine, ueCol = findLine(c.tokens, to)\n\t}\n\tfor i := range c.tr {\n\t\tc.tr[i].FromLine, c.tr[i].FromCol = findLine(c.tokens, c.tr[i].From)\n\t\tc.tr[i].ToLine, c.tr[i].ToCol = findLine(c.tokens, c.tr[i].To)\n\t}\n\treturn &ParseError{inputContent: c.tokens, Offset: c.failOffset, Line: line, Column: col, Definition: definition, Trace: c.tr, UnexpectedInputLine: ueLine, UnexpectedInputCol: ueCol}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp, found && match && to < c.readOffset, root.nodeID())\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root, false, root.nodeID())\n}\nfunc (c *context) trace(p parser, from, to int, event TraceEvent, reason ...string) {\n\tif c.maxTraceLength <= 0 {\n\t\treturn\n\t}\n\tif p.commitType()&userDefined == 0 || p.commitType()&FailPass != 0 {\n\t\treturn\n\t}\n\tif len(c.tr) == c.maxTraceLength {\n\t\tc.tr = c.tr[1:]\n\t}\n\tswitch event {\n\tcase Success, Fail:\n\t\tc.level--\n\t}\n\tc.tr = append(c.tr, TraceEntry{Level: c.level, Parser: p.nodeName(), From: from, To: to, Event: event, Reason: strings.Join(reason, \"; \")})\n\tswitch event {\n\tcase Enter:\n\t\tc.level++\n\t}\n}\n\ntype Node struct {\n\tName\tstring\n\tNodes\t[]Node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n Node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n Node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n Node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype CommitType int\n\nconst (\n\tNone\tCommitType\t= 0\n\tAlias\tCommitType\t= 1 << iota\n\tWhitespace\n\tNoWhitespace\n\tKeyword\n\tNoKeyword\n\tFailPass\n\tNoFailPass\n\tRoot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype formatOptions struct {\n\tmode\t\tformatFlags\n\ttargetWidth\tint\n}\ntype TraceEvent int\n\nconst (\n\tEnter\tTraceEvent\t= iota\n\tSuccess\n\tFail\n)\n\ntype TraceEntry struct {\n\tLevel\t\tint\n\tParser\t\tstring\n\tFrom\t\tint\n\tTo\t\tint\n\tFromLine\tint\n\tFromCol\t\tint\n\tToLine\t\tint\n\tToCol\t\tint\n\tEvent\t\tTraceEvent\n\tReason\t\tstring\n}\ntype ParseError struct {\n\tinputContent\t\t[]rune\n\tInput\t\t\tstring\n\tOffset\t\t\tint\n\tLine\t\t\tint\n\tColumn\t\t\tint\n\tDefinition\t\tstring\n\tTrace\t\t\t[]TraceEntry\n\tUnexpectedInputLine\tint\n\tUnexpectedInputCol\tint\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() CommitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]Node, bool)\n}\n\nvar ErrInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *ParseError) InputContent(from, to int) []rune {\n\tif to < 0 {\n\t\tto = len(pe.inputContent)\n\t}\n\tif to <= from {\n\t\treturn nil\n\t}\n\tc := make([]rune, to-from)\n\tcopy(c, pe.inputContent[from:])\n\treturn c\n}\nfunc (pe *ParseError) Error() string {\n\tif pe.UnexpectedInputLine >= 0 && pe.UnexpectedInputCol >= 0 {\n\t\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, unexpected input at %d:%d\", pe.Input, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1, pe.UnexpectedInputLine+1, pe.UnexpectedInputCol+1)\n\t}\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s, at %d:%d\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition, pe.Line+1, pe.Column+1)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser, maxTraceLength int) (Node, error) {\n\tc := newContext(bufio.NewReader(r), kw, maxTraceLength)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn Node{}, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*ParseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn Node{}, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" diff --git a/internal/self/self.gen.go b/internal/self/self.gen.go index a34aae6..17eb8da 100644 --- a/internal/self/self.gen.go +++ b/internal/self/self.gen.go @@ -603,28 +603,27 @@ func (r *results) unmarkPending(offset, id int) { } } -const maxTraceEntries = 36 - type context struct { - reader io.RuneReader - keywords []parser - offset int - readOffset int - consumed int - offsetLimit int - failOffset int - failingParser parser - readErr error - eof bool - results *results - tokens []rune - matchLast bool - level int - tr []TraceEntry + reader io.RuneReader + keywords []parser + offset int + readOffset int + consumed int + offsetLimit int + failOffset int + failingParser parser + readErr error + eof bool + results *results + tokens []rune + matchLast bool + level int + tr []TraceEntry + maxTraceLength int } -func newContext(r io.RuneReader, keywords []parser) *context { - return &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1} +func newContext(r io.RuneReader, keywords []parser, maxTraceLength int) *context { + return &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1, maxTraceLength: maxTraceLength} } func (c *context) read() bool { if c.eof || c.readErr != nil { @@ -753,10 +752,13 @@ func (c *context) finalizeParse(root parser) error { return c.parseError(root, false, root.nodeID()) } func (c *context) trace(p parser, from, to int, event TraceEvent, reason ...string) { + if c.maxTraceLength <= 0 { + return + } if p.commitType()&userDefined == 0 || p.commitType()&FailPass != 0 { return } - if len(c.tr) == maxTraceEntries { + if len(c.tr) == c.maxTraceLength { c.tr = c.tr[1:] } switch event { @@ -877,8 +879,8 @@ func (pe *ParseError) Error() string { } return fmt.Sprintf("%s:%d:%d:parse failed, parsing: %s, at %d:%d", pe.Input, pe.Line+1, pe.Column+1, pe.Definition, pe.Line+1, pe.Column+1) } -func parseInput(r io.Reader, p parser, b builder, kw []parser) (Node, error) { - c := newContext(bufio.NewReader(r), kw) +func parseInput(r io.Reader, p parser, b builder, kw []parser, maxTraceLength int) (Node, error) { + c := newContext(bufio.NewReader(r), kw, maxTraceLength) p.parse(c) if c.readErr != nil { return Node{}, c.readErr @@ -1468,5 +1470,5 @@ func Parse(r io.Reader) (Node, error) { var keywords = []parser{} - return parseInput(r, &p183, &b183, keywords) + return parseInput(r, &p183, &b183, keywords, 0) } diff --git a/parse_test.go b/parse_test.go index 020a6f9..148a394 100644 --- a/parse_test.go +++ b/parse_test.go @@ -776,7 +776,7 @@ func TestCharBuildNoop(t *testing.T) { c := newChar("foo", false, nil, nil) c.init(newRegistry()) b := c.builder() - ctx := newContext(bufio.NewReader(bytes.NewBuffer(nil)), nil) + ctx := newContext(bufio.NewReader(bytes.NewBuffer(nil)), nil, 0) if n, ok := b.build(ctx); len(n) != 0 || ok { t.Error("char build not noop") } diff --git a/syntax.go b/syntax.go index 169563a..855ab50 100644 --- a/syntax.go +++ b/syntax.go @@ -62,6 +62,10 @@ type Syntax struct { explicitRoot bool keywords []definition root definition + + // MaxTraceLength can be used to enable tracing by setting a positive value. It should be set before + // parsing. + MaxTraceLength int } // GeneratorOptions control the behavior of the Go code generator. @@ -570,7 +574,7 @@ func (s *Syntax) Generate(o GeneratorOptions, w io.Writer) error { fprintln() fprintln() - fprintf(`return parseInput(r, &p%d, &b%d, keywords)`, s.root.parser().nodeID(), s.root.builder().nodeID()) + fprintf(`return parseInput(r, &p%d, &b%d, keywords, 0)`, s.root.parser().nodeID(), s.root.builder().nodeID()) fprintln() fprint(`}`) fprintln() @@ -584,7 +588,7 @@ func (s *Syntax) Parse(r io.Reader) (Node, error) { return Node{}, err } - return parseInput(r, s.root.parser(), s.root.builder(), s.keywordParsers()) + return parseInput(r, s.root.parser(), s.root.builder(), s.keywordParsers(), s.MaxTraceLength) } // Format prints the loaded syntax definition to the output in a formatted way. diff --git a/syntaxhead.go b/syntaxhead.go index b5352fc..f740113 100644 --- a/syntaxhead.go +++ b/syntaxhead.go @@ -199,8 +199,8 @@ func (pe *ParseError) Error() string { ) } -func parseInput(r io.Reader, p parser, b builder, kw []parser) (Node, error) { - c := newContext(bufio.NewReader(r), kw) +func parseInput(r io.Reader, p parser, b builder, kw []parser, maxTraceLength int) (Node, error) { + c := newContext(bufio.NewReader(r), kw, maxTraceLength) p.parse(c) if c.readErr != nil { return Node{}, c.readErr