From 2fe6f88ed682d933915de764a8e559a85653061b Mon Sep 17 00:00:00 2001 From: Arpad Ryszka Date: Wed, 21 Jan 2026 20:54:16 +0100 Subject: [PATCH] error fixes and doc updates --- Makefile | 1 + cmd/treerack/check.go | 8 +-- cmd/treerack/check_test.go | 62 ++++++++++++++--------- cmd/treerack/checksyntax.go | 4 +- cmd/treerack/checksyntax_test.go | 10 ++-- cmd/treerack/docreflect.gen.go | 85 ++++++++++++++++---------------- cmd/treerack/generate.go | 4 +- cmd/treerack/generate_test.go | 14 +++--- cmd/treerack/input.go | 31 ++++++------ cmd/treerack/show.go | 8 +-- cmd/treerack/show_test.go | 60 +++++++++++----------- context.go | 5 +- docs/examples/acalc/main.go | 43 ++++++---------- docs/manual.md | 60 +++++++++++----------- docs/syntax.md | 64 ++++++++++++------------ head.go | 2 +- headexported.go | 2 +- internal/self/self.go | 3 ++ notes.txt | 37 +------------- parse_test.go | 33 +++++++++++++ readme.md | 18 ++++--- 21 files changed, 285 insertions(+), 269 deletions(-) diff --git a/Makefile b/Makefile index a15274b..0183be9 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,7 @@ lib: $(sources) head.go headexported.go internal/self/self.go cmd/treerack/docreflect.gen.go: $(sources) .build go run scripts/docreflect.go > .build/docreflect.gen.go + go fmt .build/docreflect.gen.go mv .build/docreflect.gen.go cmd/treerack cmd/treerack/readme.md: $(sources) cmd/treerack/docreflect.gen.go diff --git a/cmd/treerack/check.go b/cmd/treerack/check.go index 22c5690..9fa01ff 100644 --- a/cmd/treerack/check.go +++ b/cmd/treerack/check.go @@ -8,16 +8,16 @@ import ( type checkOptions struct { // Syntax specifies the filename of the syntax definition file. - Syntax string + Syntax *string // SyntaxString specifies the syntax as an inline string. - SyntaxString string + SyntaxString *string // Input specifies the filename of the input content to be validated. - Input string + Input *string // InputString specifies the input content as an inline string. - InputString string + InputString *string } // check parses input content against the provided syntax definition and fails if the input does not match. diff --git a/cmd/treerack/check_test.go b/cmd/treerack/check_test.go index 5095e59..9fbe2e8 100644 --- a/cmd/treerack/check_test.go +++ b/cmd/treerack/check_test.go @@ -8,9 +8,13 @@ import ( "testing" ) +func ptrto[T any](v T) *T { + return &v +} + func TestCheck(t *testing.T) { t.Run("no syntax", func(t *testing.T) { - o := checkOptions{Input: "bar_test.txt"} + o := checkOptions{Input: ptrto("bar_test.txt")} if err := check(o, nil); !errors.Is(err, errNoInput) { t.Fatal() } @@ -18,9 +22,9 @@ func TestCheck(t *testing.T) { t.Run("too many syntaxes", func(t *testing.T) { o := checkOptions{ - Syntax: "foo_test.treerack", - SyntaxString: `foo = "baz"`, - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + SyntaxString: ptrto(`foo = "baz"`), + Input: ptrto("bar_test.txt"), } if err := check(o, nil); !errors.Is(err, errMultipleInputs) { @@ -30,8 +34,8 @@ func TestCheck(t *testing.T) { t.Run("syntax file not found", func(t *testing.T) { o := checkOptions{ - Syntax: "no-file.treerack", - Input: "bar_test.txt", + Syntax: ptrto("no-file.treerack"), + Input: ptrto("bar_test.txt"), } if err := check(o, nil); !os.IsNotExist(err) { @@ -41,8 +45,8 @@ func TestCheck(t *testing.T) { t.Run("invalid syntax definition", func(t *testing.T) { o := checkOptions{ - SyntaxString: `foo`, - Input: "bar_test.txt", + SyntaxString: ptrto(`foo`), + Input: ptrto("bar_test.txt"), } var perr *treerack.ParseError @@ -53,8 +57,8 @@ func TestCheck(t *testing.T) { t.Run("invalid syntax init", func(t *testing.T) { o := checkOptions{ - SyntaxString: `foo = "bar"; foo = "baz"`, - Input: "bar_test.txt", + SyntaxString: ptrto(`foo = "bar"; foo = "baz"`), + Input: ptrto("bar_test.txt"), } if err := check(o, nil); err == nil { @@ -63,7 +67,7 @@ func TestCheck(t *testing.T) { }) t.Run("no input", func(t *testing.T) { - o := checkOptions{Syntax: "foo_test.treerack"} + o := checkOptions{Syntax: ptrto("foo_test.treerack")} if err := check(o, nil); !errors.Is(err, errNoInput) { t.Fatal() @@ -72,8 +76,8 @@ func TestCheck(t *testing.T) { t.Run("too many inputs", func(t *testing.T) { o := checkOptions{ - Syntax: "foo_test.treerack", - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + Input: ptrto("bar_test.txt"), } if err := check(o, nil, "baz_test.txt"); !errors.Is(err, errMultipleInputs) { @@ -82,14 +86,14 @@ func TestCheck(t *testing.T) { }) t.Run("empty filename for input", func(t *testing.T) { - o := checkOptions{Syntax: "foo_test.treerack"} + o := checkOptions{Syntax: ptrto("foo_test.treerack")} if err := check(o, nil, ""); !errors.Is(err, errInvalidFilename) { t.Fatal() } }) t.Run("input file not found", func(t *testing.T) { - o := checkOptions{Syntax: "foo_test.treerack"} + o := checkOptions{Syntax: ptrto("foo_test.treerack")} if err := check(o, nil, "baz_test.txt"); !os.IsNotExist(err) { t.Fatal() } @@ -97,8 +101,8 @@ func TestCheck(t *testing.T) { t.Run("input parse fail", func(t *testing.T) { o := checkOptions{ - Syntax: "foo_test.treerack", - InputString: "baz", + Syntax: ptrto("foo_test.treerack"), + InputString: ptrto("baz"), } var perr *treerack.ParseError @@ -109,8 +113,8 @@ func TestCheck(t *testing.T) { t.Run("input parse success", func(t *testing.T) { o := checkOptions{ - Syntax: "foo_test.treerack", - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + Input: ptrto("bar_test.txt"), } if err := check(o, nil); err != nil { @@ -120,8 +124,8 @@ func TestCheck(t *testing.T) { t.Run("input from string success", func(t *testing.T) { o := checkOptions{ - Syntax: "foo_test.treerack", - InputString: "bar", + Syntax: ptrto("foo_test.treerack"), + InputString: ptrto("bar"), } if err := check(o, nil); err != nil { @@ -130,17 +134,29 @@ func TestCheck(t *testing.T) { }) t.Run("input from file success", func(t *testing.T) { - o := checkOptions{Syntax: "foo_test.treerack"} + o := checkOptions{Syntax: ptrto("foo_test.treerack")} if err := check(o, nil, "bar_test.txt"); err != nil { t.Fatal(err) } }) t.Run("input from stdin success", func(t *testing.T) { - o := checkOptions{Syntax: "foo_test.treerack"} + o := checkOptions{Syntax: ptrto("foo_test.treerack")} buf := bytes.NewBufferString("bar") if err := check(o, buf); err != nil { t.Fatal(err) } }) + + t.Run("empty input filename", func(t *testing.T) { + o := checkOptions{ + Syntax: ptrto("foo_test.treerack"), + Input: ptrto(""), + } + + var buf bytes.Buffer + if err := check(o, &buf); !errors.Is(err, errInvalidFilename) { + t.Fatal() + } + }) } diff --git a/cmd/treerack/checksyntax.go b/cmd/treerack/checksyntax.go index aa0d728..5e6784c 100644 --- a/cmd/treerack/checksyntax.go +++ b/cmd/treerack/checksyntax.go @@ -8,10 +8,10 @@ import ( type checkSyntaxOptions struct { // Syntax specifies the filename of the syntax definition file. - Syntax string + Syntax *string // SyntaxString specifies the syntax as an inline string. - SyntaxString string + SyntaxString *string } // checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a diff --git a/cmd/treerack/checksyntax_test.go b/cmd/treerack/checksyntax_test.go index 7314fe9..e710e17 100644 --- a/cmd/treerack/checksyntax_test.go +++ b/cmd/treerack/checksyntax_test.go @@ -10,7 +10,7 @@ import ( func TestCheckSyntax(t *testing.T) { t.Run("too many inputs", func(t *testing.T) { - o := checkSyntaxOptions{Syntax: "foo_test.treerack", SyntaxString: `foo = "42"`} + o := checkSyntaxOptions{Syntax: ptrto("foo_test.treerack"), SyntaxString: ptrto(`foo = "42"`)} if err := checkSyntax(o, nil); !errors.Is(err, errMultipleInputs) { t.Fatal() } @@ -32,28 +32,28 @@ func TestCheckSyntax(t *testing.T) { t.Run("invalid syntax", func(t *testing.T) { var perr *treerack.ParseError - o := checkSyntaxOptions{SyntaxString: "foo"} + o := checkSyntaxOptions{SyntaxString: ptrto("foo")} if err := checkSyntax(o, nil); !errors.As(err, &perr) { t.Fatal() } }) t.Run("invalid syntax init", func(t *testing.T) { - o := checkSyntaxOptions{SyntaxString: `foo = "42"; foo = "84"`} + o := checkSyntaxOptions{SyntaxString: ptrto(`foo = "42"; foo = "84"`)} if err := checkSyntax(o, nil); err == nil { t.Fatal() } }) t.Run("success", func(t *testing.T) { - o := checkSyntaxOptions{Syntax: "foo_test.treerack"} + o := checkSyntaxOptions{Syntax: ptrto("foo_test.treerack")} if err := checkSyntax(o, nil); err != nil { t.Fatal(err) } }) t.Run("from string success", func(t *testing.T) { - o := checkSyntaxOptions{SyntaxString: `foo = "bar"`} + o := checkSyntaxOptions{SyntaxString: ptrto(`foo = "bar"`)} if err := checkSyntax(o, nil); err != nil { t.Fatal(err) } diff --git a/cmd/treerack/docreflect.gen.go b/cmd/treerack/docreflect.gen.go index 3ef65ba..d2f5c3d 100644 --- a/cmd/treerack/docreflect.gen.go +++ b/cmd/treerack/docreflect.gen.go @@ -2,48 +2,49 @@ Generated with https://code.squareroundforest.org/arpio/docreflect */ - package main + import "code.squareroundforest.org/arpio/docreflect" + func init() { -docreflect.Register("main", "") -docreflect.Register("main.check", "check parses input content against the provided syntax definition and fails if the input does not match.\nSyntax can be provided via a filename option or an inline string option. Input can be provided via a filename\noption, a positional argument filename, an inline string option, or piped from standard input.\n\nfunc(o, stdin, args)") -docreflect.Register("main.checkOptions", "") -docreflect.Register("main.checkOptions.Input", "Input specifies the filename of the input content to be validated.\n") -docreflect.Register("main.checkOptions.InputString", "InputString specifies the input content as an inline string.\n") -docreflect.Register("main.checkOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") -docreflect.Register("main.checkOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") -docreflect.Register("main.checkSyntax", "checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a\npositional argument), an inline string, or piped from standard input.\n\nfunc(o, stdin, args)") -docreflect.Register("main.checkSyntaxOptions", "") -docreflect.Register("main.checkSyntaxOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") -docreflect.Register("main.checkSyntaxOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") -docreflect.Register("main.errInvalidFilename", "") -docreflect.Register("main.errMultipleInputs", "") -docreflect.Register("main.errNoInput", "") -docreflect.Register("main.generate", "generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded\nin an application.\n\nThe syntax may be provided via a file path (using an option or a positional argument), an\ninline string, or piped from standard input.\n\nfunc(o, stdin, stdout, args)") -docreflect.Register("main.generateOptions", "") -docreflect.Register("main.generateOptions.Export", "Export determines whether the generated parse function is exported (visible outside its package).\n") -docreflect.Register("main.generateOptions.PackageName", "PackageName specifies the package name for the generated code. Defaults to main.\n") -docreflect.Register("main.generateOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") -docreflect.Register("main.generateOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") -docreflect.Register("main.init", "\nfunc()") -docreflect.Register("main.initInput", "\nfunc(filename, stringValue, stdin, args)") -docreflect.Register("main.main", "\nfunc()") -docreflect.Register("main.mapNode", "\nfunc(n)") -docreflect.Register("main.node", "") -docreflect.Register("main.node.From", "") -docreflect.Register("main.node.Name", "") -docreflect.Register("main.node.Nodes", "") -docreflect.Register("main.node.Text", "") -docreflect.Register("main.node.To", "") -docreflect.Register("main.noop", "\nfunc()") -docreflect.Register("main.show", "show input content against a provided syntax definition and outputs the resulting AST (Abstract Syntax Tree)\nin JSON format. Syntax can be provided via a filename option or an inline string option. Input can be\nprovided via a filename option, a positional argument filename, an inline string option, or piped from\nstandard input.\n\nfunc(o, stdin, stdout, args)") -docreflect.Register("main.showOptions", "") -docreflect.Register("main.showOptions.Indent", "Indent specifies a custom indentation string for the output.\n") -docreflect.Register("main.showOptions.Input", "Input specifies the filename of the input content to be validated.\n") -docreflect.Register("main.showOptions.InputString", "InputString specifies the input content as an inline string.\n") -docreflect.Register("main.showOptions.Pretty", "Pretty enables indented, human-readable output.\n") -docreflect.Register("main.showOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") -docreflect.Register("main.showOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") -docreflect.Register("main.version", "") -} \ No newline at end of file + docreflect.Register("main", "") + docreflect.Register("main.check", "check parses input content against the provided syntax definition and fails if the input does not match.\nSyntax can be provided via a filename option or an inline string option. Input can be provided via a filename\noption, a positional argument filename, an inline string option, or piped from standard input.\n\nfunc(o, stdin, args)") + docreflect.Register("main.checkOptions", "") + docreflect.Register("main.checkOptions.Input", "Input specifies the filename of the input content to be validated.\n") + docreflect.Register("main.checkOptions.InputString", "InputString specifies the input content as an inline string.\n") + docreflect.Register("main.checkOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") + docreflect.Register("main.checkOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") + docreflect.Register("main.checkSyntax", "checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a\npositional argument), an inline string, or piped from standard input.\n\nfunc(o, stdin, args)") + docreflect.Register("main.checkSyntaxOptions", "") + docreflect.Register("main.checkSyntaxOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") + docreflect.Register("main.checkSyntaxOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") + docreflect.Register("main.errInvalidFilename", "") + docreflect.Register("main.errMultipleInputs", "") + docreflect.Register("main.errNoInput", "") + docreflect.Register("main.generate", "generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded\nin an application.\n\nThe syntax may be provided via a file path (using an option or a positional argument), an\ninline string, or piped from standard input.\n\nfunc(o, stdin, stdout, args)") + docreflect.Register("main.generateOptions", "") + docreflect.Register("main.generateOptions.Export", "Export determines whether the generated parse function is exported (visible outside its package).\n") + docreflect.Register("main.generateOptions.PackageName", "PackageName specifies the package name for the generated code. Defaults to main.\n") + docreflect.Register("main.generateOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") + docreflect.Register("main.generateOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") + docreflect.Register("main.init", "\nfunc()") + docreflect.Register("main.initInput", "\nfunc(filename, stringValue, stdin, args)") + docreflect.Register("main.main", "\nfunc()") + docreflect.Register("main.mapNode", "\nfunc(n)") + docreflect.Register("main.node", "") + docreflect.Register("main.node.From", "") + docreflect.Register("main.node.Name", "") + docreflect.Register("main.node.Nodes", "") + docreflect.Register("main.node.Text", "") + docreflect.Register("main.node.To", "") + docreflect.Register("main.noop", "\nfunc()") + docreflect.Register("main.show", "show input content against a provided syntax definition and outputs the resulting AST (Abstract Syntax Tree)\nin JSON format. Syntax can be provided via a filename option or an inline string option. Input can be\nprovided via a filename option, a positional argument filename, an inline string option, or piped from\nstandard input.\n\nfunc(o, stdin, stdout, args)") + docreflect.Register("main.showOptions", "") + docreflect.Register("main.showOptions.Indent", "Indent specifies a custom indentation string for the output.\n") + docreflect.Register("main.showOptions.Input", "Input specifies the filename of the input content to be validated.\n") + docreflect.Register("main.showOptions.InputString", "InputString specifies the input content as an inline string.\n") + docreflect.Register("main.showOptions.Pretty", "Pretty enables indented, human-readable output.\n") + docreflect.Register("main.showOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") + docreflect.Register("main.showOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") + docreflect.Register("main.version", "") +} diff --git a/cmd/treerack/generate.go b/cmd/treerack/generate.go index ab09a02..1a2a890 100644 --- a/cmd/treerack/generate.go +++ b/cmd/treerack/generate.go @@ -8,10 +8,10 @@ import ( type generateOptions struct { // Syntax specifies the filename of the syntax definition file. - Syntax string + Syntax *string // SyntaxString specifies the syntax as an inline string. - SyntaxString string + SyntaxString *string // PackageName specifies the package name for the generated code. Defaults to main. PackageName string diff --git a/cmd/treerack/generate_test.go b/cmd/treerack/generate_test.go index 55f0b05..5c60462 100644 --- a/cmd/treerack/generate_test.go +++ b/cmd/treerack/generate_test.go @@ -12,7 +12,7 @@ import ( func TestGenerate(t *testing.T) { t.Run("too many inputs", func(t *testing.T) { var out bytes.Buffer - o := generateOptions{Syntax: "foo_test.treerack", SyntaxString: `foo = "42"`} + o := generateOptions{Syntax: ptrto("foo_test.treerack"), SyntaxString: ptrto(`foo = "42"`)} if err := generate(o, nil, &out); !errors.Is(err, errMultipleInputs) { t.Fatal() } @@ -46,7 +46,7 @@ func TestGenerate(t *testing.T) { perr *treerack.ParseError ) - o := generateOptions{SyntaxString: "foo"} + o := generateOptions{SyntaxString: ptrto("foo")} if err := generate(o, nil, &out); !errors.As(err, &perr) { t.Fatal() } @@ -54,7 +54,7 @@ func TestGenerate(t *testing.T) { t.Run("invalid syntax init", func(t *testing.T) { var out bytes.Buffer - o := generateOptions{SyntaxString: `foo = "42"; foo = "84"`} + o := generateOptions{SyntaxString: ptrto(`foo = "42"; foo = "84"`)} if err := generate(o, nil, &out); err == nil { t.Fatal() } @@ -62,7 +62,7 @@ func TestGenerate(t *testing.T) { t.Run("success", func(t *testing.T) { var out bytes.Buffer - o := generateOptions{Syntax: "foo_test.treerack"} + o := generateOptions{Syntax: ptrto("foo_test.treerack")} if err := generate(o, nil, &out); err != nil { t.Fatal(err) } @@ -75,7 +75,7 @@ func TestGenerate(t *testing.T) { t.Run("success string", func(t *testing.T) { var out bytes.Buffer - o := generateOptions{SyntaxString: `foo = "bar"`} + o := generateOptions{SyntaxString: ptrto(`foo = "bar"`)} if err := generate(o, nil, &out); err != nil { t.Fatal(err) } @@ -122,7 +122,7 @@ func TestGenerate(t *testing.T) { t.Run("custom package name", func(t *testing.T) { var out bytes.Buffer o := generateOptions{ - Syntax: "foo_test.treerack", + Syntax: ptrto("foo_test.treerack"), PackageName: "foo", } @@ -139,7 +139,7 @@ func TestGenerate(t *testing.T) { t.Run("export", func(t *testing.T) { var out bytes.Buffer o := generateOptions{ - Syntax: "foo_test.treerack", + Syntax: ptrto("foo_test.treerack"), Export: true, } diff --git a/cmd/treerack/input.go b/cmd/treerack/input.go index 8d43efb..29c8c65 100644 --- a/cmd/treerack/input.go +++ b/cmd/treerack/input.go @@ -17,16 +17,18 @@ var ( func noop() {} func initInput( - filename, stringValue string, stdin io.Reader, args []string, -) (input io.Reader, finalize func(), err error) { + filename, stringValue *string, stdin io.Reader, args []string, +) ( + input io.Reader, finalize func(), err error, +) { finalize = noop var inputCount int - if filename != "" { + if filename != nil { inputCount++ } - if stringValue != "" { + if stringValue != nil { inputCount++ } @@ -39,19 +41,20 @@ func initInput( return } - if len(args) > 0 && args[0] == "" { - err = errInvalidFilename - return - } - if len(args) > 0 { - filename = args[0] + filename = new(string) + *filename = args[0] } switch { - case filename != "": + case filename != nil: + if *filename == "" { + err = errInvalidFilename + return + } + var f io.ReadCloser - f, err = os.Open(filename) + f, err = os.Open(*filename) if err != nil { return } @@ -63,8 +66,8 @@ func initInput( } input = f - case stringValue != "": - input = bytes.NewBufferString(stringValue) + case stringValue != nil: + input = bytes.NewBufferString(*stringValue) default: if stdin == nil { err = errNoInput diff --git a/cmd/treerack/show.go b/cmd/treerack/show.go index fef5c0c..912a53d 100644 --- a/cmd/treerack/show.go +++ b/cmd/treerack/show.go @@ -9,16 +9,16 @@ import ( type showOptions struct { // Syntax specifies the filename of the syntax definition file. - Syntax string + Syntax *string // SyntaxString specifies the syntax as an inline string. - SyntaxString string + SyntaxString *string // Input specifies the filename of the input content to be validated. - Input string + Input *string // InputString specifies the input content as an inline string. - InputString string + InputString *string // Pretty enables indented, human-readable output. Pretty bool diff --git a/cmd/treerack/show_test.go b/cmd/treerack/show_test.go index 818335f..3b422d1 100644 --- a/cmd/treerack/show_test.go +++ b/cmd/treerack/show_test.go @@ -11,7 +11,7 @@ import ( func TestShow(t *testing.T) { t.Run("no syntax", func(t *testing.T) { var out bytes.Buffer - o := showOptions{Input: "bar_test.txt"} + o := showOptions{Input: ptrto("bar_test.txt")} if err := show(o, nil, &out); !errors.Is(err, errNoInput) { t.Fatal() } @@ -20,9 +20,9 @@ func TestShow(t *testing.T) { t.Run("too many syntaxes", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - SyntaxString: `foo = "baz"`, - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + SyntaxString: ptrto(`foo = "baz"`), + Input: ptrto("bar_test.txt"), } if err := show(o, nil, &out); !errors.Is(err, errMultipleInputs) { @@ -33,8 +33,8 @@ func TestShow(t *testing.T) { t.Run("syntax file not found", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "no-file.treerack", - Input: "bar_test.txt", + Syntax: ptrto("no-file.treerack"), + Input: ptrto("bar_test.txt"), } if err := show(o, nil, &out); !os.IsNotExist(err) { @@ -45,8 +45,8 @@ func TestShow(t *testing.T) { t.Run("invalid syntax definition", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - SyntaxString: `foo`, - Input: "bar_test.txt", + SyntaxString: ptrto(`foo`), + Input: ptrto("bar_test.txt"), } var perr *treerack.ParseError @@ -58,8 +58,8 @@ func TestShow(t *testing.T) { t.Run("invalid syntax init", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - SyntaxString: `foo = "bar"; foo = "baz"`, - Input: "bar_test.txt", + SyntaxString: ptrto(`foo = "bar"; foo = "baz"`), + Input: ptrto("bar_test.txt"), } if err := show(o, nil, &out); err == nil { @@ -69,7 +69,7 @@ func TestShow(t *testing.T) { t.Run("no input", func(t *testing.T) { var out bytes.Buffer - o := showOptions{Syntax: "foo_test.treerack"} + o := showOptions{Syntax: ptrto("foo_test.treerack")} if err := show(o, nil, &out); !errors.Is(err, errNoInput) { t.Fatal() @@ -79,8 +79,8 @@ func TestShow(t *testing.T) { t.Run("too many inputs", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + Input: ptrto("bar_test.txt"), } if err := show(o, nil, &out, "baz_test.txt"); !errors.Is(err, errMultipleInputs) { @@ -90,7 +90,7 @@ func TestShow(t *testing.T) { t.Run("empty filename for input", func(t *testing.T) { var out bytes.Buffer - o := showOptions{Syntax: "foo_test.treerack"} + o := showOptions{Syntax: ptrto("foo_test.treerack")} if err := show(o, nil, &out, ""); !errors.Is(err, errInvalidFilename) { t.Fatal() } @@ -98,7 +98,7 @@ func TestShow(t *testing.T) { t.Run("input file not found", func(t *testing.T) { var out bytes.Buffer - o := showOptions{Syntax: "foo_test.treerack"} + o := showOptions{Syntax: ptrto("foo_test.treerack")} if err := show(o, nil, &out, "baz_test.txt"); !os.IsNotExist(err) { t.Fatal() } @@ -107,8 +107,8 @@ func TestShow(t *testing.T) { t.Run("input parse fail", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - InputString: "baz", + Syntax: ptrto("foo_test.treerack"), + InputString: ptrto("baz"), } var perr *treerack.ParseError @@ -120,8 +120,8 @@ func TestShow(t *testing.T) { t.Run("show", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + Input: ptrto("bar_test.txt"), } if err := show(o, nil, &out); err != nil { @@ -136,8 +136,8 @@ func TestShow(t *testing.T) { t.Run("show string", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - InputString: "bar", + Syntax: ptrto("foo_test.treerack"), + InputString: ptrto("bar"), } if err := show(o, nil, &out); err != nil { @@ -151,9 +151,7 @@ func TestShow(t *testing.T) { t.Run("show file", func(t *testing.T) { var out bytes.Buffer - o := showOptions{ - Syntax: "foo_test.treerack", - } + o := showOptions{Syntax: ptrto("foo_test.treerack")} if err := show(o, nil, &out, "bar_test.txt"); err != nil { t.Fatal(nil) @@ -166,7 +164,7 @@ func TestShow(t *testing.T) { t.Run("show stdin", func(t *testing.T) { var out bytes.Buffer - o := showOptions{Syntax: "foo_test.treerack"} + o := showOptions{Syntax: ptrto("foo_test.treerack")} in := bytes.NewBufferString("bar") if err := show(o, in, &out); err != nil { t.Fatal(nil) @@ -180,8 +178,8 @@ func TestShow(t *testing.T) { t.Run("indent", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + Input: ptrto("bar_test.txt"), Pretty: true, } @@ -198,8 +196,8 @@ func TestShow(t *testing.T) { t.Run("custom indent", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + Input: ptrto("bar_test.txt"), Indent: "xx", } @@ -215,8 +213,8 @@ func TestShow(t *testing.T) { t.Run("redundant custom indent", func(t *testing.T) { var out bytes.Buffer o := showOptions{ - Syntax: "foo_test.treerack", - Input: "bar_test.txt", + Syntax: ptrto("foo_test.treerack"), + Input: ptrto("bar_test.txt"), Pretty: true, Indent: "xx", } diff --git a/context.go b/context.go index 2fe2417..361f357 100644 --- a/context.go +++ b/context.go @@ -120,6 +120,10 @@ func (c *context) fail(offset int) { } func findLine(tokens []rune, offset int) (line, column int) { + if offset < 0 { + return 0, 0 + } + tokens = tokens[:offset] for i := range tokens { column++ @@ -144,7 +148,6 @@ func (c *context) parseError(p parser) error { } line, col := findLine(c.tokens, c.failOffset) - return &ParseError{ Offset: c.failOffset, Line: line, diff --git a/docs/examples/acalc/main.go b/docs/examples/acalc/main.go index 7adfce3..9f3266e 100644 --- a/docs/examples/acalc/main.go +++ b/docs/examples/acalc/main.go @@ -14,54 +14,48 @@ import ( var errExit = errors.New("exit") +// repl runs the Read-Eval-Print Loop. func repl(input io.Reader, output io.Writer) { - // use buffered io, to be able to read the input line-by-line: + + // use buffered io, to read the input line-by-line: buf := bufio.NewReader(os.Stdin) - // our REPL loop: + // our REPL: for { - // print a basic prompt: + // print a input prompt marker: if _, err := output.Write([]byte("> ")); err != nil { - - // we cannot fix it if there is an error here: log.Fatalln(err) } // read the input and handle the errors: expr, err := read(buf) - // when EOF, that means the user pressed Ctrl+D. Let's terminate the output with a conventional newline - // and exit: + // handle EOF (Ctrl+D): if errors.Is(err, io.EOF) { output.Write([]byte{'\n'}) os.Exit(0) } - // when errExit, that means the user entered exit: + // handle the explicit exit command: if errors.Is(err, errExit) { os.Exit(0) } - // if it's a parser error, we print and continue from reading again, to allow the user to fix the - // problem: + // handle parser errors (allow the user to retry): var perr *parseError if errors.As(err, &perr) { log.Println(err) continue } - // in case of any other error, we don't know what's going on, so we get out of here right away: + // handle possible I/O errors: if err != nil { log.Fatalln(err) } - // if we received an expression, then we can evaluate it. We are not expecting errors here: + // evaluate and print: result := eval(expr) - - // we have the result, we need to print it: if err := print(output, result); err != nil { - - // if printing fails, we don't know how to fix it, so we get out of here: log.Fatalln(err) } } @@ -73,7 +67,7 @@ func read(input *bufio.Reader) (*node, error) { return nil, err } - // expr will be of type *node, which type is defined in the generated code + // parse the line using the generated parser: expr, err := parse(bytes.NewBufferString(line)) if err != nil { return nil, err @@ -83,15 +77,12 @@ func read(input *bufio.Reader) (*node, error) { return nil, errExit } - // we know based on the syntax, that the top level node will always have a single child, either a number - // literal or a binary operation: + // based on our syntax, the root node always has exactly one child: either a number or a binary operation. return expr.Nodes[0], nil } // eval always returns the calculated result as a float64: func eval(expr *node) float64 { - - // we know that it's either a number or a binary operation: var value float64 switch expr.Name { case "num": @@ -103,10 +94,8 @@ func eval(expr *node) float64 { return value default: - // we know that the first node is either a number of a child expression: + // evaluate binary expressions. Format: Operand [Operator Operand]... value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:] - - // we don't need to track back, so we can drop the processed nodes while consuming them: for len(expr.Nodes) > 0 { var ( operator string @@ -122,8 +111,7 @@ func eval(expr *node) float64 { case "mul": value *= operand case "div": - // Go returns -Inf or +Inf on division by zero: - value /= operand + value /= operand // Go returns on division by zero +/-Inf } } } @@ -132,12 +120,13 @@ func eval(expr *node) float64 { } func print(output io.Writer, result float64) error { + // we can use the stdlib fmt package to print float64: _, err := fmt.Fprintln(output, result) return err } func main() { - // for testability, we define the REPL loop in a separate function so that the test code can call it with + // for testability, we define the REPL in a separate function so that the test code can call it with // in-memory buffers as input and output. Our main function calls it with the stdio handles: repl(os.Stdin, os.Stdout) } diff --git a/docs/manual.md b/docs/manual.md index 8b0b204..7eaca96 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -24,7 +24,7 @@ Alternatively, we _may be able to_ install directly using the Go toolchain: ## Hello syntax -A basic syntax definition looks like this: +A trivial syntax definition looks like this: ``` hello = "Hello, world!" @@ -83,10 +83,10 @@ If our syntax definition is invalid, check-syntax will fail: treerack check-syntax --syntax-string 'foo = bar' ``` -The above command will fail because the parser called foo references an undefined parser bar. +The above command will fail because the parser called `foo` references an undefined parser `bar`. -We can use check or show to detect when the input content does not match a valid syntax. Using the hello syntax, -we can try the following: +We can use `check` or `show` to detect when the input content does not match a valid syntax. Using the hello +syntax, we can try the following: ``` treerack check --syntax-string 'hello = "Hello, world!"' --input-string 'Hi!' @@ -96,9 +96,9 @@ It will show that parsing the input failed and that it failed while using the pa ## Basic syntax - An arithmetic calculator -In this section, we will build a basic arithmetic calculator. It will read a line from standard input, parse it -as an arithmetic expression, compute the result, and print it—effectively creating a REPL (Read-Eval-Print -Loop). +In this section, we will build a simplistic arithmetic calculator. It will read a line from standard input, +parse it as an arithmetic expression, compute the result, print it, and start over - effectively creating a REPL +(Read-Eval-Print Loop). We will support addition +, subtraction -, multiplication *, division /, and grouping with parentheses (). @@ -133,10 +133,10 @@ group:alias = "(" expression ")"; // // We group operators by precedence levels to ensure correct order of operations. // -// Level 0 (High): Multiplication/Division +// Level 0 (high): multiplication/division op0:alias = mul | div; -// Level 1 (Low): Addition/Subtraction +// Level 1 (low): addition/subtraction op1:alias = add | sub; // Operands for each precedence level. @@ -401,12 +401,12 @@ var errExit = errors.New("exit") // repl runs the Read-Eval-Print Loop. func repl(input io.Reader, output io.Writer) { - // use buffered io, to be able to read the input line-by-line: + // use buffered io, to read the input line-by-line: buf := bufio.NewReader(os.Stdin) - // our REPL loop: + // our REPL: for { - // print a basic prompt: + // print a input prompt marker: if _, err := output.Write([]byte("> ")); err != nil { log.Fatalln(err) } @@ -414,29 +414,30 @@ func repl(input io.Reader, output io.Writer) { // read the input and handle the errors: expr, err := read(buf) - // Handle EOF (Ctrl+D) + // handle EOF (Ctrl+D): if errors.Is(err, io.EOF) { output.Write([]byte{'\n'}) os.Exit(0) } - // Handle explicit exit command + // handle the explicit exit command: if errors.Is(err, errExit) { os.Exit(0) } - // Handle parser errors (allow user to retry) + // handle parser errors (allow the user to retry): var perr *parseError if errors.As(err, &perr) { log.Println(err) continue } + // handle possible I/O errors: if err != nil { log.Fatalln(err) } - // Evaluate and print + // evaluate and print: result := eval(expr) if err := print(output, result); err != nil { log.Fatalln(err) @@ -450,7 +451,7 @@ func read(input *bufio.Reader) (*node, error) { return nil, err } - // Parse the line using the generated parser + // parse the line using the generated parser: expr, err := parse(bytes.NewBufferString(line)) if err != nil { return nil, err @@ -460,8 +461,7 @@ func read(input *bufio.Reader) (*node, error) { return nil, errExit } - // Based on our syntax, the root node always has exactly one child: - // either a number or a binary operation. + // based on our syntax, the root node always has exactly one child: either a number or a binary operation. return expr.Nodes[0], nil } @@ -478,8 +478,7 @@ func eval(expr *node) float64 { return value default: - // Handle binary expressions (recursively) - // Format: Operand [Operator Operand]... + // handle binary expressions. Format: Operand [Operator Operand]... value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:] for len(expr.Nodes) > 0 { var ( @@ -496,7 +495,7 @@ func eval(expr *node) float64 { case "mul": value *= operand case "div": - value /= operand // Go handles division by zero as ±Inf + value /= operand // Go handles division by zero as +/-Inf } } } @@ -505,12 +504,13 @@ func eval(expr *node) float64 { } func print(output io.Writer, result float64) error { + // we can use _, err := fmt.Fprintln(output, result) return err } func main() { - // for testability, we define the REPL loop in a separate function so that the test code can call it with + // for testability, we define the REPL in a separate function so that the test code can call it with // in-memory buffers as input and output. Our main function calls it with the stdio handles: repl(os.Stdin, os.Stdout) } @@ -533,13 +533,13 @@ $ go run . We can find the source files for this example here: [./examples/acalc](./examples/acalc). -## Important Note: Unescaping +## Important note: unescaping -Treerack does not automatically handle escape sequences (e.g., converting \n to a literal newline). If our -syntax supports escaped characters—common in string literals—the user code is responsible for "unescaping" the -raw text from the AST node. +Treerack does not automatically handle escape sequences (e.g., converting `\n` to a literal newline). If our +syntax supports escaped characters - common in string literals - the user code is responsible for "unescaping" +the raw text from the AST node. -This is analogous to how we needed to parse the numbers in the calculator example to convert the string +This is analogous to how we needed to interpret the numbers in the calculator example to convert the string representation of a number into a Go float64. ## Programmatically loading syntaxes @@ -569,7 +569,7 @@ func initAndParse(syntax, content io.Reader) (*treerack.Node, error) { } ``` -Caution: Be mindful of security implications when loading syntax definitions from untrusted sources. +Caution: be mindful of security implications when loading syntax definitions from untrusted sources. ## Programmatically defining syntaxes @@ -611,7 +611,7 @@ func initAndParse(content io.Reader) (*treerack.Node, error) { ## Summary -We have demonstrated how to use the Treerack tool to define, test, and implement a parser. We recommend the +We have demonstrated how to use the treerack tool to define, test, and implement a parser. We recommend the following workflow: 1. draft: define a syntax in a .treerack file. diff --git a/docs/syntax.md b/docs/syntax.md index 3af0c06..ad255c2 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -5,14 +5,14 @@ It allows for the concise definition of recursive descent parsers. A syntax file consists of a series of Production Rules (definitions), terminated by semicolons. -## Production Rules +## Production rules A rule assigns a name to a pattern expression. Rules may include optional flags to modify the parser's behavior or the resulting AST (Abstract Syntax Tree). ``` -RuleName = Expression; -RuleName:flag1:flag2 = Expression; +rule-name = expression; +rule-name:flag1:flag2 = expression; ``` ## Flags @@ -20,18 +20,19 @@ RuleName:flag1:flag2 = Expression; Flags are appended to the rule name, separated by colons. They control AST generation, whitespace handling, and error propagation. -- `alias`: Transparent Node. The rule validates input but does not create its own node in the AST. Children +- `alias`: transparent node. The rule validates input but does not create its own node in the AST. Children nodes (if any) are attached to the parent of this rule. -- `ws`: Global Whitespace. Marks this rule as the designated whitespace handler. The parser will attempt to +- `ws`: global whitespace. Marks this rule as the designated whitespace handler. The parser will attempt to match (and discard) this rule between tokens throughout the entire syntax. -- `nows`: No Whitespace. Disables automatic whitespace skipping inside this rule. Useful for defining tokens - like string literals where spaces are significant. -- `root`: Entry Point. Explicitly marks the rule as the starting point of the syntax. If omitted, the last +- `nows`: no whitespace. Disables automatic whitespace skipping inside this rule. Useful for defining tokens + like string literals where spaces are significant. The flag `nows` is automatically applied to char sequences + like `"abc" or [abc]+. +- `root`: entry point. Explicitly marks the rule as the starting point of the syntax. If omitted, the last defined rule is implied to be the root. -- `kw`: Keyword. Marks the content as a reserved keyword. -- `nokw`: No Keyword. Prevents the rule from matching text that matches a defined kw rule. Essential for +- `kw`: keyword. Marks the content as a reserved keyword. +- `nokw`: no keyword. Prevents the rule from matching text that matches a defined kw rule. Essential for distinguishing identifiers from keywords (e.g., ensuring var is not parsed as a variable name). -- `failpass`: Pass Failure. If this rule fails to parse, the error is reported as a failure of the parent rule, +- `failpass`: pass failure. If this rule fails to parse, the error is reported as a failure of the parent rule, not this specific rule. ## Expressions @@ -43,7 +44,7 @@ and quantifiers. Terminals match specific characters or strings in the input. -- `"abc"` (string): Matches an exact sequence of characters. +- `"abc"` (string): Matches an exact sequence of characters. Equivalent to [a][b][c]. - `.` (any char): Matches any single character (wildcard). - `[123]`, `[a-z]`, `[123a-z]` (class): Matches a single character from a set or range. - `[^123]`, `[^a-z]`, `[^123a-z]` (not class) Matches any single character not in the set. @@ -52,13 +53,13 @@ Terminals match specific characters or strings in the input. Quantifiers determine how many times an item must match. They are placed immediately after the item they modify. -- `?`: Optional (Zero or one). -- `*`: Zero or more. -- `+`: One or more. -- `{n}`: Exact count. Matches exactly n times. -- `{n,}`: At least. Matches n or more times. -- `{,m}`: At most. Matches between 0 and m times. -- `{n,m}`: Range. Matches between n and m times. +- `?`: optional (zero or one). +- `*`: zero or more. +- `+`: one or more. +- `{n}`: exact count. Matches exactly n times. +- `{n,}`: at least. Matches n or more times. +- `{,m}`: at most. Matches between 0 and m times. +- `{n,m}`: range. Matches between n and m times. ## Composites @@ -69,8 +70,8 @@ Complex patterns are built by combining terminals and other rules. Items written consecutively are matched in order. ``` -// Matches "A", then "B", then "C" -MySequence = "A" "B" "C"; +// matches "A", then "B", then "C": +my-sequence = "A" "B" "C"; ``` ### 2. Grouping @@ -78,8 +79,8 @@ MySequence = "A" "B" "C"; Parentheses (...) group items together, allowing quantifiers to apply to the entire group. ``` -// Matches "AB", "ABAB", "ABABAB"... -MyGroup = ("A" "B")+; +// matches "AB", "ABAB", "ABABAB"...: +my-group = ("A" "B")+; ``` ### 3. Choices @@ -89,21 +90,18 @@ The pipe | character represents a choice between alternatives. The parser evaluates all provided options against the input at the current position and selects the best match based on the following priority rules: -1. _Longest Match_: The option that consumes the largest number of characters takes priority. This eliminates the +1. _longest match_: the option that consumes the largest number of characters takes priority. This eliminates the need to manually order specific matches before general ones (e.g., "integer" will always be chosen over "int" if the input supports it, regardless of their order in the definition). -2. _First Definition Wins_: If multiple options consume the exact same number of characters, the option defined +2. _first definition wins_: if multiple options consume the exact same number of characters, the option defined first(left-most) in the list takes priority. ``` -// Longest match wins automatically: -// Input "integer" is matched by 'type', even though "int" comes first. +// longest match wins automatically: input "integer" is matched by 'type', even though "int" comes first. type = "int" | "integer"; -// Tie-breaker rule: -// If input is "foo", both options match 3 characters. -// Because 'identifier' is last, it takes priority over 'keyword'. -// (Use :kw and :nokw to control such situations, when it applies.) +// Tie-breaker rule: if input is "foo", both options match 3 characters. Because 'identifier' is last, it takes +// priority over 'keyword'. (Use :kw and :nokw to control such situations, when it applies.) content = keyword | identifier; ``` @@ -111,8 +109,8 @@ content = keyword | identifier; Comments follow C-style syntax and are ignored by the definition parser. -- Line comments: Start with // and end at the newline. -- Block comments: Enclosed in /* ... */. +- line comments: start with // and end at the newline. +- block comments: enclosed in /* ... */. ## Examples diff --git a/head.go b/head.go index 694248d..7695d32 100644 --- a/head.go +++ b/head.go @@ -1,4 +1,4 @@ package treerack // generated with scripts/createhead.go -const headCode = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() commitType {\n\treturn alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]*node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount\tint\n\t\tparsed\t\tbool\n\t)\n\titemIndex := 0\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]*node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []*node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]*node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []*node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&whitespace == 0 && p.commitType()&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]*node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []*node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n}\n\nfunc newContext(r io.RuneReader, keywords []parser) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = errInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\treturn &parseError{Offset: c.failOffset, Line: line, Column: col, Definition: definition}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp)\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root)\n}\n\ntype node struct {\n\tName\tstring\n\tNodes\t[]*node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n *node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n *node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n *node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype commitType int\n\nconst (\n\tnone\tcommitType\t= 0\n\talias\tcommitType\t= 1 << iota\n\twhitespace\n\tnoWhitespace\n\tkeyword\n\tnoKeyword\n\tfailPass\n\troot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype parseError struct {\n\tInput\t\tstring\n\tOffset\t\tint\n\tLine\t\tint\n\tColumn\t\tint\n\tDefinition\tstring\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() commitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]*node, bool)\n}\n\nvar errInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *parseError) Error() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser) (*node, error) {\n\tc := newContext(bufio.NewReader(r), kw)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn nil, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*parseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn nil, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" +const headCode = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() commitType {\n\treturn alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]*node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount\tint\n\t\tparsed\t\tbool\n\t)\n\titemIndex := 0\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]*node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []*node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]*node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []*node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tcommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() commitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&noKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&whitespace == 0 && p.commit&failPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&whitespace == 0 && p.commitType()&failPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]*node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []*node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n}\n\nfunc newContext(r io.RuneReader, keywords []parser) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = errInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\tif offset < 0 {\n\t\treturn 0, 0\n\t}\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\treturn &parseError{Offset: c.failOffset, Line: line, Column: col, Definition: definition}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp)\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root)\n}\n\ntype node struct {\n\tName\tstring\n\tNodes\t[]*node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n *node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n *node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n *node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype commitType int\n\nconst (\n\tnone\tcommitType\t= 0\n\talias\tcommitType\t= 1 << iota\n\twhitespace\n\tnoWhitespace\n\tkeyword\n\tnoKeyword\n\tfailPass\n\troot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype parseError struct {\n\tInput\t\tstring\n\tOffset\t\tint\n\tLine\t\tint\n\tColumn\t\tint\n\tDefinition\tstring\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() commitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]*node, bool)\n}\n\nvar errInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *parseError) Error() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser) (*node, error) {\n\tc := newContext(bufio.NewReader(r), kw)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn nil, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*parseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn nil, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" diff --git a/headexported.go b/headexported.go index b4189c0..7e7e1c8 100644 --- a/headexported.go +++ b/headexported.go @@ -1,4 +1,4 @@ package treerack // generated with scripts/createhead.go -const headCodeExported = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() CommitType {\n\treturn Alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]*Node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount\tint\n\t\tparsed\t\tbool\n\t)\n\titemIndex := 0\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&Alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []*Node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]*Node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&Whitespace == 0 && p.commitType()&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n}\n\nfunc newContext(r io.RuneReader, keywords []parser) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = ErrInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\treturn &ParseError{Offset: c.failOffset, Line: line, Column: col, Definition: definition}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp)\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root)\n}\n\ntype Node struct {\n\tName\tstring\n\tNodes\t[]*Node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n *Node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n *Node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n *Node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype CommitType int\n\nconst (\n\tNone\tCommitType\t= 0\n\tAlias\tCommitType\t= 1 << iota\n\tWhitespace\n\tNoWhitespace\n\tKeyword\n\tNoKeyword\n\tFailPass\n\tRoot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype ParseError struct {\n\tInput\t\tstring\n\tOffset\t\tint\n\tLine\t\tint\n\tColumn\t\tint\n\tDefinition\tstring\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() CommitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]*Node, bool)\n}\n\nvar ErrInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *ParseError) Error() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser) (*Node, error) {\n\tc := newContext(bufio.NewReader(r), kw)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn nil, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*ParseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn nil, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" +const headCodeExported = "import (\n\t\"strconv\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode\"\n\t\"fmt\"\n\t\"bufio\"\n)\n\ntype charParser struct {\n\tname\tstring\n\tid\tint\n\tnot\tbool\n\tchars\t[]rune\n\tranges\t[][]rune\n}\ntype charBuilder struct {\n\tname\tstring\n\tid\tint\n}\n\nfunc (p *charParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *charParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *charParser) commitType() CommitType {\n\treturn Alias\n}\nfunc matchChar(chars []rune, ranges [][]rune, not bool, char rune) bool {\n\tfor _, ci := range chars {\n\t\tif ci == char {\n\t\t\treturn !not\n\t\t}\n\t}\n\tfor _, ri := range ranges {\n\t\tif char >= ri[0] && char <= ri[1] {\n\t\t\treturn !not\n\t\t}\n\t}\n\treturn not\n}\nfunc (p *charParser) match(t rune) bool {\n\treturn matchChar(p.chars, p.ranges, p.not, t)\n}\nfunc (p *charParser) parse(c *context) {\n\tif tok, ok := c.token(); !ok || !p.match(tok) {\n\t\tif c.offset > c.failOffset {\n\t\t\tc.failOffset = c.offset\n\t\t\tc.failingParser = nil\n\t\t}\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.success(c.offset + 1)\n}\nfunc (b *charBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *charBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *charBuilder) build(c *context) ([]*Node, bool) {\n\treturn nil, false\n}\n\ntype sequenceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]parser\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\ntype sequenceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\titems\t\t[]builder\n\tranges\t\t[][]int\n\tgeneralizations\t[]int\n\tallChars\tbool\n}\n\nfunc (p *sequenceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *sequenceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *sequenceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *sequenceParser) parse(c *context) {\n\tif !p.allChars {\n\t\tif c.results.pending(c.offset, p.id) {\n\t\t\tc.fail(c.offset)\n\t\t\treturn\n\t\t}\n\t\tc.results.markPending(c.offset, p.id)\n\t}\n\tvar (\n\t\tcurrentCount\tint\n\t\tparsed\t\tbool\n\t)\n\titemIndex := 0\n\tfrom := c.offset\n\tto := c.offset\n\tfor itemIndex < len(p.items) {\n\t\tp.items[itemIndex].parse(c)\n\t\tif !c.matchLast {\n\t\t\tif currentCount >= p.ranges[itemIndex][0] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tc.offset = from\n\t\t\tif c.fromResults(p) {\n\t\t\t\tif to > c.failOffset {\n\t\t\t\t\tc.failOffset = -1\n\t\t\t\t\tc.failingParser = nil\n\t\t\t\t}\n\t\t\t\tif !p.allChars {\n\t\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tif !p.allChars {\n\t\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\tparsed = c.offset > to\n\t\tif parsed {\n\t\t\tcurrentCount++\n\t\t}\n\t\tto = c.offset\n\t\tif !parsed || p.ranges[itemIndex][1] > 0 && currentCount == p.ranges[itemIndex][1] {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t}\n\t}\n\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t\tc.fail(from)\n\t\tif !p.allChars {\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t}\n\t\treturn\n\t}\n\tfor _, g := range p.generalizations {\n\t\tif c.results.pending(from, g) {\n\t\t\tc.results.setMatch(from, g, to)\n\t\t}\n\t}\n\tif to > c.failOffset {\n\t\tc.failOffset = -1\n\t\tc.failingParser = nil\n\t}\n\tc.results.setMatch(from, p.id, to)\n\tc.success(to)\n\tif !p.allChars {\n\t\tc.results.unmarkPending(from, p.id)\n\t}\n}\nfunc (b *sequenceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *sequenceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *sequenceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif b.allChars {\n\t\tc.offset = to\n\t\tif b.commit&Alias != 0 {\n\t\t\treturn nil, true\n\t\t}\n\t\treturn []*Node{{Name: b.name, From: from, To: to, tokens: c.tokens}}, true\n\t} else if parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar (\n\t\titemIndex\tint\n\t\tcurrentCount\tint\n\t\tnodes\t\t[]*Node\n\t)\n\tfor itemIndex < len(b.items) {\n\t\titemFrom := c.offset\n\t\tn, ok := b.items[itemIndex].build(c)\n\t\tif !ok {\n\t\t\titemIndex++\n\t\t\tcurrentCount = 0\n\t\t\tcontinue\n\t\t}\n\t\tif c.offset > itemFrom {\n\t\t\tnodes = append(nodes, n...)\n\t\t\tcurrentCount++\n\t\t\tif b.ranges[itemIndex][1] > 0 && currentCount == b.ranges[itemIndex][1] {\n\t\t\t\titemIndex++\n\t\t\t\tcurrentCount = 0\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif currentCount < b.ranges[itemIndex][0] {\n\t\t\tfor i := 0; i < b.ranges[itemIndex][0]-currentCount; i++ {\n\t\t\t\tnodes = append(nodes, n...)\n\t\t\t}\n\t\t}\n\t\titemIndex++\n\t\tcurrentCount = 0\n\t}\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn nodes, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: nodes, tokens: c.tokens}}, true\n}\n\ntype choiceParser struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]parser\n\tgeneralizations\t[]int\n}\ntype choiceBuilder struct {\n\tname\t\tstring\n\tid\t\tint\n\tcommit\t\tCommitType\n\toptions\t\t[]builder\n\tgeneralizations\t[]int\n}\n\nfunc (p *choiceParser) nodeName() string {\n\treturn p.name\n}\nfunc (p *choiceParser) nodeID() int {\n\treturn p.id\n}\nfunc (p *choiceParser) commitType() CommitType {\n\treturn p.commit\n}\nfunc (p *choiceParser) parse(c *context) {\n\tif c.fromResults(p) {\n\t\treturn\n\t}\n\tif c.results.pending(c.offset, p.id) {\n\t\tc.fail(c.offset)\n\t\treturn\n\t}\n\tc.results.markPending(c.offset, p.id)\n\tvar (\n\t\tmatch\t\tbool\n\t\toptionIndex\tint\n\t\tfoundMatch\tbool\n\t\tfailingParser\tparser\n\t)\n\tfrom := c.offset\n\tto := c.offset\n\tinitialFailOffset := c.failOffset\n\tinitialFailingParser := c.failingParser\n\tfailOffset := initialFailOffset\n\tfor {\n\t\tfoundMatch = false\n\t\toptionIndex = 0\n\t\tfor optionIndex < len(p.options) {\n\t\t\tp.options[optionIndex].parse(c)\n\t\t\toptionIndex++\n\t\t\tif !c.matchLast {\n\t\t\t\tif c.failOffset > failOffset {\n\t\t\t\t\tfailOffset = c.failOffset\n\t\t\t\t\tfailingParser = c.failingParser\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !c.matchLast || match && c.offset <= to {\n\t\t\t\tc.offset = from\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tmatch = true\n\t\t\tfoundMatch = true\n\t\t\tto = c.offset\n\t\t\tc.offset = from\n\t\t\tc.results.setMatch(from, p.id, to)\n\t\t}\n\t\tif !foundMatch {\n\t\t\tbreak\n\t\t}\n\t}\n\tif match {\n\t\tif p.commit&NoKeyword != 0 && c.isKeyword(from, to) {\n\t\t\tif c.failingParser == nil && p.commit&userDefined != 0 && p.commit&Whitespace == 0 && p.commit&FailPass == 0 {\n\t\t\t\tc.failingParser = p\n\t\t\t}\n\t\t\tc.fail(from)\n\t\t\tc.results.unmarkPending(from, p.id)\n\t\t\treturn\n\t\t}\n\t\tif failOffset > to {\n\t\t\tc.failOffset = failOffset\n\t\t\tc.failingParser = failingParser\n\t\t} else if to > initialFailOffset {\n\t\t\tc.failOffset = -1\n\t\t\tc.failingParser = nil\n\t\t} else {\n\t\t\tc.failOffset = initialFailOffset\n\t\t\tc.failingParser = initialFailingParser\n\t\t}\n\t\tc.success(to)\n\t\tc.results.unmarkPending(from, p.id)\n\t\treturn\n\t}\n\tif failOffset > initialFailOffset {\n\t\tc.failOffset = failOffset\n\t\tc.failingParser = failingParser\n\t\tif c.failingParser == nil && p.commitType()&userDefined != 0 && p.commitType()&Whitespace == 0 && p.commitType()&FailPass == 0 {\n\t\t\tc.failingParser = p\n\t\t}\n\t}\n\tc.results.setNoMatch(from, p.id)\n\tc.fail(from)\n\tc.results.unmarkPending(from, p.id)\n}\nfunc (b *choiceBuilder) nodeName() string {\n\treturn b.name\n}\nfunc (b *choiceBuilder) nodeID() int {\n\treturn b.id\n}\nfunc (b *choiceBuilder) build(c *context) ([]*Node, bool) {\n\tto, ok := c.results.longestMatch(c.offset, b.id)\n\tif !ok {\n\t\treturn nil, false\n\t}\n\tfrom := c.offset\n\tparsed := to > from\n\tif parsed {\n\t\tc.results.dropMatchTo(c.offset, b.id, to)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.dropMatchTo(c.offset, g, to)\n\t\t}\n\t} else {\n\t\tif c.results.pending(c.offset, b.id) {\n\t\t\treturn nil, false\n\t\t}\n\t\tc.results.markPending(c.offset, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.markPending(c.offset, g)\n\t\t}\n\t}\n\tvar option builder\n\tfor _, o := range b.options {\n\t\tif c.results.hasMatchTo(c.offset, o.nodeID(), to) {\n\t\t\toption = o\n\t\t\tbreak\n\t\t}\n\t}\n\tn, _ := option.build(c)\n\tif !parsed {\n\t\tc.results.unmarkPending(from, b.id)\n\t\tfor _, g := range b.generalizations {\n\t\t\tc.results.unmarkPending(from, g)\n\t\t}\n\t}\n\tif b.commit&Alias != 0 {\n\t\treturn n, true\n\t}\n\treturn []*Node{{Name: b.name, From: from, To: to, Nodes: n, tokens: c.tokens}}, true\n}\n\ntype idSet struct{ ids []uint }\n\nfunc divModBits(id int) (int, int) {\n\treturn id / strconv.IntSize, id % strconv.IntSize\n}\nfunc (s *idSet) set(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\tif d < cap(s.ids) {\n\t\t\ts.ids = s.ids[:d+1]\n\t\t} else {\n\t\t\ts.ids = s.ids[:cap(s.ids)]\n\t\t\tfor i := cap(s.ids); i <= d; i++ {\n\t\t\t\ts.ids = append(s.ids, 0)\n\t\t\t}\n\t\t}\n\t}\n\ts.ids[d] |= 1 << uint(m)\n}\nfunc (s *idSet) unset(id int) {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn\n\t}\n\ts.ids[d] &^= 1 << uint(m)\n}\nfunc (s *idSet) has(id int) bool {\n\td, m := divModBits(id)\n\tif d >= len(s.ids) {\n\t\treturn false\n\t}\n\treturn s.ids[d]&(1< offset {\n\t\treturn ints\n\t}\n\tif cap(ints) > offset {\n\t\tints = ints[:offset+1]\n\t\treturn ints\n\t}\n\tints = ints[:cap(ints)]\n\tfor i := len(ints); i <= offset; i++ {\n\t\tints = append(ints, nil)\n\t}\n\treturn ints\n}\nfunc ensureOffsetIDs(ids []*idSet, offset int) []*idSet {\n\tif len(ids) > offset {\n\t\treturn ids\n\t}\n\tif cap(ids) > offset {\n\t\tids = ids[:offset+1]\n\t\treturn ids\n\t}\n\tids = ids[:cap(ids)]\n\tfor i := len(ids); i <= offset; i++ {\n\t\tids = append(ids, nil)\n\t}\n\treturn ids\n}\nfunc (r *results) setMatch(offset, id, to int) {\n\tr.match = ensureOffsetInts(r.match, offset)\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id || r.match[offset][i+1] != to {\n\t\t\tcontinue\n\t\t}\n\t\treturn\n\t}\n\tr.match[offset] = append(r.match[offset], id, to)\n}\nfunc (r *results) setNoMatch(offset, id int) {\n\tif len(r.match) > offset {\n\t\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\t\tif r.match[offset][i] != id {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t}\n\tr.noMatch = ensureOffsetIDs(r.noMatch, offset)\n\tif r.noMatch[offset] == nil {\n\t\tr.noMatch[offset] = &idSet{}\n\t}\n\tr.noMatch[offset].set(id)\n}\nfunc (r *results) hasMatchTo(offset, id, to int) bool {\n\tif len(r.match) <= offset {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) longestMatch(offset, id int) (int, bool) {\n\tif len(r.match) <= offset {\n\t\treturn 0, false\n\t}\n\tvar found bool\n\tto := -1\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] > to {\n\t\t\tto = r.match[offset][i+1]\n\t\t}\n\t\tfound = true\n\t}\n\treturn to, found\n}\nfunc (r *results) longestResult(offset, id int) (int, bool, bool) {\n\tif len(r.noMatch) > offset && r.noMatch[offset] != nil && r.noMatch[offset].has(id) {\n\t\treturn 0, false, true\n\t}\n\tto, ok := r.longestMatch(offset, id)\n\treturn to, ok, ok\n}\nfunc (r *results) dropMatchTo(offset, id, to int) {\n\tfor i := 0; i < len(r.match[offset]); i += 2 {\n\t\tif r.match[offset][i] != id {\n\t\t\tcontinue\n\t\t}\n\t\tif r.match[offset][i+1] == to {\n\t\t\tr.match[offset][i] = -1\n\t\t\treturn\n\t\t}\n\t}\n}\nfunc (r *results) resetPending() {\n\tr.isPending = nil\n}\nfunc (r *results) pending(offset, id int) bool {\n\tif len(r.isPending) <= id {\n\t\treturn false\n\t}\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (r *results) markPending(offset, id int) {\n\tr.isPending = ensureOffsetInts(r.isPending, id)\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == -1 {\n\t\t\tr.isPending[id][i] = offset\n\t\t\treturn\n\t\t}\n\t}\n\tr.isPending[id] = append(r.isPending[id], offset)\n}\nfunc (r *results) unmarkPending(offset, id int) {\n\tfor i := range r.isPending[id] {\n\t\tif r.isPending[id][i] == offset {\n\t\t\tr.isPending[id][i] = -1\n\t\t\tbreak\n\t\t}\n\t}\n}\n\ntype context struct {\n\treader\t\tio.RuneReader\n\tkeywords\t[]parser\n\toffset\t\tint\n\treadOffset\tint\n\tconsumed\tint\n\toffsetLimit\tint\n\tfailOffset\tint\n\tfailingParser\tparser\n\treadErr\t\terror\n\teof\t\tbool\n\tresults\t\t*results\n\ttokens\t\t[]rune\n\tmatchLast\tbool\n}\n\nfunc newContext(r io.RuneReader, keywords []parser) *context {\n\treturn &context{reader: r, keywords: keywords, results: &results{}, offsetLimit: -1, failOffset: -1}\n}\nfunc (c *context) read() bool {\n\tif c.eof || c.readErr != nil {\n\t\treturn false\n\t}\n\ttoken, n, err := c.reader.ReadRune()\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\tif n == 0 {\n\t\t\t\tc.eof = true\n\t\t\t\treturn false\n\t\t\t}\n\t\t} else {\n\t\t\tc.readErr = err\n\t\t\treturn false\n\t\t}\n\t}\n\tc.readOffset++\n\tif token == unicode.ReplacementChar {\n\t\tc.readErr = ErrInvalidUnicodeCharacter\n\t\treturn false\n\t}\n\tc.tokens = append(c.tokens, token)\n\treturn true\n}\nfunc (c *context) token() (rune, bool) {\n\tif c.offset == c.offsetLimit {\n\t\treturn 0, false\n\t}\n\tif c.offset == c.readOffset {\n\t\tif !c.read() {\n\t\t\treturn 0, false\n\t\t}\n\t}\n\treturn c.tokens[c.offset], true\n}\nfunc (c *context) fromResults(p parser) bool {\n\tto, m, ok := c.results.longestResult(c.offset, p.nodeID())\n\tif !ok {\n\t\treturn false\n\t}\n\tif m {\n\t\tc.success(to)\n\t} else {\n\t\tc.fail(c.offset)\n\t}\n\treturn true\n}\nfunc (c *context) isKeyword(from, to int) bool {\n\tol := c.offsetLimit\n\tc.offsetLimit = to\n\tdefer func() {\n\t\tc.offsetLimit = ol\n\t}()\n\tfor _, kw := range c.keywords {\n\t\tc.offset = from\n\t\tkw.parse(c)\n\t\tif c.matchLast && c.offset == to {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\nfunc (c *context) success(to int) {\n\tc.offset = to\n\tc.matchLast = true\n\tif to > c.consumed {\n\t\tc.consumed = to\n\t}\n}\nfunc (c *context) fail(offset int) {\n\tc.offset = offset\n\tc.matchLast = false\n}\nfunc findLine(tokens []rune, offset int) (line, column int) {\n\tif offset < 0 {\n\t\treturn 0, 0\n\t}\n\ttokens = tokens[:offset]\n\tfor i := range tokens {\n\t\tcolumn++\n\t\tif tokens[i] == '\\n' {\n\t\t\tcolumn = 0\n\t\t\tline++\n\t\t}\n\t}\n\treturn\n}\nfunc (c *context) parseError(p parser) error {\n\tdefinition := p.nodeName()\n\tflagIndex := strings.Index(definition, \":\")\n\tif flagIndex > 0 {\n\t\tdefinition = definition[:flagIndex]\n\t}\n\tif c.failingParser == nil {\n\t\tc.failOffset = c.consumed\n\t}\n\tline, col := findLine(c.tokens, c.failOffset)\n\treturn &ParseError{Offset: c.failOffset, Line: line, Column: col, Definition: definition}\n}\nfunc (c *context) finalizeParse(root parser) error {\n\tfp := c.failingParser\n\tif fp == nil {\n\t\tfp = root\n\t}\n\tto, match, found := c.results.longestResult(0, root.nodeID())\n\tif !found || !match || found && match && to < c.readOffset {\n\t\treturn c.parseError(fp)\n\t}\n\tc.read()\n\tif c.eof {\n\t\treturn nil\n\t}\n\tif c.readErr != nil {\n\t\treturn c.readErr\n\t}\n\treturn c.parseError(root)\n}\n\ntype Node struct {\n\tName\tstring\n\tNodes\t[]*Node\n\tFrom\tint\n\tTo\tint\n\ttokens\t[]rune\n}\n\nfunc (n *Node) Tokens() []rune {\n\treturn n.tokens\n}\nfunc (n *Node) String() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:%s\", n.Name, n.From, n.To, n.Text())\n}\nfunc (n *Node) Text() string {\n\treturn string(n.Tokens()[n.From:n.To])\n}\n\ntype CommitType int\n\nconst (\n\tNone\tCommitType\t= 0\n\tAlias\tCommitType\t= 1 << iota\n\tWhitespace\n\tNoWhitespace\n\tKeyword\n\tNoKeyword\n\tFailPass\n\tRoot\n\tuserDefined\n)\n\ntype formatFlags int\n\nconst (\n\tformatNone\tformatFlags\t= 0\n\tformatPretty\tformatFlags\t= 1 << iota\n\tformatIncludeComments\n)\n\ntype ParseError struct {\n\tInput\t\tstring\n\tOffset\t\tint\n\tLine\t\tint\n\tColumn\t\tint\n\tDefinition\tstring\n}\ntype parser interface {\n\tnodeName() string\n\tnodeID() int\n\tcommitType() CommitType\n\tparse(*context)\n}\ntype builder interface {\n\tnodeName() string\n\tnodeID() int\n\tbuild(*context) ([]*Node, bool)\n}\n\nvar ErrInvalidUnicodeCharacter = errors.New(\"invalid unicode character\")\n\nfunc (pe *ParseError) Error() string {\n\treturn fmt.Sprintf(\"%s:%d:%d:parse failed, parsing: %s\", pe.Input, pe.Line+1, pe.Column+1, pe.Definition)\n}\nfunc parseInput(r io.Reader, p parser, b builder, kw []parser) (*Node, error) {\n\tc := newContext(bufio.NewReader(r), kw)\n\tp.parse(c)\n\tif c.readErr != nil {\n\t\treturn nil, c.readErr\n\t}\n\tif err := c.finalizeParse(p); err != nil {\n\t\tif perr, ok := err.(*ParseError); ok {\n\t\t\tperr.Input = \"\"\n\t\t}\n\t\treturn nil, err\n\t}\n\tc.offset = 0\n\tc.results.resetPending()\n\tn, _ := b.build(c)\n\treturn n[0], nil\n}\n" diff --git a/internal/self/self.go b/internal/self/self.go index e031352..ddda63d 100644 --- a/internal/self/self.go +++ b/internal/self/self.go @@ -687,6 +687,9 @@ func (c *context) fail(offset int) { c.matchLast = false } func findLine(tokens []rune, offset int) (line, column int) { + if offset < 0 { + return 0, 0 + } tokens = tokens[:offset] for i := range tokens { column++ diff --git a/notes.txt b/notes.txt index c25c548..ea19f96 100644 --- a/notes.txt +++ b/notes.txt @@ -1,40 +1,14 @@ [next] errors -generator 1 -documentation -parser 1 -releasing -parser 2 -generator 2 formatter report unused parsers -parse hashed, storing only the results -linux packaging [errors] -take the last +don't report aliases +take the last meaningful sequence test error report on invalid flag -input name: may be just dropped because completely controlled by the client input name needed in command to differentiate between syntax and input in check and parse subcommands -[generator 1] -allchars: can have char sequence -make generator output non-random (track parsers in a list in definition order) -fix the license in the output - -[generator 2] -js - -[releasing] -spellcheck -linting -convert notes into issues -try to remove some files - -[parser 1] -try winning on allChars -retry collapsing - [parser 2] custom tokens indentation @@ -42,10 +16,3 @@ streaming support // ReadNode(io.Reader) [optimization] try preallocate larger store chunks - -[documentation] -how the char classes are different from regexp -why need nows when using ws -lib only useful for dynamic syntax definition -warn nows usage in docs, e.g. spaces in symbol = [a-z]+ -tutorial diff --git a/parse_test.go b/parse_test.go index d05e531..81b3080 100644 --- a/parse_test.go +++ b/parse_test.go @@ -162,6 +162,39 @@ func TestRecursion(t *testing.T) { }, }}, ) + + runTests( + t, + "A = A [a]", + []testItem{{ + title: "naive left recursion expects infinite stream", + text: "a", + ignorePosition: true, + fail: true, + }}, + ) + + runTests( + t, + "A = A [a]; B = A", + []testItem{{ + title: "embedded naive left recursion expects infinite stream", + text: "a", + ignorePosition: true, + fail: true, + }}, + ) + + runTests( + t, + "ws:ws = [ \t\n]; A = A [a]; B = A", + []testItem{{ + title: "embedded naive left recursion expects infinite stream with whitespace", + text: "\na", + ignorePosition: true, + fail: true, + }}, + ) } func TestSequence(t *testing.T) { diff --git a/readme.md b/readme.md index 4f5fe25..a8c11b3 100644 --- a/readme.md +++ b/readme.md @@ -20,9 +20,13 @@ syntax language supports recursive references, enabling the definition of contex We can define syntaxes during development and use the provided tool to generate static Go code, which is then built into the application. Alternatively, the library supports loading syntaxes dynamically at runtime. +The parser engine handles recursive references and left-recursion internally. This way it makes it more +convenient writing intuitive grammar definitions, and allows defining context-free languages without complex +workarounds. + ## Installation -From source: +From source (recommended): ``` git clone https://code.squareroundforest.org/arpio/treerack @@ -30,7 +34,7 @@ cd treerack make install ``` -Alternatively: +Alternatively ("best effort" basis): ``` go install code.squareroundforest.org/arpio/treerack/cmd/treerack @@ -38,8 +42,8 @@ go install code.squareroundforest.org/arpio/treerack/cmd/treerack ## Documentation -- [Manual](docs/manual.md): A guide to the main use cases supported by Treerack. -- [Syntax Definition](docs/syntax.md): Detailed reference for the Treerack definition language. +- [Manual](docs/manual.md): a guide to the main use cases supported by Treerack. +- [Syntax Definition](docs/syntax.md): detailed reference for the Treerack definition language. - [Library Documentation](https://godocs.io/code.squareroundforest.org/arpio/treerack): GoDoc reference for the runtime library. @@ -47,10 +51,10 @@ go install code.squareroundforest.org/arpio/treerack/cmd/treerack We use a Makefile to manage the build and verification lifecycle. -Important: Generating the parser for the Treerack syntax itself (bootstrapping) requires multiple phases. +Important: generating the parser for the Treerack syntax itself (bootstrapping) requires multiple phases. Consequently, running standard go build or go test commands may miss subtle consistency problems. -The authoritative way to verify changes is via the makefile: +The decisive way to verify changes is via the makefile: ``` make check @@ -60,6 +64,6 @@ make check - Lexer & UTF-8: Treerack does not require a lexer, which simplifies the architecture. However, this enforces the use of UTF-8 input. We have considered support for custom tokenizers as a potential future improvement. -- Whitespace Delimited Languages: Due to the recursive descent nature and the lack of a dedicated lexer state, +- Whitespace Delimited Languages: due to the recursive descent nature and the lack of a dedicated lexer state, defining whitespace-delimited syntaxes (such as Python-style indentation) can be difficult to achieve with the current feature set.