1
0

error fixes and doc updates

This commit is contained in:
Arpad Ryszka 2026-01-21 20:54:16 +01:00
parent 4c6c817431
commit 2fe6f88ed6
21 changed files with 285 additions and 269 deletions

View File

@ -76,6 +76,7 @@ lib: $(sources) head.go headexported.go internal/self/self.go
cmd/treerack/docreflect.gen.go: $(sources) .build cmd/treerack/docreflect.gen.go: $(sources) .build
go run scripts/docreflect.go > .build/docreflect.gen.go go run scripts/docreflect.go > .build/docreflect.gen.go
go fmt .build/docreflect.gen.go
mv .build/docreflect.gen.go cmd/treerack mv .build/docreflect.gen.go cmd/treerack
cmd/treerack/readme.md: $(sources) cmd/treerack/docreflect.gen.go cmd/treerack/readme.md: $(sources) cmd/treerack/docreflect.gen.go

View File

@ -8,16 +8,16 @@ import (
type checkOptions struct { type checkOptions struct {
// Syntax specifies the filename of the syntax definition file. // Syntax specifies the filename of the syntax definition file.
Syntax string Syntax *string
// SyntaxString specifies the syntax as an inline string. // SyntaxString specifies the syntax as an inline string.
SyntaxString string SyntaxString *string
// Input specifies the filename of the input content to be validated. // Input specifies the filename of the input content to be validated.
Input string Input *string
// InputString specifies the input content as an inline string. // InputString specifies the input content as an inline string.
InputString string InputString *string
} }
// check parses input content against the provided syntax definition and fails if the input does not match. // check parses input content against the provided syntax definition and fails if the input does not match.

View File

@ -8,9 +8,13 @@ import (
"testing" "testing"
) )
func ptrto[T any](v T) *T {
return &v
}
func TestCheck(t *testing.T) { func TestCheck(t *testing.T) {
t.Run("no syntax", func(t *testing.T) { t.Run("no syntax", func(t *testing.T) {
o := checkOptions{Input: "bar_test.txt"} o := checkOptions{Input: ptrto("bar_test.txt")}
if err := check(o, nil); !errors.Is(err, errNoInput) { if err := check(o, nil); !errors.Is(err, errNoInput) {
t.Fatal() t.Fatal()
} }
@ -18,9 +22,9 @@ func TestCheck(t *testing.T) {
t.Run("too many syntaxes", func(t *testing.T) { t.Run("too many syntaxes", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
SyntaxString: `foo = "baz"`, SyntaxString: ptrto(`foo = "baz"`),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := check(o, nil); !errors.Is(err, errMultipleInputs) { if err := check(o, nil); !errors.Is(err, errMultipleInputs) {
@ -30,8 +34,8 @@ func TestCheck(t *testing.T) {
t.Run("syntax file not found", func(t *testing.T) { t.Run("syntax file not found", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
Syntax: "no-file.treerack", Syntax: ptrto("no-file.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := check(o, nil); !os.IsNotExist(err) { if err := check(o, nil); !os.IsNotExist(err) {
@ -41,8 +45,8 @@ func TestCheck(t *testing.T) {
t.Run("invalid syntax definition", func(t *testing.T) { t.Run("invalid syntax definition", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
SyntaxString: `foo`, SyntaxString: ptrto(`foo`),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
var perr *treerack.ParseError var perr *treerack.ParseError
@ -53,8 +57,8 @@ func TestCheck(t *testing.T) {
t.Run("invalid syntax init", func(t *testing.T) { t.Run("invalid syntax init", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
SyntaxString: `foo = "bar"; foo = "baz"`, SyntaxString: ptrto(`foo = "bar"; foo = "baz"`),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := check(o, nil); err == nil { if err := check(o, nil); err == nil {
@ -63,7 +67,7 @@ func TestCheck(t *testing.T) {
}) })
t.Run("no input", func(t *testing.T) { t.Run("no input", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"} o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil); !errors.Is(err, errNoInput) { if err := check(o, nil); !errors.Is(err, errNoInput) {
t.Fatal() t.Fatal()
@ -72,8 +76,8 @@ func TestCheck(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) { t.Run("too many inputs", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := check(o, nil, "baz_test.txt"); !errors.Is(err, errMultipleInputs) { if err := check(o, nil, "baz_test.txt"); !errors.Is(err, errMultipleInputs) {
@ -82,14 +86,14 @@ func TestCheck(t *testing.T) {
}) })
t.Run("empty filename for input", func(t *testing.T) { t.Run("empty filename for input", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"} o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil, ""); !errors.Is(err, errInvalidFilename) { if err := check(o, nil, ""); !errors.Is(err, errInvalidFilename) {
t.Fatal() t.Fatal()
} }
}) })
t.Run("input file not found", func(t *testing.T) { t.Run("input file not found", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"} o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil, "baz_test.txt"); !os.IsNotExist(err) { if err := check(o, nil, "baz_test.txt"); !os.IsNotExist(err) {
t.Fatal() t.Fatal()
} }
@ -97,8 +101,8 @@ func TestCheck(t *testing.T) {
t.Run("input parse fail", func(t *testing.T) { t.Run("input parse fail", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
InputString: "baz", InputString: ptrto("baz"),
} }
var perr *treerack.ParseError var perr *treerack.ParseError
@ -109,8 +113,8 @@ func TestCheck(t *testing.T) {
t.Run("input parse success", func(t *testing.T) { t.Run("input parse success", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := check(o, nil); err != nil { if err := check(o, nil); err != nil {
@ -120,8 +124,8 @@ func TestCheck(t *testing.T) {
t.Run("input from string success", func(t *testing.T) { t.Run("input from string success", func(t *testing.T) {
o := checkOptions{ o := checkOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
InputString: "bar", InputString: ptrto("bar"),
} }
if err := check(o, nil); err != nil { if err := check(o, nil); err != nil {
@ -130,17 +134,29 @@ func TestCheck(t *testing.T) {
}) })
t.Run("input from file success", func(t *testing.T) { t.Run("input from file success", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"} o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil, "bar_test.txt"); err != nil { if err := check(o, nil, "bar_test.txt"); err != nil {
t.Fatal(err) t.Fatal(err)
} }
}) })
t.Run("input from stdin success", func(t *testing.T) { t.Run("input from stdin success", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"} o := checkOptions{Syntax: ptrto("foo_test.treerack")}
buf := bytes.NewBufferString("bar") buf := bytes.NewBufferString("bar")
if err := check(o, buf); err != nil { if err := check(o, buf); err != nil {
t.Fatal(err) t.Fatal(err)
} }
}) })
t.Run("empty input filename", func(t *testing.T) {
o := checkOptions{
Syntax: ptrto("foo_test.treerack"),
Input: ptrto(""),
}
var buf bytes.Buffer
if err := check(o, &buf); !errors.Is(err, errInvalidFilename) {
t.Fatal()
}
})
} }

View File

@ -8,10 +8,10 @@ import (
type checkSyntaxOptions struct { type checkSyntaxOptions struct {
// Syntax specifies the filename of the syntax definition file. // Syntax specifies the filename of the syntax definition file.
Syntax string Syntax *string
// SyntaxString specifies the syntax as an inline string. // SyntaxString specifies the syntax as an inline string.
SyntaxString string SyntaxString *string
} }
// checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a // checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a

View File

@ -10,7 +10,7 @@ import (
func TestCheckSyntax(t *testing.T) { func TestCheckSyntax(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) { t.Run("too many inputs", func(t *testing.T) {
o := checkSyntaxOptions{Syntax: "foo_test.treerack", SyntaxString: `foo = "42"`} o := checkSyntaxOptions{Syntax: ptrto("foo_test.treerack"), SyntaxString: ptrto(`foo = "42"`)}
if err := checkSyntax(o, nil); !errors.Is(err, errMultipleInputs) { if err := checkSyntax(o, nil); !errors.Is(err, errMultipleInputs) {
t.Fatal() t.Fatal()
} }
@ -32,28 +32,28 @@ func TestCheckSyntax(t *testing.T) {
t.Run("invalid syntax", func(t *testing.T) { t.Run("invalid syntax", func(t *testing.T) {
var perr *treerack.ParseError var perr *treerack.ParseError
o := checkSyntaxOptions{SyntaxString: "foo"} o := checkSyntaxOptions{SyntaxString: ptrto("foo")}
if err := checkSyntax(o, nil); !errors.As(err, &perr) { if err := checkSyntax(o, nil); !errors.As(err, &perr) {
t.Fatal() t.Fatal()
} }
}) })
t.Run("invalid syntax init", func(t *testing.T) { t.Run("invalid syntax init", func(t *testing.T) {
o := checkSyntaxOptions{SyntaxString: `foo = "42"; foo = "84"`} o := checkSyntaxOptions{SyntaxString: ptrto(`foo = "42"; foo = "84"`)}
if err := checkSyntax(o, nil); err == nil { if err := checkSyntax(o, nil); err == nil {
t.Fatal() t.Fatal()
} }
}) })
t.Run("success", func(t *testing.T) { t.Run("success", func(t *testing.T) {
o := checkSyntaxOptions{Syntax: "foo_test.treerack"} o := checkSyntaxOptions{Syntax: ptrto("foo_test.treerack")}
if err := checkSyntax(o, nil); err != nil { if err := checkSyntax(o, nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }
}) })
t.Run("from string success", func(t *testing.T) { t.Run("from string success", func(t *testing.T) {
o := checkSyntaxOptions{SyntaxString: `foo = "bar"`} o := checkSyntaxOptions{SyntaxString: ptrto(`foo = "bar"`)}
if err := checkSyntax(o, nil); err != nil { if err := checkSyntax(o, nil); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -2,48 +2,49 @@
Generated with https://code.squareroundforest.org/arpio/docreflect Generated with https://code.squareroundforest.org/arpio/docreflect
*/ */
package main package main
import "code.squareroundforest.org/arpio/docreflect" import "code.squareroundforest.org/arpio/docreflect"
func init() { func init() {
docreflect.Register("main", "") docreflect.Register("main", "")
docreflect.Register("main.check", "check parses input content against the provided syntax definition and fails if the input does not match.\nSyntax can be provided via a filename option or an inline string option. Input can be provided via a filename\noption, a positional argument filename, an inline string option, or piped from standard input.\n\nfunc(o, stdin, args)") docreflect.Register("main.check", "check parses input content against the provided syntax definition and fails if the input does not match.\nSyntax can be provided via a filename option or an inline string option. Input can be provided via a filename\noption, a positional argument filename, an inline string option, or piped from standard input.\n\nfunc(o, stdin, args)")
docreflect.Register("main.checkOptions", "") docreflect.Register("main.checkOptions", "")
docreflect.Register("main.checkOptions.Input", "Input specifies the filename of the input content to be validated.\n") docreflect.Register("main.checkOptions.Input", "Input specifies the filename of the input content to be validated.\n")
docreflect.Register("main.checkOptions.InputString", "InputString specifies the input content as an inline string.\n") docreflect.Register("main.checkOptions.InputString", "InputString specifies the input content as an inline string.\n")
docreflect.Register("main.checkOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.checkOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.checkOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") docreflect.Register("main.checkOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.checkSyntax", "checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a\npositional argument), an inline string, or piped from standard input.\n\nfunc(o, stdin, args)") docreflect.Register("main.checkSyntax", "checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a\npositional argument), an inline string, or piped from standard input.\n\nfunc(o, stdin, args)")
docreflect.Register("main.checkSyntaxOptions", "") docreflect.Register("main.checkSyntaxOptions", "")
docreflect.Register("main.checkSyntaxOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.checkSyntaxOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.checkSyntaxOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") docreflect.Register("main.checkSyntaxOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.errInvalidFilename", "") docreflect.Register("main.errInvalidFilename", "")
docreflect.Register("main.errMultipleInputs", "") docreflect.Register("main.errMultipleInputs", "")
docreflect.Register("main.errNoInput", "") docreflect.Register("main.errNoInput", "")
docreflect.Register("main.generate", "generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded\nin an application.\n\nThe syntax may be provided via a file path (using an option or a positional argument), an\ninline string, or piped from standard input.\n\nfunc(o, stdin, stdout, args)") docreflect.Register("main.generate", "generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded\nin an application.\n\nThe syntax may be provided via a file path (using an option or a positional argument), an\ninline string, or piped from standard input.\n\nfunc(o, stdin, stdout, args)")
docreflect.Register("main.generateOptions", "") docreflect.Register("main.generateOptions", "")
docreflect.Register("main.generateOptions.Export", "Export determines whether the generated parse function is exported (visible outside its package).\n") docreflect.Register("main.generateOptions.Export", "Export determines whether the generated parse function is exported (visible outside its package).\n")
docreflect.Register("main.generateOptions.PackageName", "PackageName specifies the package name for the generated code. Defaults to main.\n") docreflect.Register("main.generateOptions.PackageName", "PackageName specifies the package name for the generated code. Defaults to main.\n")
docreflect.Register("main.generateOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.generateOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.generateOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") docreflect.Register("main.generateOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.init", "\nfunc()") docreflect.Register("main.init", "\nfunc()")
docreflect.Register("main.initInput", "\nfunc(filename, stringValue, stdin, args)") docreflect.Register("main.initInput", "\nfunc(filename, stringValue, stdin, args)")
docreflect.Register("main.main", "\nfunc()") docreflect.Register("main.main", "\nfunc()")
docreflect.Register("main.mapNode", "\nfunc(n)") docreflect.Register("main.mapNode", "\nfunc(n)")
docreflect.Register("main.node", "") docreflect.Register("main.node", "")
docreflect.Register("main.node.From", "") docreflect.Register("main.node.From", "")
docreflect.Register("main.node.Name", "") docreflect.Register("main.node.Name", "")
docreflect.Register("main.node.Nodes", "") docreflect.Register("main.node.Nodes", "")
docreflect.Register("main.node.Text", "") docreflect.Register("main.node.Text", "")
docreflect.Register("main.node.To", "") docreflect.Register("main.node.To", "")
docreflect.Register("main.noop", "\nfunc()") docreflect.Register("main.noop", "\nfunc()")
docreflect.Register("main.show", "show input content against a provided syntax definition and outputs the resulting AST (Abstract Syntax Tree)\nin JSON format. Syntax can be provided via a filename option or an inline string option. Input can be\nprovided via a filename option, a positional argument filename, an inline string option, or piped from\nstandard input.\n\nfunc(o, stdin, stdout, args)") docreflect.Register("main.show", "show input content against a provided syntax definition and outputs the resulting AST (Abstract Syntax Tree)\nin JSON format. Syntax can be provided via a filename option or an inline string option. Input can be\nprovided via a filename option, a positional argument filename, an inline string option, or piped from\nstandard input.\n\nfunc(o, stdin, stdout, args)")
docreflect.Register("main.showOptions", "") docreflect.Register("main.showOptions", "")
docreflect.Register("main.showOptions.Indent", "Indent specifies a custom indentation string for the output.\n") docreflect.Register("main.showOptions.Indent", "Indent specifies a custom indentation string for the output.\n")
docreflect.Register("main.showOptions.Input", "Input specifies the filename of the input content to be validated.\n") docreflect.Register("main.showOptions.Input", "Input specifies the filename of the input content to be validated.\n")
docreflect.Register("main.showOptions.InputString", "InputString specifies the input content as an inline string.\n") docreflect.Register("main.showOptions.InputString", "InputString specifies the input content as an inline string.\n")
docreflect.Register("main.showOptions.Pretty", "Pretty enables indented, human-readable output.\n") docreflect.Register("main.showOptions.Pretty", "Pretty enables indented, human-readable output.\n")
docreflect.Register("main.showOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n") docreflect.Register("main.showOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.showOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n") docreflect.Register("main.showOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.version", "") docreflect.Register("main.version", "")
} }

View File

@ -8,10 +8,10 @@ import (
type generateOptions struct { type generateOptions struct {
// Syntax specifies the filename of the syntax definition file. // Syntax specifies the filename of the syntax definition file.
Syntax string Syntax *string
// SyntaxString specifies the syntax as an inline string. // SyntaxString specifies the syntax as an inline string.
SyntaxString string SyntaxString *string
// PackageName specifies the package name for the generated code. Defaults to main. // PackageName specifies the package name for the generated code. Defaults to main.
PackageName string PackageName string

View File

@ -12,7 +12,7 @@ import (
func TestGenerate(t *testing.T) { func TestGenerate(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) { t.Run("too many inputs", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := generateOptions{Syntax: "foo_test.treerack", SyntaxString: `foo = "42"`} o := generateOptions{Syntax: ptrto("foo_test.treerack"), SyntaxString: ptrto(`foo = "42"`)}
if err := generate(o, nil, &out); !errors.Is(err, errMultipleInputs) { if err := generate(o, nil, &out); !errors.Is(err, errMultipleInputs) {
t.Fatal() t.Fatal()
} }
@ -46,7 +46,7 @@ func TestGenerate(t *testing.T) {
perr *treerack.ParseError perr *treerack.ParseError
) )
o := generateOptions{SyntaxString: "foo"} o := generateOptions{SyntaxString: ptrto("foo")}
if err := generate(o, nil, &out); !errors.As(err, &perr) { if err := generate(o, nil, &out); !errors.As(err, &perr) {
t.Fatal() t.Fatal()
} }
@ -54,7 +54,7 @@ func TestGenerate(t *testing.T) {
t.Run("invalid syntax init", func(t *testing.T) { t.Run("invalid syntax init", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := generateOptions{SyntaxString: `foo = "42"; foo = "84"`} o := generateOptions{SyntaxString: ptrto(`foo = "42"; foo = "84"`)}
if err := generate(o, nil, &out); err == nil { if err := generate(o, nil, &out); err == nil {
t.Fatal() t.Fatal()
} }
@ -62,7 +62,7 @@ func TestGenerate(t *testing.T) {
t.Run("success", func(t *testing.T) { t.Run("success", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := generateOptions{Syntax: "foo_test.treerack"} o := generateOptions{Syntax: ptrto("foo_test.treerack")}
if err := generate(o, nil, &out); err != nil { if err := generate(o, nil, &out); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -75,7 +75,7 @@ func TestGenerate(t *testing.T) {
t.Run("success string", func(t *testing.T) { t.Run("success string", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := generateOptions{SyntaxString: `foo = "bar"`} o := generateOptions{SyntaxString: ptrto(`foo = "bar"`)}
if err := generate(o, nil, &out); err != nil { if err := generate(o, nil, &out); err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -122,7 +122,7 @@ func TestGenerate(t *testing.T) {
t.Run("custom package name", func(t *testing.T) { t.Run("custom package name", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := generateOptions{ o := generateOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
PackageName: "foo", PackageName: "foo",
} }
@ -139,7 +139,7 @@ func TestGenerate(t *testing.T) {
t.Run("export", func(t *testing.T) { t.Run("export", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := generateOptions{ o := generateOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Export: true, Export: true,
} }

View File

@ -17,16 +17,18 @@ var (
func noop() {} func noop() {}
func initInput( func initInput(
filename, stringValue string, stdin io.Reader, args []string, filename, stringValue *string, stdin io.Reader, args []string,
) (input io.Reader, finalize func(), err error) { ) (
input io.Reader, finalize func(), err error,
) {
finalize = noop finalize = noop
var inputCount int var inputCount int
if filename != "" { if filename != nil {
inputCount++ inputCount++
} }
if stringValue != "" { if stringValue != nil {
inputCount++ inputCount++
} }
@ -39,19 +41,20 @@ func initInput(
return return
} }
if len(args) > 0 && args[0] == "" {
err = errInvalidFilename
return
}
if len(args) > 0 { if len(args) > 0 {
filename = args[0] filename = new(string)
*filename = args[0]
} }
switch { switch {
case filename != "": case filename != nil:
if *filename == "" {
err = errInvalidFilename
return
}
var f io.ReadCloser var f io.ReadCloser
f, err = os.Open(filename) f, err = os.Open(*filename)
if err != nil { if err != nil {
return return
} }
@ -63,8 +66,8 @@ func initInput(
} }
input = f input = f
case stringValue != "": case stringValue != nil:
input = bytes.NewBufferString(stringValue) input = bytes.NewBufferString(*stringValue)
default: default:
if stdin == nil { if stdin == nil {
err = errNoInput err = errNoInput

View File

@ -9,16 +9,16 @@ import (
type showOptions struct { type showOptions struct {
// Syntax specifies the filename of the syntax definition file. // Syntax specifies the filename of the syntax definition file.
Syntax string Syntax *string
// SyntaxString specifies the syntax as an inline string. // SyntaxString specifies the syntax as an inline string.
SyntaxString string SyntaxString *string
// Input specifies the filename of the input content to be validated. // Input specifies the filename of the input content to be validated.
Input string Input *string
// InputString specifies the input content as an inline string. // InputString specifies the input content as an inline string.
InputString string InputString *string
// Pretty enables indented, human-readable output. // Pretty enables indented, human-readable output.
Pretty bool Pretty bool

View File

@ -11,7 +11,7 @@ import (
func TestShow(t *testing.T) { func TestShow(t *testing.T) {
t.Run("no syntax", func(t *testing.T) { t.Run("no syntax", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{Input: "bar_test.txt"} o := showOptions{Input: ptrto("bar_test.txt")}
if err := show(o, nil, &out); !errors.Is(err, errNoInput) { if err := show(o, nil, &out); !errors.Is(err, errNoInput) {
t.Fatal() t.Fatal()
} }
@ -20,9 +20,9 @@ func TestShow(t *testing.T) {
t.Run("too many syntaxes", func(t *testing.T) { t.Run("too many syntaxes", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
SyntaxString: `foo = "baz"`, SyntaxString: ptrto(`foo = "baz"`),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := show(o, nil, &out); !errors.Is(err, errMultipleInputs) { if err := show(o, nil, &out); !errors.Is(err, errMultipleInputs) {
@ -33,8 +33,8 @@ func TestShow(t *testing.T) {
t.Run("syntax file not found", func(t *testing.T) { t.Run("syntax file not found", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "no-file.treerack", Syntax: ptrto("no-file.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := show(o, nil, &out); !os.IsNotExist(err) { if err := show(o, nil, &out); !os.IsNotExist(err) {
@ -45,8 +45,8 @@ func TestShow(t *testing.T) {
t.Run("invalid syntax definition", func(t *testing.T) { t.Run("invalid syntax definition", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
SyntaxString: `foo`, SyntaxString: ptrto(`foo`),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
var perr *treerack.ParseError var perr *treerack.ParseError
@ -58,8 +58,8 @@ func TestShow(t *testing.T) {
t.Run("invalid syntax init", func(t *testing.T) { t.Run("invalid syntax init", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
SyntaxString: `foo = "bar"; foo = "baz"`, SyntaxString: ptrto(`foo = "bar"; foo = "baz"`),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := show(o, nil, &out); err == nil { if err := show(o, nil, &out); err == nil {
@ -69,7 +69,7 @@ func TestShow(t *testing.T) {
t.Run("no input", func(t *testing.T) { t.Run("no input", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"} o := showOptions{Syntax: ptrto("foo_test.treerack")}
if err := show(o, nil, &out); !errors.Is(err, errNoInput) { if err := show(o, nil, &out); !errors.Is(err, errNoInput) {
t.Fatal() t.Fatal()
@ -79,8 +79,8 @@ func TestShow(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) { t.Run("too many inputs", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := show(o, nil, &out, "baz_test.txt"); !errors.Is(err, errMultipleInputs) { if err := show(o, nil, &out, "baz_test.txt"); !errors.Is(err, errMultipleInputs) {
@ -90,7 +90,7 @@ func TestShow(t *testing.T) {
t.Run("empty filename for input", func(t *testing.T) { t.Run("empty filename for input", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"} o := showOptions{Syntax: ptrto("foo_test.treerack")}
if err := show(o, nil, &out, ""); !errors.Is(err, errInvalidFilename) { if err := show(o, nil, &out, ""); !errors.Is(err, errInvalidFilename) {
t.Fatal() t.Fatal()
} }
@ -98,7 +98,7 @@ func TestShow(t *testing.T) {
t.Run("input file not found", func(t *testing.T) { t.Run("input file not found", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"} o := showOptions{Syntax: ptrto("foo_test.treerack")}
if err := show(o, nil, &out, "baz_test.txt"); !os.IsNotExist(err) { if err := show(o, nil, &out, "baz_test.txt"); !os.IsNotExist(err) {
t.Fatal() t.Fatal()
} }
@ -107,8 +107,8 @@ func TestShow(t *testing.T) {
t.Run("input parse fail", func(t *testing.T) { t.Run("input parse fail", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
InputString: "baz", InputString: ptrto("baz"),
} }
var perr *treerack.ParseError var perr *treerack.ParseError
@ -120,8 +120,8 @@ func TestShow(t *testing.T) {
t.Run("show", func(t *testing.T) { t.Run("show", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
} }
if err := show(o, nil, &out); err != nil { if err := show(o, nil, &out); err != nil {
@ -136,8 +136,8 @@ func TestShow(t *testing.T) {
t.Run("show string", func(t *testing.T) { t.Run("show string", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
InputString: "bar", InputString: ptrto("bar"),
} }
if err := show(o, nil, &out); err != nil { if err := show(o, nil, &out); err != nil {
@ -151,9 +151,7 @@ func TestShow(t *testing.T) {
t.Run("show file", func(t *testing.T) { t.Run("show file", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{Syntax: ptrto("foo_test.treerack")}
Syntax: "foo_test.treerack",
}
if err := show(o, nil, &out, "bar_test.txt"); err != nil { if err := show(o, nil, &out, "bar_test.txt"); err != nil {
t.Fatal(nil) t.Fatal(nil)
@ -166,7 +164,7 @@ func TestShow(t *testing.T) {
t.Run("show stdin", func(t *testing.T) { t.Run("show stdin", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"} o := showOptions{Syntax: ptrto("foo_test.treerack")}
in := bytes.NewBufferString("bar") in := bytes.NewBufferString("bar")
if err := show(o, in, &out); err != nil { if err := show(o, in, &out); err != nil {
t.Fatal(nil) t.Fatal(nil)
@ -180,8 +178,8 @@ func TestShow(t *testing.T) {
t.Run("indent", func(t *testing.T) { t.Run("indent", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
Pretty: true, Pretty: true,
} }
@ -198,8 +196,8 @@ func TestShow(t *testing.T) {
t.Run("custom indent", func(t *testing.T) { t.Run("custom indent", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
Indent: "xx", Indent: "xx",
} }
@ -215,8 +213,8 @@ func TestShow(t *testing.T) {
t.Run("redundant custom indent", func(t *testing.T) { t.Run("redundant custom indent", func(t *testing.T) {
var out bytes.Buffer var out bytes.Buffer
o := showOptions{ o := showOptions{
Syntax: "foo_test.treerack", Syntax: ptrto("foo_test.treerack"),
Input: "bar_test.txt", Input: ptrto("bar_test.txt"),
Pretty: true, Pretty: true,
Indent: "xx", Indent: "xx",
} }

View File

@ -120,6 +120,10 @@ func (c *context) fail(offset int) {
} }
func findLine(tokens []rune, offset int) (line, column int) { func findLine(tokens []rune, offset int) (line, column int) {
if offset < 0 {
return 0, 0
}
tokens = tokens[:offset] tokens = tokens[:offset]
for i := range tokens { for i := range tokens {
column++ column++
@ -144,7 +148,6 @@ func (c *context) parseError(p parser) error {
} }
line, col := findLine(c.tokens, c.failOffset) line, col := findLine(c.tokens, c.failOffset)
return &ParseError{ return &ParseError{
Offset: c.failOffset, Offset: c.failOffset,
Line: line, Line: line,

View File

@ -14,54 +14,48 @@ import (
var errExit = errors.New("exit") var errExit = errors.New("exit")
// repl runs the Read-Eval-Print Loop.
func repl(input io.Reader, output io.Writer) { func repl(input io.Reader, output io.Writer) {
// use buffered io, to be able to read the input line-by-line:
// use buffered io, to read the input line-by-line:
buf := bufio.NewReader(os.Stdin) buf := bufio.NewReader(os.Stdin)
// our REPL loop: // our REPL:
for { for {
// print a basic prompt: // print a input prompt marker:
if _, err := output.Write([]byte("> ")); err != nil { if _, err := output.Write([]byte("> ")); err != nil {
// we cannot fix it if there is an error here:
log.Fatalln(err) log.Fatalln(err)
} }
// read the input and handle the errors: // read the input and handle the errors:
expr, err := read(buf) expr, err := read(buf)
// when EOF, that means the user pressed Ctrl+D. Let's terminate the output with a conventional newline // handle EOF (Ctrl+D):
// and exit:
if errors.Is(err, io.EOF) { if errors.Is(err, io.EOF) {
output.Write([]byte{'\n'}) output.Write([]byte{'\n'})
os.Exit(0) os.Exit(0)
} }
// when errExit, that means the user entered exit: // handle the explicit exit command:
if errors.Is(err, errExit) { if errors.Is(err, errExit) {
os.Exit(0) os.Exit(0)
} }
// if it's a parser error, we print and continue from reading again, to allow the user to fix the // handle parser errors (allow the user to retry):
// problem:
var perr *parseError var perr *parseError
if errors.As(err, &perr) { if errors.As(err, &perr) {
log.Println(err) log.Println(err)
continue continue
} }
// in case of any other error, we don't know what's going on, so we get out of here right away: // handle possible I/O errors:
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
// if we received an expression, then we can evaluate it. We are not expecting errors here: // evaluate and print:
result := eval(expr) result := eval(expr)
// we have the result, we need to print it:
if err := print(output, result); err != nil { if err := print(output, result); err != nil {
// if printing fails, we don't know how to fix it, so we get out of here:
log.Fatalln(err) log.Fatalln(err)
} }
} }
@ -73,7 +67,7 @@ func read(input *bufio.Reader) (*node, error) {
return nil, err return nil, err
} }
// expr will be of type *node, which type is defined in the generated code // parse the line using the generated parser:
expr, err := parse(bytes.NewBufferString(line)) expr, err := parse(bytes.NewBufferString(line))
if err != nil { if err != nil {
return nil, err return nil, err
@ -83,15 +77,12 @@ func read(input *bufio.Reader) (*node, error) {
return nil, errExit return nil, errExit
} }
// we know based on the syntax, that the top level node will always have a single child, either a number // based on our syntax, the root node always has exactly one child: either a number or a binary operation.
// literal or a binary operation:
return expr.Nodes[0], nil return expr.Nodes[0], nil
} }
// eval always returns the calculated result as a float64: // eval always returns the calculated result as a float64:
func eval(expr *node) float64 { func eval(expr *node) float64 {
// we know that it's either a number or a binary operation:
var value float64 var value float64
switch expr.Name { switch expr.Name {
case "num": case "num":
@ -103,10 +94,8 @@ func eval(expr *node) float64 {
return value return value
default: default:
// we know that the first node is either a number of a child expression: // evaluate binary expressions. Format: Operand [Operator Operand]...
value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:] value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:]
// we don't need to track back, so we can drop the processed nodes while consuming them:
for len(expr.Nodes) > 0 { for len(expr.Nodes) > 0 {
var ( var (
operator string operator string
@ -122,8 +111,7 @@ func eval(expr *node) float64 {
case "mul": case "mul":
value *= operand value *= operand
case "div": case "div":
// Go returns -Inf or +Inf on division by zero: value /= operand // Go returns on division by zero +/-Inf
value /= operand
} }
} }
} }
@ -132,12 +120,13 @@ func eval(expr *node) float64 {
} }
func print(output io.Writer, result float64) error { func print(output io.Writer, result float64) error {
// we can use the stdlib fmt package to print float64:
_, err := fmt.Fprintln(output, result) _, err := fmt.Fprintln(output, result)
return err return err
} }
func main() { func main() {
// for testability, we define the REPL loop in a separate function so that the test code can call it with // for testability, we define the REPL in a separate function so that the test code can call it with
// in-memory buffers as input and output. Our main function calls it with the stdio handles: // in-memory buffers as input and output. Our main function calls it with the stdio handles:
repl(os.Stdin, os.Stdout) repl(os.Stdin, os.Stdout)
} }

View File

@ -24,7 +24,7 @@ Alternatively, we _may be able to_ install directly using the Go toolchain:
## Hello syntax ## Hello syntax
A basic syntax definition looks like this: A trivial syntax definition looks like this:
``` ```
hello = "Hello, world!" hello = "Hello, world!"
@ -83,10 +83,10 @@ If our syntax definition is invalid, check-syntax will fail:
treerack check-syntax --syntax-string 'foo = bar' treerack check-syntax --syntax-string 'foo = bar'
``` ```
The above command will fail because the parser called foo references an undefined parser bar. The above command will fail because the parser called `foo` references an undefined parser `bar`.
We can use check or show to detect when the input content does not match a valid syntax. Using the hello syntax, We can use `check` or `show` to detect when the input content does not match a valid syntax. Using the hello
we can try the following: syntax, we can try the following:
``` ```
treerack check --syntax-string 'hello = "Hello, world!"' --input-string 'Hi!' treerack check --syntax-string 'hello = "Hello, world!"' --input-string 'Hi!'
@ -96,9 +96,9 @@ It will show that parsing the input failed and that it failed while using the pa
## Basic syntax - An arithmetic calculator ## Basic syntax - An arithmetic calculator
In this section, we will build a basic arithmetic calculator. It will read a line from standard input, parse it In this section, we will build a simplistic arithmetic calculator. It will read a line from standard input,
as an arithmetic expression, compute the result, and print it—effectively creating a REPL (Read-Eval-Print parse it as an arithmetic expression, compute the result, print it, and start over - effectively creating a REPL
Loop). (Read-Eval-Print Loop).
We will support addition +, subtraction -, multiplication *, division /, and grouping with parentheses (). We will support addition +, subtraction -, multiplication *, division /, and grouping with parentheses ().
@ -133,10 +133,10 @@ group:alias = "(" expression ")";
// //
// We group operators by precedence levels to ensure correct order of operations. // We group operators by precedence levels to ensure correct order of operations.
// //
// Level 0 (High): Multiplication/Division // Level 0 (high): multiplication/division
op0:alias = mul | div; op0:alias = mul | div;
// Level 1 (Low): Addition/Subtraction // Level 1 (low): addition/subtraction
op1:alias = add | sub; op1:alias = add | sub;
// Operands for each precedence level. // Operands for each precedence level.
@ -401,12 +401,12 @@ var errExit = errors.New("exit")
// repl runs the Read-Eval-Print Loop. // repl runs the Read-Eval-Print Loop.
func repl(input io.Reader, output io.Writer) { func repl(input io.Reader, output io.Writer) {
// use buffered io, to be able to read the input line-by-line: // use buffered io, to read the input line-by-line:
buf := bufio.NewReader(os.Stdin) buf := bufio.NewReader(os.Stdin)
// our REPL loop: // our REPL:
for { for {
// print a basic prompt: // print a input prompt marker:
if _, err := output.Write([]byte("> ")); err != nil { if _, err := output.Write([]byte("> ")); err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
@ -414,29 +414,30 @@ func repl(input io.Reader, output io.Writer) {
// read the input and handle the errors: // read the input and handle the errors:
expr, err := read(buf) expr, err := read(buf)
// Handle EOF (Ctrl+D) // handle EOF (Ctrl+D):
if errors.Is(err, io.EOF) { if errors.Is(err, io.EOF) {
output.Write([]byte{'\n'}) output.Write([]byte{'\n'})
os.Exit(0) os.Exit(0)
} }
// Handle explicit exit command // handle the explicit exit command:
if errors.Is(err, errExit) { if errors.Is(err, errExit) {
os.Exit(0) os.Exit(0)
} }
// Handle parser errors (allow user to retry) // handle parser errors (allow the user to retry):
var perr *parseError var perr *parseError
if errors.As(err, &perr) { if errors.As(err, &perr) {
log.Println(err) log.Println(err)
continue continue
} }
// handle possible I/O errors:
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
// Evaluate and print // evaluate and print:
result := eval(expr) result := eval(expr)
if err := print(output, result); err != nil { if err := print(output, result); err != nil {
log.Fatalln(err) log.Fatalln(err)
@ -450,7 +451,7 @@ func read(input *bufio.Reader) (*node, error) {
return nil, err return nil, err
} }
// Parse the line using the generated parser // parse the line using the generated parser:
expr, err := parse(bytes.NewBufferString(line)) expr, err := parse(bytes.NewBufferString(line))
if err != nil { if err != nil {
return nil, err return nil, err
@ -460,8 +461,7 @@ func read(input *bufio.Reader) (*node, error) {
return nil, errExit return nil, errExit
} }
// Based on our syntax, the root node always has exactly one child: // based on our syntax, the root node always has exactly one child: either a number or a binary operation.
// either a number or a binary operation.
return expr.Nodes[0], nil return expr.Nodes[0], nil
} }
@ -478,8 +478,7 @@ func eval(expr *node) float64 {
return value return value
default: default:
// Handle binary expressions (recursively) // handle binary expressions. Format: Operand [Operator Operand]...
// Format: Operand [Operator Operand]...
value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:] value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:]
for len(expr.Nodes) > 0 { for len(expr.Nodes) > 0 {
var ( var (
@ -496,7 +495,7 @@ func eval(expr *node) float64 {
case "mul": case "mul":
value *= operand value *= operand
case "div": case "div":
value /= operand // Go handles division by zero as ±Inf value /= operand // Go handles division by zero as +/-Inf
} }
} }
} }
@ -505,12 +504,13 @@ func eval(expr *node) float64 {
} }
func print(output io.Writer, result float64) error { func print(output io.Writer, result float64) error {
// we can use
_, err := fmt.Fprintln(output, result) _, err := fmt.Fprintln(output, result)
return err return err
} }
func main() { func main() {
// for testability, we define the REPL loop in a separate function so that the test code can call it with // for testability, we define the REPL in a separate function so that the test code can call it with
// in-memory buffers as input and output. Our main function calls it with the stdio handles: // in-memory buffers as input and output. Our main function calls it with the stdio handles:
repl(os.Stdin, os.Stdout) repl(os.Stdin, os.Stdout)
} }
@ -533,13 +533,13 @@ $ go run .
We can find the source files for this example here: [./examples/acalc](./examples/acalc). We can find the source files for this example here: [./examples/acalc](./examples/acalc).
## Important Note: Unescaping ## Important note: unescaping
Treerack does not automatically handle escape sequences (e.g., converting \n to a literal newline). If our Treerack does not automatically handle escape sequences (e.g., converting `\n` to a literal newline). If our
syntax supports escaped characters—common in string literals—the user code is responsible for "unescaping" the syntax supports escaped characters - common in string literals - the user code is responsible for "unescaping"
raw text from the AST node. the raw text from the AST node.
This is analogous to how we needed to parse the numbers in the calculator example to convert the string This is analogous to how we needed to interpret the numbers in the calculator example to convert the string
representation of a number into a Go float64. representation of a number into a Go float64.
## Programmatically loading syntaxes ## Programmatically loading syntaxes
@ -569,7 +569,7 @@ func initAndParse(syntax, content io.Reader) (*treerack.Node, error) {
} }
``` ```
Caution: Be mindful of security implications when loading syntax definitions from untrusted sources. Caution: be mindful of security implications when loading syntax definitions from untrusted sources.
## Programmatically defining syntaxes ## Programmatically defining syntaxes
@ -611,7 +611,7 @@ func initAndParse(content io.Reader) (*treerack.Node, error) {
## Summary ## Summary
We have demonstrated how to use the Treerack tool to define, test, and implement a parser. We recommend the We have demonstrated how to use the treerack tool to define, test, and implement a parser. We recommend the
following workflow: following workflow:
1. draft: define a syntax in a .treerack file. 1. draft: define a syntax in a .treerack file.

View File

@ -5,14 +5,14 @@ It allows for the concise definition of recursive descent parsers.
A syntax file consists of a series of Production Rules (definitions), terminated by semicolons. A syntax file consists of a series of Production Rules (definitions), terminated by semicolons.
## Production Rules ## Production rules
A rule assigns a name to a pattern expression. Rules may include optional flags to modify the parser's behavior A rule assigns a name to a pattern expression. Rules may include optional flags to modify the parser's behavior
or the resulting AST (Abstract Syntax Tree). or the resulting AST (Abstract Syntax Tree).
``` ```
RuleName = Expression; rule-name = expression;
RuleName:flag1:flag2 = Expression; rule-name:flag1:flag2 = expression;
``` ```
## Flags ## Flags
@ -20,18 +20,19 @@ RuleName:flag1:flag2 = Expression;
Flags are appended to the rule name, separated by colons. They control AST generation, whitespace handling, and Flags are appended to the rule name, separated by colons. They control AST generation, whitespace handling, and
error propagation. error propagation.
- `alias`: Transparent Node. The rule validates input but does not create its own node in the AST. Children - `alias`: transparent node. The rule validates input but does not create its own node in the AST. Children
nodes (if any) are attached to the parent of this rule. nodes (if any) are attached to the parent of this rule.
- `ws`: Global Whitespace. Marks this rule as the designated whitespace handler. The parser will attempt to - `ws`: global whitespace. Marks this rule as the designated whitespace handler. The parser will attempt to
match (and discard) this rule between tokens throughout the entire syntax. match (and discard) this rule between tokens throughout the entire syntax.
- `nows`: No Whitespace. Disables automatic whitespace skipping inside this rule. Useful for defining tokens - `nows`: no whitespace. Disables automatic whitespace skipping inside this rule. Useful for defining tokens
like string literals where spaces are significant. like string literals where spaces are significant. The flag `nows` is automatically applied to char sequences
- `root`: Entry Point. Explicitly marks the rule as the starting point of the syntax. If omitted, the last like `"abc" or [abc]+.
- `root`: entry point. Explicitly marks the rule as the starting point of the syntax. If omitted, the last
defined rule is implied to be the root. defined rule is implied to be the root.
- `kw`: Keyword. Marks the content as a reserved keyword. - `kw`: keyword. Marks the content as a reserved keyword.
- `nokw`: No Keyword. Prevents the rule from matching text that matches a defined kw rule. Essential for - `nokw`: no keyword. Prevents the rule from matching text that matches a defined kw rule. Essential for
distinguishing identifiers from keywords (e.g., ensuring var is not parsed as a variable name). distinguishing identifiers from keywords (e.g., ensuring var is not parsed as a variable name).
- `failpass`: Pass Failure. If this rule fails to parse, the error is reported as a failure of the parent rule, - `failpass`: pass failure. If this rule fails to parse, the error is reported as a failure of the parent rule,
not this specific rule. not this specific rule.
## Expressions ## Expressions
@ -43,7 +44,7 @@ and quantifiers.
Terminals match specific characters or strings in the input. Terminals match specific characters or strings in the input.
- `"abc"` (string): Matches an exact sequence of characters. - `"abc"` (string): Matches an exact sequence of characters. Equivalent to [a][b][c].
- `.` (any char): Matches any single character (wildcard). - `.` (any char): Matches any single character (wildcard).
- `[123]`, `[a-z]`, `[123a-z]` (class): Matches a single character from a set or range. - `[123]`, `[a-z]`, `[123a-z]` (class): Matches a single character from a set or range.
- `[^123]`, `[^a-z]`, `[^123a-z]` (not class) Matches any single character not in the set. - `[^123]`, `[^a-z]`, `[^123a-z]` (not class) Matches any single character not in the set.
@ -52,13 +53,13 @@ Terminals match specific characters or strings in the input.
Quantifiers determine how many times an item must match. They are placed immediately after the item they modify. Quantifiers determine how many times an item must match. They are placed immediately after the item they modify.
- `?`: Optional (Zero or one). - `?`: optional (zero or one).
- `*`: Zero or more. - `*`: zero or more.
- `+`: One or more. - `+`: one or more.
- `{n}`: Exact count. Matches exactly n times. - `{n}`: exact count. Matches exactly n times.
- `{n,}`: At least. Matches n or more times. - `{n,}`: at least. Matches n or more times.
- `{,m}`: At most. Matches between 0 and m times. - `{,m}`: at most. Matches between 0 and m times.
- `{n,m}`: Range. Matches between n and m times. - `{n,m}`: range. Matches between n and m times.
## Composites ## Composites
@ -69,8 +70,8 @@ Complex patterns are built by combining terminals and other rules.
Items written consecutively are matched in order. Items written consecutively are matched in order.
``` ```
// Matches "A", then "B", then "C" // matches "A", then "B", then "C":
MySequence = "A" "B" "C"; my-sequence = "A" "B" "C";
``` ```
### 2. Grouping ### 2. Grouping
@ -78,8 +79,8 @@ MySequence = "A" "B" "C";
Parentheses (...) group items together, allowing quantifiers to apply to the entire group. Parentheses (...) group items together, allowing quantifiers to apply to the entire group.
``` ```
// Matches "AB", "ABAB", "ABABAB"... // matches "AB", "ABAB", "ABABAB"...:
MyGroup = ("A" "B")+; my-group = ("A" "B")+;
``` ```
### 3. Choices ### 3. Choices
@ -89,21 +90,18 @@ The pipe | character represents a choice between alternatives.
The parser evaluates all provided options against the input at the current position and selects the best match The parser evaluates all provided options against the input at the current position and selects the best match
based on the following priority rules: based on the following priority rules:
1. _Longest Match_: The option that consumes the largest number of characters takes priority. This eliminates the 1. _longest match_: the option that consumes the largest number of characters takes priority. This eliminates the
need to manually order specific matches before general ones (e.g., "integer" will always be chosen over "int" if need to manually order specific matches before general ones (e.g., "integer" will always be chosen over "int" if
the input supports it, regardless of their order in the definition). the input supports it, regardless of their order in the definition).
2. _First Definition Wins_: If multiple options consume the exact same number of characters, the option defined 2. _first definition wins_: if multiple options consume the exact same number of characters, the option defined
first(left-most) in the list takes priority. first(left-most) in the list takes priority.
``` ```
// Longest match wins automatically: // longest match wins automatically: input "integer" is matched by 'type', even though "int" comes first.
// Input "integer" is matched by 'type', even though "int" comes first.
type = "int" | "integer"; type = "int" | "integer";
// Tie-breaker rule: // Tie-breaker rule: if input is "foo", both options match 3 characters. Because 'identifier' is last, it takes
// If input is "foo", both options match 3 characters. // priority over 'keyword'. (Use :kw and :nokw to control such situations, when it applies.)
// Because 'identifier' is last, it takes priority over 'keyword'.
// (Use :kw and :nokw to control such situations, when it applies.)
content = keyword | identifier; content = keyword | identifier;
``` ```
@ -111,8 +109,8 @@ content = keyword | identifier;
Comments follow C-style syntax and are ignored by the definition parser. Comments follow C-style syntax and are ignored by the definition parser.
- Line comments: Start with // and end at the newline. - line comments: start with // and end at the newline.
- Block comments: Enclosed in /* ... */. - block comments: enclosed in /* ... */.
## Examples ## Examples

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -687,6 +687,9 @@ func (c *context) fail(offset int) {
c.matchLast = false c.matchLast = false
} }
func findLine(tokens []rune, offset int) (line, column int) { func findLine(tokens []rune, offset int) (line, column int) {
if offset < 0 {
return 0, 0
}
tokens = tokens[:offset] tokens = tokens[:offset]
for i := range tokens { for i := range tokens {
column++ column++

View File

@ -1,40 +1,14 @@
[next] [next]
errors errors
generator 1
documentation
parser 1
releasing
parser 2
generator 2
formatter formatter
report unused parsers report unused parsers
parse hashed, storing only the results
linux packaging
[errors] [errors]
take the last don't report aliases
take the last meaningful sequence
test error report on invalid flag test error report on invalid flag
input name: may be just dropped because completely controlled by the client
input name needed in command to differentiate between syntax and input in check and parse subcommands input name needed in command to differentiate between syntax and input in check and parse subcommands
[generator 1]
allchars: can have char sequence
make generator output non-random (track parsers in a list in definition order)
fix the license in the output
[generator 2]
js
[releasing]
spellcheck
linting
convert notes into issues
try to remove some files
[parser 1]
try winning on allChars
retry collapsing
[parser 2] [parser 2]
custom tokens custom tokens
indentation indentation
@ -42,10 +16,3 @@ streaming support // ReadNode(io.Reader)
[optimization] [optimization]
try preallocate larger store chunks try preallocate larger store chunks
[documentation]
how the char classes are different from regexp
why need nows when using ws
lib only useful for dynamic syntax definition
warn nows usage in docs, e.g. spaces in symbol = [a-z]+
tutorial

View File

@ -162,6 +162,39 @@ func TestRecursion(t *testing.T) {
}, },
}}, }},
) )
runTests(
t,
"A = A [a]",
[]testItem{{
title: "naive left recursion expects infinite stream",
text: "a",
ignorePosition: true,
fail: true,
}},
)
runTests(
t,
"A = A [a]; B = A",
[]testItem{{
title: "embedded naive left recursion expects infinite stream",
text: "a",
ignorePosition: true,
fail: true,
}},
)
runTests(
t,
"ws:ws = [ \t\n]; A = A [a]; B = A",
[]testItem{{
title: "embedded naive left recursion expects infinite stream with whitespace",
text: "\na",
ignorePosition: true,
fail: true,
}},
)
} }
func TestSequence(t *testing.T) { func TestSequence(t *testing.T) {

View File

@ -20,9 +20,13 @@ syntax language supports recursive references, enabling the definition of contex
We can define syntaxes during development and use the provided tool to generate static Go code, which is then We can define syntaxes during development and use the provided tool to generate static Go code, which is then
built into the application. Alternatively, the library supports loading syntaxes dynamically at runtime. built into the application. Alternatively, the library supports loading syntaxes dynamically at runtime.
The parser engine handles recursive references and left-recursion internally. This way it makes it more
convenient writing intuitive grammar definitions, and allows defining context-free languages without complex
workarounds.
## Installation ## Installation
From source: From source (recommended):
``` ```
git clone https://code.squareroundforest.org/arpio/treerack git clone https://code.squareroundforest.org/arpio/treerack
@ -30,7 +34,7 @@ cd treerack
make install make install
``` ```
Alternatively: Alternatively ("best effort" basis):
``` ```
go install code.squareroundforest.org/arpio/treerack/cmd/treerack go install code.squareroundforest.org/arpio/treerack/cmd/treerack
@ -38,8 +42,8 @@ go install code.squareroundforest.org/arpio/treerack/cmd/treerack
## Documentation ## Documentation
- [Manual](docs/manual.md): A guide to the main use cases supported by Treerack. - [Manual](docs/manual.md): a guide to the main use cases supported by Treerack.
- [Syntax Definition](docs/syntax.md): Detailed reference for the Treerack definition language. - [Syntax Definition](docs/syntax.md): detailed reference for the Treerack definition language.
- [Library Documentation](https://godocs.io/code.squareroundforest.org/arpio/treerack): GoDoc reference for the - [Library Documentation](https://godocs.io/code.squareroundforest.org/arpio/treerack): GoDoc reference for the
runtime library. runtime library.
@ -47,10 +51,10 @@ go install code.squareroundforest.org/arpio/treerack/cmd/treerack
We use a Makefile to manage the build and verification lifecycle. We use a Makefile to manage the build and verification lifecycle.
Important: Generating the parser for the Treerack syntax itself (bootstrapping) requires multiple phases. Important: generating the parser for the Treerack syntax itself (bootstrapping) requires multiple phases.
Consequently, running standard go build or go test commands may miss subtle consistency problems. Consequently, running standard go build or go test commands may miss subtle consistency problems.
The authoritative way to verify changes is via the makefile: The decisive way to verify changes is via the makefile:
``` ```
make check make check
@ -60,6 +64,6 @@ make check
- Lexer & UTF-8: Treerack does not require a lexer, which simplifies the architecture. However, this enforces - Lexer & UTF-8: Treerack does not require a lexer, which simplifies the architecture. However, this enforces
the use of UTF-8 input. We have considered support for custom tokenizers as a potential future improvement. the use of UTF-8 input. We have considered support for custom tokenizers as a potential future improvement.
- Whitespace Delimited Languages: Due to the recursive descent nature and the lack of a dedicated lexer state, - Whitespace Delimited Languages: due to the recursive descent nature and the lack of a dedicated lexer state,
defining whitespace-delimited syntaxes (such as Python-style indentation) can be difficult to achieve with the defining whitespace-delimited syntaxes (such as Python-style indentation) can be difficult to achieve with the
current feature set. current feature set.