arpio/treerack
1
0
Fork 0
Code

error fixes and doc updates

Browse Source
This commit is contained in:
Arpad Ryszka 2026-01-21 20:54:16 +01:00
parent 4c6c817431
commit 2fe6f88ed6
21 changed files with 285 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -76,6 +76,7 @@ lib: $(sources) head.go headexported.go internal/self/self.go
cmd/treerack/docreflect.gen.go: $(sources) .build
go run scripts/docreflect.go > .build/docreflect.gen.go
go fmt .build/docreflect.gen.go
mv .build/docreflect.gen.go cmd/treerack
cmd/treerack/readme.md: $(sources) cmd/treerack/docreflect.gen.go

8
cmd/treerack/check.go
View File

@ -8,16 +8,16 @@ import (
type checkOptions struct {
// Syntax specifies the filename of the syntax definition file.
Syntax string
Syntax *string
// SyntaxString specifies the syntax as an inline string.
SyntaxString string
SyntaxString *string
// Input specifies the filename of the input content to be validated.
Input string
Input *string
// InputString specifies the input content as an inline string.
InputString string
InputString *string
}
// check parses input content against the provided syntax definition and fails if the input does not match.

62
cmd/treerack/check_test.go
View File

@ -8,9 +8,13 @@ import (
"testing"
)
func ptrto[T any](v T) *T {
return &v
}
func TestCheck(t *testing.T) {
t.Run("no syntax", func(t *testing.T) {
o := checkOptions{Input: "bar_test.txt"}
o := checkOptions{Input: ptrto("bar_test.txt")}
if err := check(o, nil); !errors.Is(err, errNoInput) {
t.Fatal()
}
@ -18,9 +22,9 @@ func TestCheck(t *testing.T) {
t.Run("too many syntaxes", func(t *testing.T) {
o := checkOptions{
Syntax: "foo_test.treerack",
SyntaxString: `foo = "baz"`,
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
SyntaxString: ptrto(`foo = "baz"`),
Input: ptrto("bar_test.txt"),
}
if err := check(o, nil); !errors.Is(err, errMultipleInputs) {
@ -30,8 +34,8 @@ func TestCheck(t *testing.T) {
t.Run("syntax file not found", func(t *testing.T) {
o := checkOptions{
Syntax: "no-file.treerack",
Input: "bar_test.txt",
Syntax: ptrto("no-file.treerack"),
Input: ptrto("bar_test.txt"),
}
if err := check(o, nil); !os.IsNotExist(err) {
@ -41,8 +45,8 @@ func TestCheck(t *testing.T) {
t.Run("invalid syntax definition", func(t *testing.T) {
o := checkOptions{
SyntaxString: `foo`,
Input: "bar_test.txt",
SyntaxString: ptrto(`foo`),
Input: ptrto("bar_test.txt"),
}
var perr *treerack.ParseError
@ -53,8 +57,8 @@ func TestCheck(t *testing.T) {
t.Run("invalid syntax init", func(t *testing.T) {
o := checkOptions{
SyntaxString: `foo = "bar"; foo = "baz"`,
Input: "bar_test.txt",
SyntaxString: ptrto(`foo = "bar"; foo = "baz"`),
Input: ptrto("bar_test.txt"),
}
if err := check(o, nil); err == nil {
@ -63,7 +67,7 @@ func TestCheck(t *testing.T) {
})
t.Run("no input", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"}
o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil); !errors.Is(err, errNoInput) {
t.Fatal()
@ -72,8 +76,8 @@ func TestCheck(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) {
o := checkOptions{
Syntax: "foo_test.treerack",
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
Input: ptrto("bar_test.txt"),
}
if err := check(o, nil, "baz_test.txt"); !errors.Is(err, errMultipleInputs) {
@ -82,14 +86,14 @@ func TestCheck(t *testing.T) {
})
t.Run("empty filename for input", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"}
o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil, ""); !errors.Is(err, errInvalidFilename) {
t.Fatal()
}
})
t.Run("input file not found", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"}
o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil, "baz_test.txt"); !os.IsNotExist(err) {
t.Fatal()
}
@ -97,8 +101,8 @@ func TestCheck(t *testing.T) {
t.Run("input parse fail", func(t *testing.T) {
o := checkOptions{
Syntax: "foo_test.treerack",
InputString: "baz",
Syntax: ptrto("foo_test.treerack"),
InputString: ptrto("baz"),
}
var perr *treerack.ParseError
@ -109,8 +113,8 @@ func TestCheck(t *testing.T) {
t.Run("input parse success", func(t *testing.T) {
o := checkOptions{
Syntax: "foo_test.treerack",
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
Input: ptrto("bar_test.txt"),
}
if err := check(o, nil); err != nil {
@ -120,8 +124,8 @@ func TestCheck(t *testing.T) {
t.Run("input from string success", func(t *testing.T) {
o := checkOptions{
Syntax: "foo_test.treerack",
InputString: "bar",
Syntax: ptrto("foo_test.treerack"),
InputString: ptrto("bar"),
}
if err := check(o, nil); err != nil {
@ -130,17 +134,29 @@ func TestCheck(t *testing.T) {
})
t.Run("input from file success", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"}
o := checkOptions{Syntax: ptrto("foo_test.treerack")}
if err := check(o, nil, "bar_test.txt"); err != nil {
t.Fatal(err)
}
})
t.Run("input from stdin success", func(t *testing.T) {
o := checkOptions{Syntax: "foo_test.treerack"}
o := checkOptions{Syntax: ptrto("foo_test.treerack")}
buf := bytes.NewBufferString("bar")
if err := check(o, buf); err != nil {
t.Fatal(err)
}
})
t.Run("empty input filename", func(t *testing.T) {
o := checkOptions{
Syntax: ptrto("foo_test.treerack"),
Input: ptrto(""),
}
var buf bytes.Buffer
if err := check(o, &buf); !errors.Is(err, errInvalidFilename) {
t.Fatal()
}
})
}

4
cmd/treerack/checksyntax.go
View File

@ -8,10 +8,10 @@ import (
type checkSyntaxOptions struct {
// Syntax specifies the filename of the syntax definition file.
Syntax string
Syntax *string
// SyntaxString specifies the syntax as an inline string.
SyntaxString string
SyntaxString *string
}
// checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a

10
cmd/treerack/checksyntax_test.go
View File

@ -10,7 +10,7 @@ import (
func TestCheckSyntax(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) {
o := checkSyntaxOptions{Syntax: "foo_test.treerack", SyntaxString: `foo = "42"`}
o := checkSyntaxOptions{Syntax: ptrto("foo_test.treerack"), SyntaxString: ptrto(`foo = "42"`)}
if err := checkSyntax(o, nil); !errors.Is(err, errMultipleInputs) {
t.Fatal()
}
@ -32,28 +32,28 @@ func TestCheckSyntax(t *testing.T) {
t.Run("invalid syntax", func(t *testing.T) {
var perr *treerack.ParseError
o := checkSyntaxOptions{SyntaxString: "foo"}
o := checkSyntaxOptions{SyntaxString: ptrto("foo")}
if err := checkSyntax(o, nil); !errors.As(err, &perr) {
t.Fatal()
}
})
t.Run("invalid syntax init", func(t *testing.T) {
o := checkSyntaxOptions{SyntaxString: `foo = "42"; foo = "84"`}
o := checkSyntaxOptions{SyntaxString: ptrto(`foo = "42"; foo = "84"`)}
if err := checkSyntax(o, nil); err == nil {
t.Fatal()
}
})
t.Run("success", func(t *testing.T) {
o := checkSyntaxOptions{Syntax: "foo_test.treerack"}
o := checkSyntaxOptions{Syntax: ptrto("foo_test.treerack")}
if err := checkSyntax(o, nil); err != nil {
t.Fatal(err)
}
})
t.Run("from string success", func(t *testing.T) {
o := checkSyntaxOptions{SyntaxString: `foo = "bar"`}
o := checkSyntaxOptions{SyntaxString: ptrto(`foo = "bar"`)}
if err := checkSyntax(o, nil); err != nil {
t.Fatal(err)
}

83
cmd/treerack/docreflect.gen.go
View File

@ -2,48 +2,49 @@
Generated with https://code.squareroundforest.org/arpio/docreflect
*/
package main
import "code.squareroundforest.org/arpio/docreflect"
func init() {
docreflect.Register("main", "")
docreflect.Register("main.check", "check parses input content against the provided syntax definition and fails if the input does not match.\nSyntax can be provided via a filename option or an inline string option. Input can be provided via a filename\noption, a positional argument filename, an inline string option, or piped from standard input.\n\nfunc(o, stdin, args)")
docreflect.Register("main.checkOptions", "")
docreflect.Register("main.checkOptions.Input", "Input specifies the filename of the input content to be validated.\n")
docreflect.Register("main.checkOptions.InputString", "InputString specifies the input content as an inline string.\n")
docreflect.Register("main.checkOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.checkOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.checkSyntax", "checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a\npositional argument), an inline string, or piped from standard input.\n\nfunc(o, stdin, args)")
docreflect.Register("main.checkSyntaxOptions", "")
docreflect.Register("main.checkSyntaxOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.checkSyntaxOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.errInvalidFilename", "")
docreflect.Register("main.errMultipleInputs", "")
docreflect.Register("main.errNoInput", "")
docreflect.Register("main.generate", "generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded\nin an application.\n\nThe syntax may be provided via a file path (using an option or a positional argument), an\ninline string, or piped from standard input.\n\nfunc(o, stdin, stdout, args)")
docreflect.Register("main.generateOptions", "")
docreflect.Register("main.generateOptions.Export", "Export determines whether the generated parse function is exported (visible outside its package).\n")
docreflect.Register("main.generateOptions.PackageName", "PackageName specifies the package name for the generated code. Defaults to main.\n")
docreflect.Register("main.generateOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.generateOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.init", "\nfunc()")
docreflect.Register("main.initInput", "\nfunc(filename, stringValue, stdin, args)")
docreflect.Register("main.main", "\nfunc()")
docreflect.Register("main.mapNode", "\nfunc(n)")
docreflect.Register("main.node", "")
docreflect.Register("main.node.From", "")
docreflect.Register("main.node.Name", "")
docreflect.Register("main.node.Nodes", "")
docreflect.Register("main.node.Text", "")
docreflect.Register("main.node.To", "")
docreflect.Register("main.noop", "\nfunc()")
docreflect.Register("main.show", "show input content against a provided syntax definition and outputs the resulting AST (Abstract Syntax Tree)\nin JSON format. Syntax can be provided via a filename option or an inline string option. Input can be\nprovided via a filename option, a positional argument filename, an inline string option, or piped from\nstandard input.\n\nfunc(o, stdin, stdout, args)")
docreflect.Register("main.showOptions", "")
docreflect.Register("main.showOptions.Indent", "Indent specifies a custom indentation string for the output.\n")
docreflect.Register("main.showOptions.Input", "Input specifies the filename of the input content to be validated.\n")
docreflect.Register("main.showOptions.InputString", "InputString specifies the input content as an inline string.\n")
docreflect.Register("main.showOptions.Pretty", "Pretty enables indented, human-readable output.\n")
docreflect.Register("main.showOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.showOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.version", "")
docreflect.Register("main", "")
docreflect.Register("main.check", "check parses input content against the provided syntax definition and fails if the input does not match.\nSyntax can be provided via a filename option or an inline string option. Input can be provided via a filename\noption, a positional argument filename, an inline string option, or piped from standard input.\n\nfunc(o, stdin, args)")
docreflect.Register("main.checkOptions", "")
docreflect.Register("main.checkOptions.Input", "Input specifies the filename of the input content to be validated.\n")
docreflect.Register("main.checkOptions.InputString", "InputString specifies the input content as an inline string.\n")
docreflect.Register("main.checkOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.checkOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.checkSyntax", "checkSyntax validates a syntax definition. The syntax may be provided via a file path (using an option or a\npositional argument), an inline string, or piped from standard input.\n\nfunc(o, stdin, args)")
docreflect.Register("main.checkSyntaxOptions", "")
docreflect.Register("main.checkSyntaxOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.checkSyntaxOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.errInvalidFilename", "")
docreflect.Register("main.errMultipleInputs", "")
docreflect.Register("main.errNoInput", "")
docreflect.Register("main.generate", "generate generates Go code that can parse arbitrary input with the provided syntax, and can be used embedded\nin an application.\n\nThe syntax may be provided via a file path (using an option or a positional argument), an\ninline string, or piped from standard input.\n\nfunc(o, stdin, stdout, args)")
docreflect.Register("main.generateOptions", "")
docreflect.Register("main.generateOptions.Export", "Export determines whether the generated parse function is exported (visible outside its package).\n")
docreflect.Register("main.generateOptions.PackageName", "PackageName specifies the package name for the generated code. Defaults to main.\n")
docreflect.Register("main.generateOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.generateOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.init", "\nfunc()")
docreflect.Register("main.initInput", "\nfunc(filename, stringValue, stdin, args)")
docreflect.Register("main.main", "\nfunc()")
docreflect.Register("main.mapNode", "\nfunc(n)")
docreflect.Register("main.node", "")
docreflect.Register("main.node.From", "")
docreflect.Register("main.node.Name", "")
docreflect.Register("main.node.Nodes", "")
docreflect.Register("main.node.Text", "")
docreflect.Register("main.node.To", "")
docreflect.Register("main.noop", "\nfunc()")
docreflect.Register("main.show", "show input content against a provided syntax definition and outputs the resulting AST (Abstract Syntax Tree)\nin JSON format. Syntax can be provided via a filename option or an inline string option. Input can be\nprovided via a filename option, a positional argument filename, an inline string option, or piped from\nstandard input.\n\nfunc(o, stdin, stdout, args)")
docreflect.Register("main.showOptions", "")
docreflect.Register("main.showOptions.Indent", "Indent specifies a custom indentation string for the output.\n")
docreflect.Register("main.showOptions.Input", "Input specifies the filename of the input content to be validated.\n")
docreflect.Register("main.showOptions.InputString", "InputString specifies the input content as an inline string.\n")
docreflect.Register("main.showOptions.Pretty", "Pretty enables indented, human-readable output.\n")
docreflect.Register("main.showOptions.Syntax", "Syntax specifies the filename of the syntax definition file.\n")
docreflect.Register("main.showOptions.SyntaxString", "SyntaxString specifies the syntax as an inline string.\n")
docreflect.Register("main.version", "")
}

4
cmd/treerack/generate.go
View File

@ -8,10 +8,10 @@ import (
type generateOptions struct {
// Syntax specifies the filename of the syntax definition file.
Syntax string
Syntax *string
// SyntaxString specifies the syntax as an inline string.
SyntaxString string
SyntaxString *string
// PackageName specifies the package name for the generated code. Defaults to main.
PackageName string

14
cmd/treerack/generate_test.go
View File

@ -12,7 +12,7 @@ import (
func TestGenerate(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) {
var out bytes.Buffer
o := generateOptions{Syntax: "foo_test.treerack", SyntaxString: `foo = "42"`}
o := generateOptions{Syntax: ptrto("foo_test.treerack"), SyntaxString: ptrto(`foo = "42"`)}
if err := generate(o, nil, &out); !errors.Is(err, errMultipleInputs) {
t.Fatal()
}
@ -46,7 +46,7 @@ func TestGenerate(t *testing.T) {
perr *treerack.ParseError
)
o := generateOptions{SyntaxString: "foo"}
o := generateOptions{SyntaxString: ptrto("foo")}
if err := generate(o, nil, &out); !errors.As(err, &perr) {
t.Fatal()
}
@ -54,7 +54,7 @@ func TestGenerate(t *testing.T) {
t.Run("invalid syntax init", func(t *testing.T) {
var out bytes.Buffer
o := generateOptions{SyntaxString: `foo = "42"; foo = "84"`}
o := generateOptions{SyntaxString: ptrto(`foo = "42"; foo = "84"`)}
if err := generate(o, nil, &out); err == nil {
t.Fatal()
}
@ -62,7 +62,7 @@ func TestGenerate(t *testing.T) {
t.Run("success", func(t *testing.T) {
var out bytes.Buffer
o := generateOptions{Syntax: "foo_test.treerack"}
o := generateOptions{Syntax: ptrto("foo_test.treerack")}
if err := generate(o, nil, &out); err != nil {
t.Fatal(err)
}
@ -75,7 +75,7 @@ func TestGenerate(t *testing.T) {
t.Run("success string", func(t *testing.T) {
var out bytes.Buffer
o := generateOptions{SyntaxString: `foo = "bar"`}
o := generateOptions{SyntaxString: ptrto(`foo = "bar"`)}
if err := generate(o, nil, &out); err != nil {
t.Fatal(err)
}
@ -122,7 +122,7 @@ func TestGenerate(t *testing.T) {
t.Run("custom package name", func(t *testing.T) {
var out bytes.Buffer
o := generateOptions{
Syntax: "foo_test.treerack",
Syntax: ptrto("foo_test.treerack"),
PackageName: "foo",
}
@ -139,7 +139,7 @@ func TestGenerate(t *testing.T) {
t.Run("export", func(t *testing.T) {
var out bytes.Buffer
o := generateOptions{
Syntax: "foo_test.treerack",
Syntax: ptrto("foo_test.treerack"),
Export: true,
}

31
cmd/treerack/input.go
View File

@ -17,16 +17,18 @@ var (
func noop() {}
func initInput(
filename, stringValue string, stdin io.Reader, args []string,
) (input io.Reader, finalize func(), err error) {
filename, stringValue *string, stdin io.Reader, args []string,
) (
input io.Reader, finalize func(), err error,
) {
finalize = noop
var inputCount int
if filename != "" {
if filename != nil {
inputCount++
}
if stringValue != "" {
if stringValue != nil {
inputCount++
}
@ -39,19 +41,20 @@ func initInput(
return
}
if len(args) > 0 && args[0] == "" {
if len(args) > 0 {
filename = new(string)
*filename = args[0]
}
switch {
case filename != nil:
if *filename == "" {
err = errInvalidFilename
return
}
if len(args) > 0 {
filename = args[0]
}
switch {
case filename != "":
var f io.ReadCloser
f, err = os.Open(filename)
f, err = os.Open(*filename)
if err != nil {
return
}
@ -63,8 +66,8 @@ func initInput(
}
input = f
case stringValue != "":
input = bytes.NewBufferString(stringValue)
case stringValue != nil:
input = bytes.NewBufferString(*stringValue)
default:
if stdin == nil {
err = errNoInput

8
cmd/treerack/show.go
View File

@ -9,16 +9,16 @@ import (
type showOptions struct {
// Syntax specifies the filename of the syntax definition file.
Syntax string
Syntax *string
// SyntaxString specifies the syntax as an inline string.
SyntaxString string
SyntaxString *string
// Input specifies the filename of the input content to be validated.
Input string
Input *string
// InputString specifies the input content as an inline string.
InputString string
InputString *string
// Pretty enables indented, human-readable output.
Pretty bool

60
cmd/treerack/show_test.go
View File

@ -11,7 +11,7 @@ import (
func TestShow(t *testing.T) {
t.Run("no syntax", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{Input: "bar_test.txt"}
o := showOptions{Input: ptrto("bar_test.txt")}
if err := show(o, nil, &out); !errors.Is(err, errNoInput) {
t.Fatal()
}
@ -20,9 +20,9 @@ func TestShow(t *testing.T) {
t.Run("too many syntaxes", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
SyntaxString: `foo = "baz"`,
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
SyntaxString: ptrto(`foo = "baz"`),
Input: ptrto("bar_test.txt"),
}
if err := show(o, nil, &out); !errors.Is(err, errMultipleInputs) {
@ -33,8 +33,8 @@ func TestShow(t *testing.T) {
t.Run("syntax file not found", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "no-file.treerack",
Input: "bar_test.txt",
Syntax: ptrto("no-file.treerack"),
Input: ptrto("bar_test.txt"),
}
if err := show(o, nil, &out); !os.IsNotExist(err) {
@ -45,8 +45,8 @@ func TestShow(t *testing.T) {
t.Run("invalid syntax definition", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
SyntaxString: `foo`,
Input: "bar_test.txt",
SyntaxString: ptrto(`foo`),
Input: ptrto("bar_test.txt"),
}
var perr *treerack.ParseError
@ -58,8 +58,8 @@ func TestShow(t *testing.T) {
t.Run("invalid syntax init", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
SyntaxString: `foo = "bar"; foo = "baz"`,
Input: "bar_test.txt",
SyntaxString: ptrto(`foo = "bar"; foo = "baz"`),
Input: ptrto("bar_test.txt"),
}
if err := show(o, nil, &out); err == nil {
@ -69,7 +69,7 @@ func TestShow(t *testing.T) {
t.Run("no input", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"}
o := showOptions{Syntax: ptrto("foo_test.treerack")}
if err := show(o, nil, &out); !errors.Is(err, errNoInput) {
t.Fatal()
@ -79,8 +79,8 @@ func TestShow(t *testing.T) {
t.Run("too many inputs", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
Input: ptrto("bar_test.txt"),
}
if err := show(o, nil, &out, "baz_test.txt"); !errors.Is(err, errMultipleInputs) {
@ -90,7 +90,7 @@ func TestShow(t *testing.T) {
t.Run("empty filename for input", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"}
o := showOptions{Syntax: ptrto("foo_test.treerack")}
if err := show(o, nil, &out, ""); !errors.Is(err, errInvalidFilename) {
t.Fatal()
}
@ -98,7 +98,7 @@ func TestShow(t *testing.T) {
t.Run("input file not found", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"}
o := showOptions{Syntax: ptrto("foo_test.treerack")}
if err := show(o, nil, &out, "baz_test.txt"); !os.IsNotExist(err) {
t.Fatal()
}
@ -107,8 +107,8 @@ func TestShow(t *testing.T) {
t.Run("input parse fail", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
InputString: "baz",
Syntax: ptrto("foo_test.treerack"),
InputString: ptrto("baz"),
}
var perr *treerack.ParseError
@ -120,8 +120,8 @@ func TestShow(t *testing.T) {
t.Run("show", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
Input: ptrto("bar_test.txt"),
}
if err := show(o, nil, &out); err != nil {
@ -136,8 +136,8 @@ func TestShow(t *testing.T) {
t.Run("show string", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
InputString: "bar",
Syntax: ptrto("foo_test.treerack"),
InputString: ptrto("bar"),
}
if err := show(o, nil, &out); err != nil {
@ -151,9 +151,7 @@ func TestShow(t *testing.T) {
t.Run("show file", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
}
o := showOptions{Syntax: ptrto("foo_test.treerack")}
if err := show(o, nil, &out, "bar_test.txt"); err != nil {
t.Fatal(nil)
@ -166,7 +164,7 @@ func TestShow(t *testing.T) {
t.Run("show stdin", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{Syntax: "foo_test.treerack"}
o := showOptions{Syntax: ptrto("foo_test.treerack")}
in := bytes.NewBufferString("bar")
if err := show(o, in, &out); err != nil {
t.Fatal(nil)
@ -180,8 +178,8 @@ func TestShow(t *testing.T) {
t.Run("indent", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
Input: ptrto("bar_test.txt"),
Pretty: true,
}
@ -198,8 +196,8 @@ func TestShow(t *testing.T) {
t.Run("custom indent", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
Input: ptrto("bar_test.txt"),
Indent: "xx",
}
@ -215,8 +213,8 @@ func TestShow(t *testing.T) {
t.Run("redundant custom indent", func(t *testing.T) {
var out bytes.Buffer
o := showOptions{
Syntax: "foo_test.treerack",
Input: "bar_test.txt",
Syntax: ptrto("foo_test.treerack"),
Input: ptrto("bar_test.txt"),
Pretty: true,
Indent: "xx",
}

5
context.go
View File

@ -120,6 +120,10 @@ func (c *context) fail(offset int) {
}
func findLine(tokens []rune, offset int) (line, column int) {
if offset < 0 {
return 0, 0
}
tokens = tokens[:offset]
for i := range tokens {
column++
@ -144,7 +148,6 @@ func (c *context) parseError(p parser) error {
}
line, col := findLine(c.tokens, c.failOffset)
return &ParseError{
Offset: c.failOffset,
Line: line,

43
docs/examples/acalc/main.go
View File

@ -14,54 +14,48 @@ import (
var errExit = errors.New("exit")
// repl runs the Read-Eval-Print Loop.
func repl(input io.Reader, output io.Writer) {
// use buffered io, to be able to read the input line-by-line:
// use buffered io, to read the input line-by-line:
buf := bufio.NewReader(os.Stdin)
// our REPL loop:
// our REPL:
for {
// print a basic prompt:
// print a input prompt marker:
if _, err := output.Write([]byte("> ")); err != nil {
// we cannot fix it if there is an error here:
log.Fatalln(err)
}
// read the input and handle the errors:
expr, err := read(buf)
// when EOF, that means the user pressed Ctrl+D. Let's terminate the output with a conventional newline
// and exit:
// handle EOF (Ctrl+D):
if errors.Is(err, io.EOF) {
output.Write([]byte{'\n'})
os.Exit(0)
}
// when errExit, that means the user entered exit:
// handle the explicit exit command:
if errors.Is(err, errExit) {
os.Exit(0)
}
// if it's a parser error, we print and continue from reading again, to allow the user to fix the
// problem:
// handle parser errors (allow the user to retry):
var perr *parseError
if errors.As(err, &perr) {
log.Println(err)
continue
}
// in case of any other error, we don't know what's going on, so we get out of here right away:
// handle possible I/O errors:
if err != nil {
log.Fatalln(err)
}
// if we received an expression, then we can evaluate it. We are not expecting errors here:
// evaluate and print:
result := eval(expr)
// we have the result, we need to print it:
if err := print(output, result); err != nil {
// if printing fails, we don't know how to fix it, so we get out of here:
log.Fatalln(err)
}
}
@ -73,7 +67,7 @@ func read(input *bufio.Reader) (*node, error) {
return nil, err
}
// expr will be of type *node, which type is defined in the generated code
// parse the line using the generated parser:
expr, err := parse(bytes.NewBufferString(line))
if err != nil {
return nil, err
@ -83,15 +77,12 @@ func read(input *bufio.Reader) (*node, error) {
return nil, errExit
}
// we know based on the syntax, that the top level node will always have a single child, either a number
// literal or a binary operation:
// based on our syntax, the root node always has exactly one child: either a number or a binary operation.
return expr.Nodes[0], nil
}
// eval always returns the calculated result as a float64:
func eval(expr *node) float64 {
// we know that it's either a number or a binary operation:
var value float64
switch expr.Name {
case "num":
@ -103,10 +94,8 @@ func eval(expr *node) float64 {
return value
default:
// we know that the first node is either a number of a child expression:
// evaluate binary expressions. Format: Operand [Operator Operand]...
value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:]
// we don't need to track back, so we can drop the processed nodes while consuming them:
for len(expr.Nodes) > 0 {
var (
operator string
@ -122,8 +111,7 @@ func eval(expr *node) float64 {
case "mul":
value *= operand
case "div":
// Go returns -Inf or +Inf on division by zero:
value /= operand
value /= operand // Go returns on division by zero +/-Inf
}
}
}
@ -132,12 +120,13 @@ func eval(expr *node) float64 {
}
func print(output io.Writer, result float64) error {
// we can use the stdlib fmt package to print float64:
_, err := fmt.Fprintln(output, result)
return err
}
func main() {
// for testability, we define the REPL loop in a separate function so that the test code can call it with
// for testability, we define the REPL in a separate function so that the test code can call it with
// in-memory buffers as input and output. Our main function calls it with the stdio handles:
repl(os.Stdin, os.Stdout)
}

60
docs/manual.md
View File

@ -24,7 +24,7 @@ Alternatively, we _may be able to_ install directly using the Go toolchain:
## Hello syntax
A basic syntax definition looks like this:
A trivial syntax definition looks like this:
```
hello = "Hello, world!"
@ -83,10 +83,10 @@ If our syntax definition is invalid, check-syntax will fail:
treerack check-syntax --syntax-string 'foo = bar'
```
The above command will fail because the parser called foo references an undefined parser bar.
The above command will fail because the parser called `foo` references an undefined parser `bar`.
We can use check or show to detect when the input content does not match a valid syntax. Using the hello syntax,
we can try the following:
We can use `check` or `show` to detect when the input content does not match a valid syntax. Using the hello
syntax, we can try the following:
```
treerack check --syntax-string 'hello = "Hello, world!"' --input-string 'Hi!'
@ -96,9 +96,9 @@ It will show that parsing the input failed and that it failed while using the pa
## Basic syntax - An arithmetic calculator
In this section, we will build a basic arithmetic calculator. It will read a line from standard input, parse it
as an arithmetic expression, compute the result, and print it—effectively creating a REPL (Read-Eval-Print
Loop).
In this section, we will build a simplistic arithmetic calculator. It will read a line from standard input,
parse it as an arithmetic expression, compute the result, print it, and start over - effectively creating a REPL
(Read-Eval-Print Loop).
We will support addition +, subtraction -, multiplication *, division /, and grouping with parentheses ().
@ -133,10 +133,10 @@ group:alias = "(" expression ")";
//
// We group operators by precedence levels to ensure correct order of operations.
//
// Level 0 (High): Multiplication/Division
// Level 0 (high): multiplication/division
op0:alias = mul | div;
// Level 1 (Low): Addition/Subtraction
// Level 1 (low): addition/subtraction
op1:alias = add | sub;
// Operands for each precedence level.
@ -401,12 +401,12 @@ var errExit = errors.New("exit")
// repl runs the Read-Eval-Print Loop.
func repl(input io.Reader, output io.Writer) {
// use buffered io, to be able to read the input line-by-line:
// use buffered io, to read the input line-by-line:
buf := bufio.NewReader(os.Stdin)
// our REPL loop:
// our REPL:
for {
// print a basic prompt:
// print a input prompt marker:
if _, err := output.Write([]byte("> ")); err != nil {
log.Fatalln(err)
}
@ -414,29 +414,30 @@ func repl(input io.Reader, output io.Writer) {
// read the input and handle the errors:
expr, err := read(buf)
// Handle EOF (Ctrl+D)
// handle EOF (Ctrl+D):
if errors.Is(err, io.EOF) {
output.Write([]byte{'\n'})
os.Exit(0)
}
// Handle explicit exit command
// handle the explicit exit command:
if errors.Is(err, errExit) {
os.Exit(0)
}
// Handle parser errors (allow user to retry)
// handle parser errors (allow the user to retry):
var perr *parseError
if errors.As(err, &perr) {
log.Println(err)
continue
}
// handle possible I/O errors:
if err != nil {
log.Fatalln(err)
}
// Evaluate and print
// evaluate and print:
result := eval(expr)
if err := print(output, result); err != nil {
log.Fatalln(err)
@ -450,7 +451,7 @@ func read(input *bufio.Reader) (*node, error) {
return nil, err
}
// Parse the line using the generated parser
// parse the line using the generated parser:
expr, err := parse(bytes.NewBufferString(line))
if err != nil {
return nil, err
@ -460,8 +461,7 @@ func read(input *bufio.Reader) (*node, error) {
return nil, errExit
}
// Based on our syntax, the root node always has exactly one child:
// either a number or a binary operation.
// based on our syntax, the root node always has exactly one child: either a number or a binary operation.
return expr.Nodes[0], nil
}
@ -478,8 +478,7 @@ func eval(expr *node) float64 {
return value
default:
// Handle binary expressions (recursively)
// Format: Operand [Operator Operand]...
// handle binary expressions. Format: Operand [Operator Operand]...
value, expr.Nodes = eval(expr.Nodes[0]), expr.Nodes[1:]
for len(expr.Nodes) > 0 {
var (
@ -496,7 +495,7 @@ func eval(expr *node) float64 {
case "mul":
value *= operand
case "div":
value /= operand // Go handles division by zero as ±Inf
value /= operand // Go handles division by zero as +/-Inf
}
}
}
@ -505,12 +504,13 @@ func eval(expr *node) float64 {
}
func print(output io.Writer, result float64) error {
// we can use
_, err := fmt.Fprintln(output, result)
return err
}
func main() {
// for testability, we define the REPL loop in a separate function so that the test code can call it with
// for testability, we define the REPL in a separate function so that the test code can call it with
// in-memory buffers as input and output. Our main function calls it with the stdio handles:
repl(os.Stdin, os.Stdout)
}
@ -533,13 +533,13 @@ $ go run .
We can find the source files for this example here: [./examples/acalc](./examples/acalc).
## Important Note: Unescaping
## Important note: unescaping
Treerack does not automatically handle escape sequences (e.g., converting \n to a literal newline). If our
syntax supports escaped characters—common in string literals—the user code is responsible for "unescaping" the
raw text from the AST node.
Treerack does not automatically handle escape sequences (e.g., converting `\n` to a literal newline). If our
syntax supports escaped characters - common in string literals - the user code is responsible for "unescaping"
the raw text from the AST node.
This is analogous to how we needed to parse the numbers in the calculator example to convert the string
This is analogous to how we needed to interpret the numbers in the calculator example to convert the string
representation of a number into a Go float64.
## Programmatically loading syntaxes
@ -569,7 +569,7 @@ func initAndParse(syntax, content io.Reader) (*treerack.Node, error) {
}
```
Caution: Be mindful of security implications when loading syntax definitions from untrusted sources.
Caution: be mindful of security implications when loading syntax definitions from untrusted sources.
## Programmatically defining syntaxes
@ -611,7 +611,7 @@ func initAndParse(content io.Reader) (*treerack.Node, error) {
## Summary
We have demonstrated how to use the Treerack tool to define, test, and implement a parser. We recommend the
We have demonstrated how to use the treerack tool to define, test, and implement a parser. We recommend the
following workflow:
1. draft: define a syntax in a .treerack file.

64
docs/syntax.md
View File

@ -5,14 +5,14 @@ It allows for the concise definition of recursive descent parsers.
A syntax file consists of a series of Production Rules (definitions), terminated by semicolons.
## Production Rules
## Production rules
A rule assigns a name to a pattern expression. Rules may include optional flags to modify the parser's behavior
or the resulting AST (Abstract Syntax Tree).
```
RuleName = Expression;
RuleName:flag1:flag2 = Expression;
rule-name = expression;
rule-name:flag1:flag2 = expression;
```
## Flags
@ -20,18 +20,19 @@ RuleName:flag1:flag2 = Expression;
Flags are appended to the rule name, separated by colons. They control AST generation, whitespace handling, and
error propagation.
- `alias`: Transparent Node. The rule validates input but does not create its own node in the AST. Children
- `alias`: transparent node. The rule validates input but does not create its own node in the AST. Children
nodes (if any) are attached to the parent of this rule.
- `ws`: Global Whitespace. Marks this rule as the designated whitespace handler. The parser will attempt to
- `ws`: global whitespace. Marks this rule as the designated whitespace handler. The parser will attempt to
match (and discard) this rule between tokens throughout the entire syntax.
- `nows`: No Whitespace. Disables automatic whitespace skipping inside this rule. Useful for defining tokens
like string literals where spaces are significant.
- `root`: Entry Point. Explicitly marks the rule as the starting point of the syntax. If omitted, the last
- `nows`: no whitespace. Disables automatic whitespace skipping inside this rule. Useful for defining tokens
like string literals where spaces are significant. The flag `nows` is automatically applied to char sequences
like `"abc" or [abc]+.
- `root`: entry point. Explicitly marks the rule as the starting point of the syntax. If omitted, the last
defined rule is implied to be the root.
- `kw`: Keyword. Marks the content as a reserved keyword.
- `nokw`: No Keyword. Prevents the rule from matching text that matches a defined kw rule. Essential for
- `kw`: keyword. Marks the content as a reserved keyword.
- `nokw`: no keyword. Prevents the rule from matching text that matches a defined kw rule. Essential for
distinguishing identifiers from keywords (e.g., ensuring var is not parsed as a variable name).
- `failpass`: Pass Failure. If this rule fails to parse, the error is reported as a failure of the parent rule,
- `failpass`: pass failure. If this rule fails to parse, the error is reported as a failure of the parent rule,
not this specific rule.
## Expressions
@ -43,7 +44,7 @@ and quantifiers.
Terminals match specific characters or strings in the input.
- `"abc"` (string): Matches an exact sequence of characters.
- `"abc"` (string): Matches an exact sequence of characters. Equivalent to [a][b][c].
- `.` (any char): Matches any single character (wildcard).
- `[123]`, `[a-z]`, `[123a-z]` (class): Matches a single character from a set or range.
- `[^123]`, `[^a-z]`, `[^123a-z]` (not class) Matches any single character not in the set.
@ -52,13 +53,13 @@ Terminals match specific characters or strings in the input.
Quantifiers determine how many times an item must match. They are placed immediately after the item they modify.
- `?`: Optional (Zero or one).
- `*`: Zero or more.
- `+`: One or more.
- `{n}`: Exact count. Matches exactly n times.
- `{n,}`: At least. Matches n or more times.
- `{,m}`: At most. Matches between 0 and m times.
- `{n,m}`: Range. Matches between n and m times.
- `?`: optional (zero or one).
- `*`: zero or more.
- `+`: one or more.
- `{n}`: exact count. Matches exactly n times.
- `{n,}`: at least. Matches n or more times.
- `{,m}`: at most. Matches between 0 and m times.
- `{n,m}`: range. Matches between n and m times.
## Composites
@ -69,8 +70,8 @@ Complex patterns are built by combining terminals and other rules.
Items written consecutively are matched in order.
```
// Matches "A", then "B", then "C"
MySequence = "A" "B" "C";
// matches "A", then "B", then "C":
my-sequence = "A" "B" "C";
```
### 2. Grouping
@ -78,8 +79,8 @@ MySequence = "A" "B" "C";
Parentheses (...) group items together, allowing quantifiers to apply to the entire group.
```
// Matches "AB", "ABAB", "ABABAB"...
MyGroup = ("A" "B")+;
// matches "AB", "ABAB", "ABABAB"...:
my-group = ("A" "B")+;
```
### 3. Choices
@ -89,21 +90,18 @@ The pipe | character represents a choice between alternatives.
The parser evaluates all provided options against the input at the current position and selects the best match
based on the following priority rules:
1. _Longest Match_: The option that consumes the largest number of characters takes priority. This eliminates the
1. _longest match_: the option that consumes the largest number of characters takes priority. This eliminates the
need to manually order specific matches before general ones (e.g., "integer" will always be chosen over "int" if
the input supports it, regardless of their order in the definition).
2. _First Definition Wins_: If multiple options consume the exact same number of characters, the option defined
2. _first definition wins_: if multiple options consume the exact same number of characters, the option defined
first(left-most) in the list takes priority.
```
// Longest match wins automatically:
// Input "integer" is matched by 'type', even though "int" comes first.
// longest match wins automatically: input "integer" is matched by 'type', even though "int" comes first.
type = "int" | "integer";
// Tie-breaker rule:
// If input is "foo", both options match 3 characters.
// Because 'identifier' is last, it takes priority over 'keyword'.
// (Use :kw and :nokw to control such situations, when it applies.)
// Tie-breaker rule: if input is "foo", both options match 3 characters. Because 'identifier' is last, it takes
// priority over 'keyword'. (Use :kw and :nokw to control such situations, when it applies.)
content = keyword | identifier;
```
@ -111,8 +109,8 @@ content = keyword | identifier;
Comments follow C-style syntax and are ignored by the definition parser.
- Line comments: Start with // and end at the newline.
- Block comments: Enclosed in /* ... */.
- line comments: start with // and end at the newline.
- block comments: enclosed in /* ... */.
## Examples

2
head.go
View File

File diff suppressed because one or more lines are too long

2
headexported.go
View File

File diff suppressed because one or more lines are too long

3
internal/self/self.go
View File

@ -687,6 +687,9 @@ func (c *context) fail(offset int) {
c.matchLast = false
}
func findLine(tokens []rune, offset int) (line, column int) {
if offset < 0 {
return 0, 0
}
tokens = tokens[:offset]
for i := range tokens {
column++

37
notes.txt
View File

@ -1,40 +1,14 @@
[next]
errors
generator 1
documentation
parser 1
releasing
parser 2
generator 2
formatter
report unused parsers
parse hashed, storing only the results
linux packaging
[errors]
take the last
don't report aliases
take the last meaningful sequence
test error report on invalid flag
input name: may be just dropped because completely controlled by the client
input name needed in command to differentiate between syntax and input in check and parse subcommands
[generator 1]
allchars: can have char sequence
make generator output non-random (track parsers in a list in definition order)
fix the license in the output
[generator 2]
js
[releasing]
spellcheck
linting
convert notes into issues
try to remove some files
[parser 1]
try winning on allChars
retry collapsing
[parser 2]
custom tokens
indentation
@ -42,10 +16,3 @@ streaming support // ReadNode(io.Reader)
[optimization]
try preallocate larger store chunks
[documentation]
how the char classes are different from regexp
why need nows when using ws
lib only useful for dynamic syntax definition
warn nows usage in docs, e.g. spaces in symbol = [a-z]+
tutorial

33
parse_test.go
View File

@ -162,6 +162,39 @@ func TestRecursion(t *testing.T) {
},
}},
)
runTests(
t,
"A = A [a]",
[]testItem{{
title: "naive left recursion expects infinite stream",
text: "a",
ignorePosition: true,
fail: true,
}},
)
runTests(
t,
"A = A [a]; B = A",
[]testItem{{
title: "embedded naive left recursion expects infinite stream",
text: "a",
ignorePosition: true,
fail: true,
}},
)
runTests(
t,
"ws:ws = [ \t\n]; A = A [a]; B = A",
[]testItem{{
title: "embedded naive left recursion expects infinite stream with whitespace",
text: "\na",
ignorePosition: true,
fail: true,
}},
)
}
func TestSequence(t *testing.T) {

18
readme.md
View File

@ -20,9 +20,13 @@ syntax language supports recursive references, enabling the definition of contex
We can define syntaxes during development and use the provided tool to generate static Go code, which is then
built into the application. Alternatively, the library supports loading syntaxes dynamically at runtime.
The parser engine handles recursive references and left-recursion internally. This way it makes it more
convenient writing intuitive grammar definitions, and allows defining context-free languages without complex
workarounds.
## Installation
From source:
From source (recommended):
```
git clone https://code.squareroundforest.org/arpio/treerack
@ -30,7 +34,7 @@ cd treerack
make install
```
Alternatively:
Alternatively ("best effort" basis):
```
go install code.squareroundforest.org/arpio/treerack/cmd/treerack
@ -38,8 +42,8 @@ go install code.squareroundforest.org/arpio/treerack/cmd/treerack
## Documentation
- [Manual](docs/manual.md): A guide to the main use cases supported by Treerack.
- [Syntax Definition](docs/syntax.md): Detailed reference for the Treerack definition language.
- [Manual](docs/manual.md): a guide to the main use cases supported by Treerack.
- [Syntax Definition](docs/syntax.md): detailed reference for the Treerack definition language.
- [Library Documentation](https://godocs.io/code.squareroundforest.org/arpio/treerack): GoDoc reference for the
runtime library.
@ -47,10 +51,10 @@ go install code.squareroundforest.org/arpio/treerack/cmd/treerack
We use a Makefile to manage the build and verification lifecycle.
Important: Generating the parser for the Treerack syntax itself (bootstrapping) requires multiple phases.
Important: generating the parser for the Treerack syntax itself (bootstrapping) requires multiple phases.
Consequently, running standard go build or go test commands may miss subtle consistency problems.
The authoritative way to verify changes is via the makefile:
The decisive way to verify changes is via the makefile:
```
make check
@ -60,6 +64,6 @@ make check
- Lexer & UTF-8: Treerack does not require a lexer, which simplifies the architecture. However, this enforces
the use of UTF-8 input. We have considered support for custom tokenizers as a potential future improvement.
- Whitespace Delimited Languages: Due to the recursive descent nature and the lack of a dedicated lexer state,
- Whitespace Delimited Languages: due to the recursive descent nature and the lack of a dedicated lexer state,
defining whitespace-delimited syntaxes (such as Python-style indentation) can be difficult to achieve with the
current feature set.