refactor command options

cmd/treerack/args.go

@@ -0,0 +1,63 @@
+package main
+
+import "flag"
+
+type syntaxOptions struct {
+	usage      string
+	example    string
+	args       []string
+	positional []string
+	syntax     string
+	syntaxFile string
+	flagSet    *flag.FlagSet
+}
+
+func initOptions(usage, example string, args []string) *syntaxOptions {
+	var o syntaxOptions
+	o.usage = usage
+	o.example = example
+	o.args = args
+	o.flagSet = flag.NewFlagSet("", flag.ContinueOnError)
+	o.flagSet.Usage = func() {}
+	o.flagSet.SetOutput(werr)
+	o.flagSet.StringVar(&o.syntax, "syntax-string", "", syntaxStringUsage)
+	o.flagSet.StringVar(&o.syntaxFile, "syntax", "", syntaxFileUsage)
+	return &o
+}
+
+func flagError(fs *flag.FlagSet) {
+	stderr()
+	stderr("Options:")
+	fs.PrintDefaults()
+}
+
+func (o *syntaxOptions) parse() (exit int) {
+	if err := o.flagSet.Parse(o.args); err != nil {
+		flagError(o.flagSet)
+		exit = -1
+	}
+
+	o.positional = o.flagSet.Args()
+	return
+}
+
+func (o *syntaxOptions) help() {
+	stdout(o.usage)
+	stdout()
+	stdout("Options:")
+	o.flagSet.SetOutput(wout)
+	o.flagSet.PrintDefaults()
+	stdout()
+	stdout(o.example)
+	stdout()
+	stdout(docRef)
+}
+
+func (o *syntaxOptions) checkHelp() bool {
+	if len(o.args) == 0 || o.args[0] != "-help" {
+		return false
+	}
+
+	o.help()
+	return true
+}

cmd/treerack/check.go

@@ -1,54 +1,5 @@
 package main
 
-import (
+func check([]string) int {
-	"flag"
+	return 0
-	"io"
-)
-
-type checkOptions struct {
-	syntaxOptions
-}
-
-func flagSetCheck(o *checkOptions, output io.Writer) *flag.FlagSet {
-	fs := flag.NewFlagSet("", flag.ContinueOnError)
-	fs.Usage = func() {}
-	fs.SetOutput(output)
-	fs.StringVar(&o.syntax, "syntax-string", "", syntaxStringUsage)
-	fs.StringVar(&o.syntaxFile, "syntax", "", syntaxFileUsage)
-	return fs
-}
-
-func flagErrorCheck(fs *flag.FlagSet) {
-	stderr()
-	stderr("Options:")
-	fs.PrintDefaults()
-}
-
-func helpCheck() {
-	stdout(checkUsage)
-	stdout()
-	stdout("Options:")
-	fs := flagSetCheck(&checkOptions{}, wout)
-	fs.PrintDefaults()
-	stdout()
-	stdout(checkExample)
-	stdout()
-	stdout(docRef)
-}
-
-func check(args []string) int {
-	if len(args) > 0 && args[0] == "-help" {
-		helpCheck()
-		return 0
-	}
-
-	var options checkOptions
-	fs := flagSetCheck(&options, werr)
-	if err := fs.Parse(args); err != nil {
-		flagErrorCheck(fs)
-		return -1
-	}
-
-	_, code := open(options.syntaxOptions, fs)
-	return code
 }

cmd/treerack/check_test.go

@@ -1,103 +1 @@
 package main
-
-import "testing"
-
-func TestCheck(t *testing.T) {
-	runMainTest(t,
-		mainTest{
-			title: "help",
-			args: []string{
-				"treerack", "check", "-help",
-			},
-			stdout: []string{
-				checkUsage,
-				"-syntax",
-				"-syntax-string",
-				checkExample,
-				docRef,
-			},
-		},
-
-		mainTest{
-			title: "invalid flag",
-			args: []string{
-				"treerack", "check", "-foo",
-			},
-			exit: -1,
-			stderr: []string{
-				"-syntax",
-				"-syntax-string",
-			},
-		},
-
-		mainTest{
-			title: "multiple inputs",
-			args: []string{
-				"treerack", "check", "-syntax", "foo.treerack", "-syntax-string", `foo = "bar"`,
-			},
-			exit: -1,
-			stderr: []string{
-				"only one",
-				"-syntax",
-				"-syntax-string",
-			},
-		},
-
-		mainTest{
-			title: "no input",
-			args: []string{
-				"treerack", "check",
-			},
-			exit: -1,
-			stderr: []string{
-				"missing syntax input",
-				"-syntax",
-				"-syntax-string",
-			},
-		},
-
-		mainTest{
-			title: "invalid input",
-			args: []string{
-				"treerack", "check", "-syntax-string", "foo",
-			},
-			exit: -1,
-			stderr: []string{
-				"parse failed",
-			},
-		},
-
-		mainTest{
-			title: "file open fails",
-			args: []string{
-				"treerack", "check", "-syntax", "noexist.treerack",
-			},
-			exit: -1,
-			stderr: []string{
-				"file",
-			},
-		},
-
-		mainTest{
-			title: "syntax as stdin",
-			args: []string{
-				"treerack", "check",
-			},
-			stdin: `foo = "bar"`,
-		},
-
-		mainTest{
-			title: "syntax as file",
-			args: []string{
-				"treerack", "generate", "-syntax", "foo_test.treerack",
-			},
-		},
-
-		mainTest{
-			title: "syntax as string",
-			args: []string{
-				"treerack", "generate", "-syntax-string", `foo = "bar"`,
-			},
-		},
-	)
-}

cmd/treerack/checksyntax.go

@@ -0,0 +1,15 @@
+package main
+
+func checkSyntax(args []string) int {
+	options := initOptions(checkSyntaxUsage, checkSyntaxExample, args)
+	if options.checkHelp() {
+		return 0
+	}
+
+	if code := options.parse(); code != 0 {
+		return code
+	}
+
+	_, code := openSyntax(options)
+	return code
+}

cmd/treerack/checksyntax_test.go

@@ -0,0 +1,129 @@
+package main
+
+import "testing"
+
+func TestCheckSyntax(t *testing.T) {
+	runMainTest(t,
+		mainTest{
+			title: "help",
+			args: []string{
+				"treerack", "check-syntax", "-help",
+			},
+			stdout: []string{
+				checkSyntaxUsage,
+				"-syntax",
+				"-syntax-string",
+				checkSyntaxExample,
+				docRef,
+			},
+		},
+
+		mainTest{
+			title: "invalid flag",
+			args: []string{
+				"treerack", "check-syntax", "-foo",
+			},
+			exit: -1,
+			stderr: []string{
+				"-syntax",
+				"-syntax-string",
+			},
+		},
+
+		mainTest{
+			title: "multiple inputs",
+			args: []string{
+				"treerack", "check-syntax", "-syntax", "foo.treerack", "-syntax-string", `foo = "bar"`,
+			},
+			exit: -1,
+			stderr: []string{
+				"only one",
+				"-syntax",
+				"-syntax-string",
+			},
+		},
+
+		mainTest{
+			title: "multiple inputs, positional",
+			args: []string{
+				"treerack", "check-syntax", "foo.treerack", "bar.treerack",
+			},
+			exit: -1,
+			stderr: []string{
+				"only one",
+				"-syntax",
+				"-syntax-string",
+			},
+		},
+
+		mainTest{
+			title: "multiple inputs, positional and explicit file",
+			args: []string{
+				"treerack", "check-syntax", "-syntax", "foo.treerack", "bar.treerack",
+			},
+			exit: -1,
+			stderr: []string{
+				"only one",
+				"-syntax",
+				"-syntax-string",
+			},
+		},
+
+		mainTest{
+			title: "no input",
+			args: []string{
+				"treerack", "check-syntax",
+			},
+			exit: -1,
+			stderr: []string{
+				"missing syntax input",
+				"-syntax",
+				"-syntax-string",
+			},
+		},
+
+		mainTest{
+			title: "invalid input",
+			args: []string{
+				"treerack", "check-syntax", "-syntax-string", "foo",
+			},
+			exit: -1,
+			stderr: []string{
+				"parse failed",
+			},
+		},
+
+		mainTest{
+			title: "file open fails",
+			args: []string{
+				"treerack", "check-syntax", "-syntax", "noexist.treerack",
+			},
+			exit: -1,
+			stderr: []string{
+				"file",
+			},
+		},
+
+		mainTest{
+			title: "syntax as stdin",
+			args: []string{
+				"treerack", "check-syntax",
+			},
+			stdin: `foo = "bar"`,
+		},
+
+		mainTest{
+			title: "syntax as file",
+			args: []string{
+				"treerack", "generate", "-syntax", "foo_test.treerack",
+			},
+		},
+
+		mainTest{
+			title: "syntax as string",
+			args: []string{
+				"treerack", "generate", "-syntax-string", `foo = "bar"`,
+			},
+		},
+	)
+}

cmd/treerack/doc.go

@@ -3,8 +3,11 @@ package main
 const summary = `treerack - parser generator - https://github.com/aryszka/treerack`
 
 const commandsHelp = `Available commands:
-generate     generates a parser from a syntax definition
+check          validates an arbitrary input against a syntax definition
-help         prints the current help
+parse          parses an arbitrary input with a syntax definition into an abstract syntax tree
+check-syntax   validates a syntax definition
+generate       generates a parser from a syntax definition
+help           prints the current help
 
 See more details about a particular command by calling:
 treerack <command> -help`
@@ -19,12 +22,17 @@ const packageNameUsage = `package name of the generated Go code`
 
 const exportUsage = `when the export flag is set, the generated code will have exported symbols to allow using it as a separate package`
 
-const generateUsage = `treerack generate takes a syntax description from the standard input, or a file, or inline string, and generates parser code implementing the described syntax. It prints the parser code to the standard output.`
+const parseUsage = `'treerack parse' takes a syntax description from a file or inline string, an arbitrary piece of text from the standard input, or a file, or inline string, and parses the input text with the defined syntax.  If it was successfully parsed, it prints the resulting abstract syntax tree (AST) in JSON format.`
+
+const parseExample = `Example:
+treerack parse -syntax example.treerack foo.example`
+
+const checkSyntaxUsage = `'treerack check-syntax' takes a syntax description from the standard input, or a file, or inline string, and validates it to check whether it represents a valid syntax. It returns with non-zero exit code and prints the problem if the syntax is not valid.`
+
+const checkSyntaxExample = `Example:
+treerack check-syntax example.treerack`
+
+const generateUsage = `'treerack generate' takes a syntax description from the standard input, or a file, or inline string, and generates parser code implementing the described syntax. It prints the parser code to the standard output.`
 
 const generateExample = `Example:
-treerack generate -syntax syntax.treerack > parser.go`
+treerack generate example.treerack > parser.go`
-
-const checkUsage = `treerack check takes a syntax description from the standard input, or a file, or inline string, and validates it to check whether it represents a valid syntax. It returns with non-zero exit code and prints the problem if the syntax is not valid.`
-
-const checkExample = `Example:
-treerack check -syntax syntax.treerack`

cmd/treerack/generate.go

@@ -1,62 +1,24 @@
 package main
 
-import (
+import "github.com/aryszka/treerack"
-	"flag"
-	"io"
-
-	"github.com/aryszka/treerack"
-)
 
 type generateOptions struct {
-	syntaxOptions
+	*syntaxOptions
 	packageName string
 	export      bool
 }
 
-func flagSetGenerate(o *generateOptions, output io.Writer) *flag.FlagSet {
-	fs := flag.NewFlagSet("", flag.ContinueOnError)
-	fs.Usage = func() {}
-	fs.SetOutput(output)
-	fs.StringVar(&o.syntax, "syntax-string", "", syntaxStringUsage)
-	fs.StringVar(&o.syntaxFile, "syntax", "", syntaxFileUsage)
-	fs.StringVar(&o.packageName, "package-name", "", packageNameUsage)
-	fs.BoolVar(&o.export, "export", false, exportUsage)
-	return fs
-}
-
-func flagErrorGenerate(fs *flag.FlagSet) {
-	stderr()
-	stderr("Options:")
-	fs.PrintDefaults()
-}
-
-func helpGenerate() {
-	stdout(generateUsage)
-	stdout()
-	stdout("Options:")
-	fs := flagSetGenerate(&generateOptions{}, wout)
-	fs.PrintDefaults()
-	stdout()
-	stdout(generateExample)
-	stdout()
-	stdout(docRef)
-}
-
 func generate(args []string) int {
-	if len(args) > 0 && args[0] == "-help" {
+	var options generateOptions
-		helpGenerate()
+	options.syntaxOptions = initOptions(generateUsage, generateExample, args)
+	options.flagSet.BoolVar(&options.export, "export", false, exportUsage)
+	options.flagSet.StringVar(&options.packageName, "package-name", "", packageNameUsage)
+
+	if options.checkHelp() {
 		return 0
 	}
 
-	var options generateOptions
+	if code := options.parse(); code != 0 {
-	fs := flagSetGenerate(&options, werr)
-	if err := fs.Parse(args); err != nil {
-		flagErrorGenerate(fs)
-		return -1
-	}
-
-	s, code := open(options.syntaxOptions, fs)
-	if code != 0 {
 		return code
 	}
 
@@ -64,6 +26,11 @@ func generate(args []string) int {
 	goptions.PackageName = options.packageName
 	goptions.Export = options.export
 
+	s, code := openSyntax(options.syntaxOptions)
+	if code != 0 {
+		return code
+	}
+
 	if err := s.Generate(goptions, wout); err != nil {
 		stderr(err)
 		return -1

cmd/treerack/generate_test.go

@@ -49,6 +49,36 @@ func TestGenerate(t *testing.T) {
 			},
 		},
 
+		mainTest{
+			title: "multiple inputs, positional",
+			args: []string{
+				"treerack", "generate", "foo.treerack", "bar.treerack",
+			},
+			exit: -1,
+			stderr: []string{
+				"only one",
+				"-export",
+				"-package-name",
+				"-syntax",
+				"-syntax-string",
+			},
+		},
+
+		mainTest{
+			title: "multiple inputs, positional and explicit file",
+			args: []string{
+				"treerack", "generate", "-syntax", "foo.treerack", "bar.treerack",
+			},
+			exit: -1,
+			stderr: []string{
+				"only one",
+				"-export",
+				"-package-name",
+				"-syntax",
+				"-syntax-string",
+			},
+		},
+
 		mainTest{
 			title: "no input",
 			args: []string{

cmd/treerack/main.go

@@ -21,19 +21,27 @@ func main() {
 		return
 	}
 
+	var cmd func([]string) int
+
 	switch os.Args[1] {
-	case "check":
+	case "check-syntax":
-		code := check(os.Args[2:])
+		cmd = checkSyntax
-		exit(code)
 	case "generate":
-		code := generate(os.Args[2:])
+		cmd = generate
-		exit(code)
+	case "check":
+		cmd = check
+	case "parse":
+		cmd = parse
 	case "help", "-help":
 		mainHelp()
+		return
 	default:
 		stderr("invalid command")
 		stderr()
 		stderr(commandsHelp)
 		exit(-1)
+		return
 	}
+
+	exit(cmd(os.Args[2:]))
 }

cmd/treerack/open.go

@@ -10,11 +10,6 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-type syntaxOptions struct {
-	syntax     string
-	syntaxFile string
-}
-
 func multipleSyntaxesError(fs *flag.FlagSet) {
 	stderr("only one of syntax file or syntax string is allowed")
 	stderr()
@@ -29,27 +24,62 @@ func missingSyntaxError(fs *flag.FlagSet) {
 	fs.PrintDefaults()
 }
 
-func open(options syntaxOptions, fs *flag.FlagSet) (*treerack.Syntax, int) {
+func getSource(options *syntaxOptions) (hasInput bool, fileName string, syntax string, code int) {
-	if options.syntaxFile != "" && options.syntax != "" {
+	if len(options.positional) > 1 {
-		multipleSyntaxesError(fs)
+		multipleSyntaxesError(options.flagSet)
-		return nil, -1
+		code = -1
+		return
 	}
 
-	var hasInput bool
+	hasPositional := len(options.positional) == 1
-	if options.syntaxFile == "" && options.syntax == "" {
+	hasFile := options.syntaxFile != ""
-		hasInput = isTest && rin != nil || !isTest && !terminal.IsTerminal(0)
+	hasSyntax := options.syntax != ""
+
+	var has bool
+	for _, h := range []bool{hasPositional, hasFile, hasSyntax} {
+		if h && has {
+			multipleSyntaxesError(options.flagSet)
+			code = -1
+			return
+		}
+
+		has = h
 	}
 
-	if !hasInput && options.syntaxFile == "" && options.syntax == "" {
+	switch {
-		missingSyntaxError(fs)
+	case hasPositional:
-		return nil, -1
+		fileName = options.positional[0]
+		return
+	case hasFile:
+		fileName = options.syntaxFile
+		return
+	case hasSyntax:
+		syntax = options.syntax
+		return
+	}
+
+	// check input last to allow explicit syntax in non-TTY environments:
+	hasInput = isTest && rin != nil || !isTest && !terminal.IsTerminal(0)
+	if !hasInput {
+		missingSyntaxError(options.flagSet)
+		code = -1
+		return
+	}
+
+	return
+}
+
+func openSyntax(options *syntaxOptions) (*treerack.Syntax, int) {
+	hasInput, fileName, syntax, code := getSource(options)
+	if code != 0 {
+		return nil, code
 	}
 
 	var input io.Reader
 	if hasInput {
 		input = rin
-	} else if options.syntaxFile != "" {
+	} else if fileName != "" {
-		f, err := os.Open(options.syntaxFile)
+		f, err := os.Open(fileName)
 		if err != nil {
 			stderr(err)
 			return nil, -1
@@ -57,8 +87,8 @@ func open(options syntaxOptions, fs *flag.FlagSet) (*treerack.Syntax, int) {
 
 		defer f.Close()
 		input = f
-	} else if options.syntax != "" {
+	} else {
-		input = bytes.NewBufferString(options.syntax)
+		input = bytes.NewBufferString(syntax)
 	}
 
 	s := &treerack.Syntax{}

cmd/treerack/parse.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"flag"
+	"io"
+)
+
+type parseOptions struct {
+	syntaxOptions
+	input     string
+	inputFile string
+	pretty    bool
+	indent    string
+}
+
+func flagSetParse(o *parseOptions, output io.Writer) *flag.FlagSet {
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.Usage = func() {}
+	fs.SetOutput(output)
+	fs.StringVar(&o.syntax, "syntax-string", "", syntaxStringUsage)
+	fs.StringVar(&o.syntaxFile, "syntax", "", syntaxFileUsage)
+	return fs
+}
+
+func flagErrorParse(fs *flag.FlagSet) {
+	stderr()
+	stderr("Options:")
+	fs.PrintDefaults()
+}
+
+func helpParse() {
+	stdout(parseUsage)
+	stdout()
+	stdout("Options:")
+	fs := flagSetParse(&parseOptions{}, wout)
+	fs.PrintDefaults()
+	stdout()
+	stdout(parseExample)
+	stdout()
+	stdout(docRef)
+}
+
+func parse(args []string) int {
+	return 0
+}

notes.txt

@@ -1,59 +1,35 @@
-error reporting
-- longest parse
-- count the lines
-- print the line
-- print the deepest non-alias node name
-- print the documentation of the node name
-- make it work with custom tokens
-read, with error reporting
-what was the bug with the large json from eskip?
-
 [next]
-check
+formatter
-parse
+report unused parsers
-error reports
+parse hashed, storing only the results
-- take the last
+linux packaging
-simplify generator output
+
-make generator output non-random (track parsers in a list in definition order)
+[parser]
-missing tests, coverage:
-- validation
-- error cases
-- whitespace cases
-error reporting
-coverage
 custom tokens
 indentation
-streaming
+streaming support // ReadNode(io.Reader)
-code generation go:
+
-- find things that depend on the syntax input
+[cmd]
-- char matches can be generated into switches
+check
-code generation js
+parse
-documentation flag
+help for positional argument
-support custom tokenization
+test explicit input priority
-streaming
+
-verify choice and sequence preference
+[errors]
-formatter
+take the last
-pretty
-report unused parsers
-go through the tests for docs
-test all docs
-warn nows usage in docs, e.g. spaces in symbol = [a-z]+
 test error report on invalid flag
-report unused definitions
-simplify generated output
-parse hashed, storing only the results
-allchars: can have char sequence
 input name: may be just dropped because completely controlled by the client
 
+[generator]
+allchars: can have char sequence
+make generator output non-random (track parsers in a list in definition order)
+js
+
 [optimization]
 try preallocate larger store chunks
-notes in formatting
-format: test back and forth equivalence of char classes
-format: test comments, first apply comments in the syntax
-
-[problems]
-can the root be an alias? check the commit mechanism
 
 [documentation]
 how the char classes are different from regexp
 why need nows when using ws
+lib only useful for dynamic syntax definition
+warn nows usage in docs, e.g. spaces in symbol = [a-z]+