diff --git a/tools/beamsplitter/README.md b/tools/beamsplitter/README.md index efb13685a3..66aef90fe8 100644 --- a/tools/beamsplitter/README.md +++ b/tools/beamsplitter/README.md @@ -2,17 +2,17 @@ - [Description](#description) - [Instructions](#instructions) -- [Input Limitations](#input-limitations) - [Emitter Flags](#emitter-flags) -- [Input Files](#input-files) +- [Source Files](#source-files) - [Output Files](#output-files) +- [Input Format](#input-format) -### Description +## Description This Go program consumes C++ header file(s) and generates Java bindings, JavaScript bindings, and C++ code that performs JSON serialization. -### Instructions +## Instructions To install the Go compiler on macOS, just do: @@ -22,19 +22,7 @@ To build and invoke the code generator, do: cd tools/beamsplitter ; go run . -### Input Limitations - -The source files must have very simple C++ syntax. Some of the limitations include: - -- Only `enum class` is supported; no old-style enums. -- Opening braces for `enum` and `struct` must live at the end of a codeline. -- Enum values must be sequential and cannot have custom values. -- There are no namespaces other than the top-level namespace. -- Every struct field must supply a default value on a single codeline using the = operator. -- If the default value of a field is a vector, it must be in the form: `{ x, y, z }`. -- There must be no string literals that contain keywords. - -### Emitter Flags +## Emitter Flags Special directives in the form `%codegen_foo%` are called *emitter flags*. They are typically embedded in a comment associated with a particular struct field. @@ -43,14 +31,14 @@ flag | description --------------------------- | ---- **codegen_skip_json** | Field is skipped when generating JSON serialization code. **codegen_skip_javascript** | Field is skipped when generating JavaScript and TypeScript bindings. -**codegen_java_flatten** | Field is replaced with constituent sub-fields. (TBD) +**codegen_java_flatten** | Field is replaced with constituent sub-fields. **codegen_java_float** | Field will be forced to have a `float` representation in Java. -### Input Files +## Source Files - `filament/include/filament/Options.h` -### Output Files +## Output Files The following files are created: @@ -64,3 +52,93 @@ Additionally, in-place edits are made to the following files: - `web/filament-js/filament.d.ts` - `android/filament-android/src/main/java/.../View.java` + +## Input Format + +There are many ways in which the source file format is more restrictive than the full C++ +language, but here are some of the highlights: + +- All enums must be class enums. +- External headers pulled in with `#include` files are ignored. +- Expressions in the RHS of default value assignments are not parsed, they are just exposed by + the lexer as blobs. +- Struct fields, class fields, and method arguments must have fairly simple types. e.g. they cannot + have parentheses. If a type is C style callback, then it should be specified with an alias. +- Multiline strings and macro definitions are not allowed. +- Enum values must be sequential and cannot have custom values. + +The following formal grammar describes the above limitations in greater detail, but with some +caveats: + +- All C preprocessor directives are discarded during lexical analysis; they do not exist in the AST. +- Whitespace is similarly discarded, so there is no "space" concept in the AST. +- Macro invocations are also removed by the lexer if they are known Filament-specific macros (e.g. + `UTILS_PUBLIC` and `UTILS_DEPRECATED`). +- Comments are removed by the lexer and are generally not part of the resulting AST. However + the lexer proffers a mapping from line numbers to comments to allow for docstring extraction. +- Emitter flags in the form `%codegen_foo%` are detected in a post-processing phase and removed from + all comments. + +### Grammar + +```eBNF +root = namespace ; +namespace = "namespace" , [ ident ] , "{" , { block } , "}" ; +block = class | struct | enum | namespace | using | forward_declaration; +forward_declaration = ("class" | "struct" ) , ident , ";" ; +template = "template" , "TemplateArgs" ; +class = [template] , "class" , ident , [ ":" , [ "public" ] , "SimpleType" ] + , "{" , struct_body , "}" , ";" ; +struct = [template] , "struct" , ident , "{" , struct_body , "}" , ";" ; +enum = "enum" , "class" , ident , [ ":" , type ] + , "{" , , ident , { "," , ident } , [ "," ] , "}" , ";" ; +using = "using" , ident , "=", type , ";" ; +struct_body = { access_specifier | field | method | block } ; +access_specifier = ("public" | "private" | "protected" ) , ":" ; +method = [template] , { "constexpr" , "friend" } , + , type , ident , "MethodArgs" , specifiers , ( ";" | "MethodBody" ) ; +specifiers = { "const" | "noexcept" } ; +field = type , ident , [ array ] , [ "=" , "DefaultValue" ] ";" ; +array = "[" , "ArrayLength", "]" ; +type = "SimpleType" ; +ident = "Identifier" ; +``` +The above grammar uses the following notation: +- `" ... "` denotes a terminal +- `{ ... }` denotes zero or more repetition +- `[ ... ]` denotes an optional quantity +- `( ... )` is used for grouping +- `a | b` denotes a choice +- `a , b` denotes concatenation +- `;` terminates a production + +Terminal name | Description +--------------------------- | ---- +SimpleType (*) | examples: `Texture* const`, `uint8_t`, `BlendMode` +MethodBody | unparsed implementation of a function or method, including outer `{}` +MethodArgs | similar to above; an unparsed blob, but delimited with `()` +TemplateArgs | similar to above; an unparsed blob, but delimited with `<>` +DefaultValue (**) | an unparsed expression with certain restrictions +Identifier | `[A-Za-z_][A-Za-z0-9_]*` +ArrayLength | `[1-9][0-9]*` + +(*) `SimpleType` should not contain parentheses or commas, so C callbacks are not allowed unless +you alias them first. + +(**) If `DefaultValue` is a vector, it must be in the form: `{ x, y, z }`. + +## References + +Initially inspired by the following Rob Pike talk. +- https://www.youtube.com/watch?v=HxaD_trXwRE + +Beamsplitter does not use the state machine described in the above prezo, but it does use a channel +for separating the parser from the lexer. The beamsplitter lexer is actually a recursive descent +parser with simple lookahead functionality. This makes it easy for the "real" parser to create a +coarse-grained AST. + +The companion to the above talk is Go's template lexer, which can be studied here: +- https://cs.opensource.google/go/go/+/master:src/text/template/parse/lex.go + +Wikipedia has a good example of recursive descent: +- https://en.wikipedia.org/wiki/Recursive_descent_parser diff --git a/tools/beamsplitter/database/database.go b/tools/beamsplitter/database/database.go new file mode 100644 index 0000000000..3a603c564d --- /dev/null +++ b/tools/beamsplitter/database/database.go @@ -0,0 +1,332 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package database + +import ( + "beamsplitter/parse" + "bufio" + "io" + "log" + "regexp" + "strings" +) + +func Create(root *parse.RootNode, contents string) []TypeDefinition { + context := gatherContext{} + context.compileRegexps() + + // Line numbers are 1-based in beamsplitter, so we add an extra newline at the top. + context.codelines = strings.Split("\n"+contents, "\n") + context.commentBlocks = gatherCommentBlocks(strings.NewReader(contents)) + + // Recurse into the AST using pre-order traversal, gathering type information along the way. + context.gatherTypeDefinitions(root.Child, "", nil) + + context.addTypeQualifiers() + + return context.definitions +} + +type TypeDefinition interface { + BaseName() string + QualifiedName() string + Parent() TypeDefinition +} + +type StructField struct { + TypeString string + Name string + DefaultValue string + Description string + EmitterFlags map[string]struct{} + CustomType TypeDefinition +} + +type StructDefinition struct { + name string + qualifier string + Fields []StructField + Description string + parent TypeDefinition +} + +func (defn StructDefinition) BaseName() string { return defn.name } +func (defn StructDefinition) QualifiedName() string { return defn.qualifier + defn.name } +func (defn StructDefinition) Parent() TypeDefinition { return defn.parent } + +type EnumValue struct { + Description string + Name string +} + +type EnumDefinition struct { + name string + qualifier string + Values []EnumValue + Description string + parent TypeDefinition +} + +func (defn EnumDefinition) BaseName() string { return defn.name } +func (defn EnumDefinition) QualifiedName() string { return defn.qualifier + defn.name } +func (defn EnumDefinition) Parent() TypeDefinition { return defn.parent } + +type Documented interface{ GetDoc() string } + +func (defn EnumDefinition) GetDoc() string { return defn.Description } +func (defn StructDefinition) GetDoc() string { return defn.Description } +func (field StructField) GetDoc() string { return field.Description } +func (value EnumValue) GetDoc() string { return value.Description } + +type generalScope struct{} + +type scope interface { + BaseName() string + QualifiedName() string + Parent() TypeDefinition +} + +func (defn generalScope) BaseName() string { return "" } +func (defn generalScope) QualifiedName() string { return "" } +func (defn generalScope) Parent() TypeDefinition { return nil } + +type gatherContext struct { + definitions []TypeDefinition + stack []scope + commentBlocks map[int]string + codelines []string + floatMatcher *regexp.Regexp + vectorMatcher *regexp.Regexp + fieldDocParser *regexp.Regexp + emitterFlagFinder *regexp.Regexp +} + +// https://github.com/google/re2/wiki/Syntax +func (context *gatherContext) compileRegexps() { + context.floatMatcher = regexp.MustCompile(`(\-?[0-9]+\.[0-9]*)f?`) + context.vectorMatcher = regexp.MustCompile(`\{(\s*\-?[0-9\.]+\s*(,\s*\-?[0-9\.]+\s*){1,})\}`) + context.emitterFlagFinder = regexp.MustCompile(`\s*\%codegen_([a-zA-Z0-9_]+)\%\s*`) + context.fieldDocParser = regexp.MustCompile(`(?://\s*\!\<\s*(.*))`) +} + +// Creates a mapping from line numbers to strings, where the strings are entire block comments +// and the line numbers correspond to the last line of each block comment. +func gatherCommentBlocks(sourceFile io.Reader) map[int]string { + comments := make(map[int]string) + scanner := bufio.NewScanner(sourceFile) + var comment = "" + var indention = 0 + for lineNumber := 1; scanner.Scan(); lineNumber++ { + codeline := scanner.Text() + if strings.Contains(codeline, `/**`) { + indention = strings.Index(codeline, `/**`) + if strings.Contains(codeline, `*/`) { + comments[lineNumber] = codeline[indention:] + "\n" + continue + } + comment = codeline[indention:] + "\n" + continue + } + if comment != "" { + if len(codeline) > indention { + codeline = codeline[indention:] + } + comment += codeline + "\n" + if strings.Contains(codeline, `*/`) { + comments[lineNumber] = comment + comment = "" + } + } + } + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + return comments +} + +// Annotates struct fields that have custom types (i.e. enums or structs). +func (context gatherContext) addTypeQualifiers() { + typeMap := make(map[string]TypeDefinition) + for _, defn := range context.definitions { + typeMap[defn.QualifiedName()] = defn + } + for _, defn := range context.definitions { + structDefn, isStruct := defn.(*StructDefinition) + if !isStruct { + continue + } + for fieldIndex, field := range structDefn.Fields { + // Extract the namespace prefix (if any) explicitly specified for this field type. + var namespace string + localTypeName := field.TypeString + if index := strings.LastIndex(field.TypeString, "::"); index > -1 { + namespace = field.TypeString[:index] + localTypeName = field.TypeString[index+2:] + } + + // Prepend additional qualifiers to the type string by searching upward through + // the current namespace hierarchy, and looking for a match. + mutable := &structDefn.Fields[fieldIndex] + for ancestor := defn; ; ancestor = ancestor.Parent() { + var qualified string + if namespace != "" && strings.HasSuffix(ancestor.QualifiedName(), namespace) { + qualified = ancestor.QualifiedName() + "::" + localTypeName + } else { + qualified = ancestor.QualifiedName() + "::" + field.TypeString + } + if fieldType, found := typeMap[qualified]; found { + mutable.TypeString = qualified + mutable.CustomType = fieldType + break + } + if ancestor.Parent() == nil { + if fieldType, found := typeMap[field.TypeString]; found { + mutable.CustomType = fieldType + } + break + } + } + + if mutable.CustomType == nil { + continue + } + + // Prepend additional qualifiers to the value string if it is a known enum. + var fieldType TypeDefinition = structDefn.Fields[fieldIndex].CustomType + enumDefn, isEnum := fieldType.(*EnumDefinition) + if isEnum { + selectedEnum := field.DefaultValue + if index := strings.LastIndex(field.DefaultValue, "::"); index > -1 { + selectedEnum = field.DefaultValue[index+2:] + } + mutable.DefaultValue = enumDefn.QualifiedName() + "::" + selectedEnum + } + } + } +} + +// Validates and transforms the RHS of an assignment. +// For vectors, this converts curly braces into square brackets. +func (context gatherContext) distillValue(cppvalue string, lineNumber int) string { + cppvalue = strings.TrimSpace(cppvalue) + + // Remove trailing "f" from floats, which isn't allowed in JavaScript. + if context.floatMatcher.MatchString(cppvalue) { + cppvalue = context.floatMatcher.ReplaceAllString(cppvalue, "$1") + } + + // There are many ways to declare vector values (multi-arg constructor, single-arg + // constructor, curly braces with type, curly braces without type), so just poop out if + // the syntax is anything other than "curly braces without type". + if strings.Contains(cppvalue, "math::") || strings.Contains(cppvalue, "Color") { + log.Fatalf("%d: vectors must have the form {x, y ...}", lineNumber) + } + + // Assume it's a vector if there's a curly brace. + if strings.Contains(cppvalue, "{") { + if context.vectorMatcher.MatchString(cppvalue) { + cppvalue = context.vectorMatcher.ReplaceAllString(cppvalue, "[$1]") + } else { + log.Fatalf("%d: vectors must have the form {x, y ...}", lineNumber) + } + } + return cppvalue +} + +func (context *gatherContext) getDescription(line int) string { + desc := context.commentBlocks[line-1] + if desc == "" { + codeline := context.codelines[line] + if matches := context.fieldDocParser.FindStringSubmatch(codeline); matches != nil { + desc = matches[1] + } + } + return context.emitterFlagFinder.ReplaceAllString(desc, "") +} + +func (context *gatherContext) getEmitterFlags(line int) map[string]struct{} { + codeline := context.codelines[line] + if matches := context.emitterFlagFinder.FindAllStringSubmatch(codeline, -1); matches != nil { + result := make(map[string]struct{}, len(matches)) + for _, flag := range matches { + result[flag[1]] = struct{}{} + } + return result + } + return nil +} + +// Search for all enums and structs and gather them into a flat list of type definitions. +func (context *gatherContext) gatherTypeDefinitions(node parse.Node, prefix string, parent TypeDefinition) { + switch concrete := node.(type) { + case *parse.NamespaceNode: + // HACK: filament namespace is a special case, remove it from the type database. + if concrete.Name != "filament" { + prefix = prefix + concrete.Name + "::" + } + for _, child := range concrete.Children { + context.gatherTypeDefinitions(child, prefix, parent) + } + case *parse.EnumNode: + defn := &EnumDefinition{ + name: concrete.Name, + qualifier: prefix, + Values: make([]EnumValue, len(concrete.Values)), + Description: context.getDescription(int(concrete.Line)), + parent: parent, + } + for i, val := range concrete.Values { + defn.Values[i] = EnumValue{ + Name: val, + Description: context.getDescription(int(concrete.ValueLines[i])), + } + } + context.definitions = append(context.definitions, defn) + case *parse.StructNode: + defn := &StructDefinition{ + name: concrete.Name, + qualifier: prefix, + Fields: make([]StructField, 0), + Description: context.getDescription(int(concrete.Line)), + parent: parent, + } + prefix = prefix + concrete.Name + "::" + for _, child := range concrete.Members { + switch member := child.(type) { + case *parse.StructNode, *parse.ClassNode, *parse.EnumNode, *parse.NamespaceNode: + context.gatherTypeDefinitions(child, prefix, defn) + case *parse.FieldNode: + defn.Fields = append(defn.Fields, StructField{ + TypeString: member.Type, + Name: member.Name, + DefaultValue: context.distillValue(member.Rhs, int(member.Line)), + Description: context.getDescription(int(member.Line)), + EmitterFlags: context.getEmitterFlags(int(member.Line)), + }) + } + } + context.definitions = append(context.definitions, defn) + case *parse.ClassNode: + prefix = prefix + concrete.Name + "::" + for _, child := range concrete.Members { + switch child.(type) { + case *parse.StructNode, *parse.ClassNode, *parse.EnumNode, *parse.NamespaceNode: + context.gatherTypeDefinitions(child, prefix, parent) + } + } + } +} diff --git a/tools/beamsplitter/java.go b/tools/beamsplitter/emitters/java.go similarity index 85% rename from tools/beamsplitter/java.go rename to tools/beamsplitter/emitters/java.go index a31faa3df6..badd8956b1 100644 --- a/tools/beamsplitter/java.go +++ b/tools/beamsplitter/emitters/java.go @@ -14,9 +14,10 @@ * limitations under the License. */ -package main +package emitters import ( + db "beamsplitter/database" "bufio" "bytes" "fmt" @@ -26,43 +27,9 @@ import ( "path/filepath" "strings" "text/template" - - "beamsplitter/parse" ) -// Adds one level of indention to the given multi-line string. -// Isolated newlines are intentially not indented. -func indent(src string, depth int) string { - dst := &bytes.Buffer{} - buf := bytes.NewBufferString(src) - scanner := bufio.NewScanner(buf) - for scanner.Scan() { - codeline := scanner.Text() - if codeline != "" { - dst.WriteString(strings.Repeat(" ", depth)) - dst.WriteString(scanner.Text()) - } - dst.WriteByte('\n') - } - return dst.String() -} - -// Wrapper for ExecuteTemplate that performs error checking. Takes an output stream, a template name -// to invoke, and a template context object. -type templateFn = func(io.Writer, string, parse.TypeDefinition) - -func createJavaCodeGenerator(customExtensions template.FuncMap) templateFn { - templ := template.New("beamsplitter").Funcs(customExtensions) - templ = template.Must(templ.ParseFiles("java.template")) - return func(writer io.Writer, section string, definition parse.TypeDefinition) { - err := templ.ExecuteTemplate(writer, section, definition) - if err != nil { - log.Fatal(err.Error()) - } - } -} - -func editJava(definitions []parse.TypeDefinition, classname string, folder string) { +func EditJava(definitions []db.TypeDefinition, classname string, folder string) { path := filepath.Join(folder, classname+".java") var codelines []string { @@ -98,10 +65,10 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin file.WriteString(" // " + kCodelineMarker + "\n") // Forward declarations for usage in a closure. - var flattener func(*parse.StructDefinition) string + var flattener func(*db.StructDefinition) string var sharedExtensions template.FuncMap - javifyType := func(field parse.StructField) string { + javifyType := func(field db.StructField) string { if _, exists := field.EmitterFlags["java_float"]; exists { return " float" } @@ -118,7 +85,7 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin return " " + result } - javifyValue := func(field parse.StructField) string { + javifyValue := func(field db.StructField) string { // When forcing an array to be bound to a float, extract the first component and use // that as the default value. @@ -163,7 +130,7 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin return " " + value } - getDocBlock := func(defn parse.Documented, depth int) string { + getDocBlock := func(defn db.Documented, depth int) string { doc := defn.GetDoc() if doc == "" { return "" @@ -175,7 +142,7 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin return "/**\n" + indent + " * " + doc + "\n" + indent + " */\n" + indent } - getFieldAnnotation := func(field parse.StructField, depth int) string { + getFieldAnnotation := func(field db.StructField, depth int) string { if _, exists := field.EmitterFlags["java_float"]; exists { return "" } @@ -197,7 +164,7 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin return annotation + "\n" + strings.Repeat(" ", depth) } - flattenStruct := func(defn *parse.StructDefinition) string { + flattenStruct := func(defn *db.StructDefinition) string { prefix := strings.ToLower(defn.BaseName()) buf := &bytes.Buffer{} for _, field := range defn.Fields { @@ -219,10 +186,10 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin // These template extensions are used to transmogrify C++ symbols and value literals to Java. customExtensions := template.FuncMap{ "docblock": getDocBlock, - "nested_type_declarations": func(parent parse.TypeDefinition) string { + "nested_type_declarations": func(parent db.TypeDefinition) string { // Look for all fields that request flattening since we should skip their emission. - flattenedTypes := make(map[parse.TypeDefinition]struct{}) - if structDefn, isStruct := parent.(*parse.StructDefinition); isStruct { + flattenedTypes := make(map[db.TypeDefinition]struct{}) + if structDefn, isStruct := parent.(*db.StructDefinition); isStruct { for _, field := range structDefn.Fields { _, flatten := field.EmitterFlags["java_flatten"] if flatten && field.CustomType != nil { @@ -243,9 +210,9 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin continue } switch definition.(type) { - case *parse.StructDefinition: + case *db.StructDefinition: generate(buf, "Struct", definition) - case *parse.EnumDefinition: + case *db.EnumDefinition: generate(buf, "Enum", definition) } } @@ -254,14 +221,14 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin "annotation": getFieldAnnotation, "java_type": javifyType, "java_value": javifyValue, - "flatten": func(field *parse.StructField) string { - if structDefn, isStruct := field.CustomType.(*parse.StructDefinition); isStruct { + "flatten": func(field *db.StructField) string { + if structDefn, isStruct := field.CustomType.(*db.StructDefinition); isStruct { return strings.TrimLeft(flattener(structDefn), " ") } log.Fatal("Unexpected flatten flag.") return "" }, - "flag": func(field *parse.StructField, flag string) bool { + "flag": func(field *db.StructField, flag string) bool { _, exists := field.EmitterFlags[flag] return exists }, @@ -275,12 +242,44 @@ func editJava(definitions []parse.TypeDefinition, classname string, folder strin continue } switch definition.(type) { - case *parse.StructDefinition: + case *db.StructDefinition: generate(file, "Struct", definition) - case *parse.EnumDefinition: + case *db.EnumDefinition: generate(file, "Enum", definition) } } file.WriteString("}\n") } + +// Adds one level of indention to the given multi-line string. +// Isolated newlines are intentially not indented. +func indent(src string, depth int) string { + dst := &bytes.Buffer{} + buf := bytes.NewBufferString(src) + scanner := bufio.NewScanner(buf) + for scanner.Scan() { + codeline := scanner.Text() + if codeline != "" { + dst.WriteString(strings.Repeat(" ", depth)) + dst.WriteString(scanner.Text()) + } + dst.WriteByte('\n') + } + return dst.String() +} + +// Wrapper for ExecuteTemplate that performs error checking. Takes an output stream, a template name +// to invoke, and a template context object. +type templateFn = func(io.Writer, string, db.TypeDefinition) + +func createJavaCodeGenerator(customExtensions template.FuncMap) templateFn { + templ := template.New("beamsplitter").Funcs(customExtensions) + templ = template.Must(templ.ParseFiles("emitters/java.template")) + return func(writer io.Writer, section string, definition db.TypeDefinition) { + err := templ.ExecuteTemplate(writer, section, definition) + if err != nil { + log.Fatal(err.Error()) + } + } +} diff --git a/tools/beamsplitter/java.template b/tools/beamsplitter/emitters/java.template similarity index 100% rename from tools/beamsplitter/java.template rename to tools/beamsplitter/emitters/java.template diff --git a/tools/beamsplitter/javascript.go b/tools/beamsplitter/emitters/javascript.go similarity index 88% rename from tools/beamsplitter/javascript.go rename to tools/beamsplitter/emitters/javascript.go index e97bbdd10b..6651d2c4ff 100644 --- a/tools/beamsplitter/javascript.go +++ b/tools/beamsplitter/emitters/javascript.go @@ -14,10 +14,10 @@ * limitations under the License. */ -package main +package emitters import ( - "beamsplitter/parse" + db "beamsplitter/database" "bufio" "fmt" "log" @@ -27,85 +27,9 @@ import ( "text/template" ) -// Returns a templating function that automatically checks for fatal errors. The returned function -// takes an output stream, a template name to invoke, and a template context object. -func createJsCodeGenerator(namespace string) func(*os.File, string, parse.TypeDefinition) { - jsPrefix := "" - classPrefix := "" - cppPrefix := "" - if namespace != "" { - jsPrefix = namespace + "$" - classPrefix = namespace + ".prototype." - cppPrefix = namespace + "::" - } - // These template extensions are used to transmogrify C++ symbols and value literals into - // JavaScript. We mostly don't need to do anything since the parser has already done some - // massaging and verification (e.g. it removed the trailing "f" from floating point literals). - // However enums need some special care here. Emscripten bindings are flat, so our own - // convention is to use $ for the scoping delimiter, which is a legal symbol character in JS. - // However we still use . to separate the enum value from the enum type, because emscripten has - // first-class support for class enums. - customExtensions := template.FuncMap{ - "qualifiedtype": func(typename string) string { - typename = strings.ReplaceAll(typename, "::", "$") - return typename - }, - "flag": func(field *parse.StructField, flag string) bool { - _, exists := field.EmitterFlags[flag] - return exists - }, - "qualifiedvalue": func(name string) string { - count := strings.Count(name, "::") - if count > 0 { - name = "Filament." + jsPrefix + name - } - name = strings.Replace(name, "::", "$", count-1) - name = strings.Replace(name, "::", ".", 1) - return name - }, - "tstype": func(cpptype string) string { - if strings.HasPrefix(cpptype, "math::") { - return cpptype[6:] - } - switch cpptype { - case "float", "uint8_t", "uint32_t", "uint16_t": - return "number" - case "bool": - return "boolean" - case "LinearColorA": - return "float4" - case "LinearColor": - return "float3" - } - return jsPrefix + strings.ReplaceAll(cpptype, "::", "$") - }, - "jsprefix": func() string { return jsPrefix }, - "cprefix": func() string { return cppPrefix }, - "classprefix": func() string { return classPrefix }, - "docblock": func(defn parse.Documented, depth int) string { - doc := defn.GetDoc() - if doc == "" { - return "" - } - indent := strings.Repeat(" ", depth) - if strings.Count(doc, "\n") > 0 { - return strings.ReplaceAll(doc, "\n", "\n"+indent) - } - return "/**\n" + indent + " * " + doc + "\n" + indent + " */\n" + indent - }, - } +const kCodelineMarker = "The remainder of this file is generated by beamsplitter" - templ := template.New("beamsplitter").Funcs(customExtensions) - templ = template.Must(templ.ParseFiles("javascript.template")) - return func(file *os.File, section string, definition parse.TypeDefinition) { - err := templ.ExecuteTemplate(file, section, definition) - if err != nil { - log.Fatal(err.Error()) - } - } -} - -func emitJavaScript(definitions []parse.TypeDefinition, namespace string, outputFolder string) { +func EmitJavaScript(definitions []db.TypeDefinition, namespace string, outputFolder string) { generate := createJsCodeGenerator(namespace) { path := filepath.Join(outputFolder, "jsbindings_generated.cpp") @@ -120,7 +44,7 @@ func emitJavaScript(definitions []parse.TypeDefinition, namespace string, output for _, definition := range definitions { switch definition.(type) { - case *parse.StructDefinition: + case *db.StructDefinition: generate(file, "JsBindingsStruct", definition) } } @@ -139,7 +63,7 @@ func emitJavaScript(definitions []parse.TypeDefinition, namespace string, output for _, definition := range definitions { switch definition.(type) { - case *parse.EnumDefinition: + case *db.EnumDefinition: generate(file, "JsEnum", definition) } } @@ -159,7 +83,7 @@ func emitJavaScript(definitions []parse.TypeDefinition, namespace string, output for _, definition := range definitions { switch definition.(type) { - case *parse.StructDefinition: + case *db.StructDefinition: generate(file, "JsExtension", definition) } } @@ -167,7 +91,7 @@ func emitJavaScript(definitions []parse.TypeDefinition, namespace string, output } } -func editTypeScript(definitions []parse.TypeDefinition, namespace string, folder string) { +func EditTypeScript(definitions []db.TypeDefinition, namespace string, folder string) { path := filepath.Join(folder, "filament.d.ts") var codelines []string { @@ -207,10 +131,88 @@ func editTypeScript(definitions []parse.TypeDefinition, namespace string, folder generate := createJsCodeGenerator(namespace) for _, definition := range definitions { switch definition.(type) { - case *parse.StructDefinition: + case *db.StructDefinition: generate(file, "TsStruct", definition) - case *parse.EnumDefinition: + case *db.EnumDefinition: generate(file, "TsEnum", definition) } } } + +// Returns a templating function that automatically checks for fatal errors. The returned function +// takes an output stream, a template name to invoke, and a template context object. +func createJsCodeGenerator(namespace string) func(*os.File, string, db.TypeDefinition) { + jsPrefix := "" + classPrefix := "" + cppPrefix := "" + if namespace != "" { + jsPrefix = namespace + "$" + classPrefix = namespace + ".prototype." + cppPrefix = namespace + "::" + } + // These template extensions are used to transmogrify C++ symbols and value literals into + // JavaScript. We mostly don't need to do anything since the parser has already done some + // massaging and verification (e.g. it removed the trailing "f" from floating point literals). + // However enums need some special care here. Emscripten bindings are flat, so our own + // convention is to use $ for the scoping delimiter, which is a legal symbol character in JS. + // However we still use . to separate the enum value from the enum type, because emscripten has + // first-class support for class enums. + customExtensions := template.FuncMap{ + "qualifiedtype": func(typename string) string { + typename = strings.ReplaceAll(typename, "::", "$") + return typename + }, + "flag": func(field *db.StructField, flag string) bool { + _, exists := field.EmitterFlags[flag] + return exists + }, + "qualifiedvalue": func(name string) string { + count := strings.Count(name, "::") + if count > 0 { + name = "Filament." + jsPrefix + name + } + name = strings.Replace(name, "::", "$", count-1) + name = strings.Replace(name, "::", ".", 1) + return name + }, + "tstype": func(cpptype string) string { + if strings.HasPrefix(cpptype, "math::") { + return cpptype[6:] + } + switch cpptype { + case "float", "uint8_t", "uint32_t", "uint16_t": + return "number" + case "bool": + return "boolean" + case "LinearColorA": + return "float4" + case "LinearColor": + return "float3" + } + return jsPrefix + strings.ReplaceAll(cpptype, "::", "$") + }, + "jsprefix": func() string { return jsPrefix }, + "cprefix": func() string { return cppPrefix }, + "classprefix": func() string { return classPrefix }, + "docblock": func(defn db.Documented, depth int) string { + doc := defn.GetDoc() + if doc == "" { + return "" + } + indent := strings.Repeat(" ", depth) + if strings.Count(doc, "\n") > 0 { + return strings.ReplaceAll(doc, "\n", "\n"+indent) + } + return "/**\n" + indent + " * " + doc + "\n" + indent + " */\n" + indent + }, + } + + templ := template.New("beamsplitter").Funcs(customExtensions) + templ = template.Must(templ.ParseFiles("emitters/javascript.template")) + return func(file *os.File, section string, definition db.TypeDefinition) { + err := templ.ExecuteTemplate(file, section, definition) + if err != nil { + log.Fatal(err.Error()) + } + } +} diff --git a/tools/beamsplitter/javascript.template b/tools/beamsplitter/emitters/javascript.template similarity index 100% rename from tools/beamsplitter/javascript.template rename to tools/beamsplitter/emitters/javascript.template diff --git a/tools/beamsplitter/serializer.go b/tools/beamsplitter/emitters/serializer.go similarity index 85% rename from tools/beamsplitter/serializer.go rename to tools/beamsplitter/emitters/serializer.go index 92750af17a..fc540ece83 100644 --- a/tools/beamsplitter/serializer.go +++ b/tools/beamsplitter/emitters/serializer.go @@ -14,10 +14,10 @@ * limitations under the License. */ -package main +package emitters import ( - "beamsplitter/parse" + db "beamsplitter/database" "fmt" "log" "os" @@ -25,7 +25,7 @@ import ( "text/template" ) -func emitSerializer(definitions []parse.TypeDefinition, outputFolder string) { +func EmitSerializer(definitions []db.TypeDefinition, outputFolder string) { // The following template extensions make it possible to generate valid C++ code with // fewer if-then-else blocks in the template file. customExtensions := template.FuncMap{ @@ -35,7 +35,7 @@ func emitSerializer(definitions []parse.TypeDefinition, outputFolder string) { } return "," }, - "flag": func(field *parse.StructField, flag string) bool { + "flag": func(field *db.StructField, flag string) bool { _, exists := field.EmitterFlags[flag] return exists }, @@ -63,9 +63,9 @@ func emitSerializer(definitions []parse.TypeDefinition, outputFolder string) { } codegen := template.New("beamsplitter").Funcs(customExtensions) - codegen = template.Must(codegen.ParseFiles("serializer.template")) + codegen = template.Must(codegen.ParseFiles("emitters/serializer.template")) - generate := func(file *os.File, section string, definition parse.TypeDefinition) { + generate := func(file *os.File, section string, definition db.TypeDefinition) { err := codegen.ExecuteTemplate(file, section, definition) if err != nil { log.Fatal(err.Error()) @@ -83,10 +83,10 @@ func emitSerializer(definitions []parse.TypeDefinition, outputFolder string) { generate(file, "CppHeader", nil) for _, definition := range definitions { switch definition.(type) { - case *parse.StructDefinition: + case *db.StructDefinition: generate(file, "CppStructReader", definition) generate(file, "CppStructWriter", definition) - case *parse.EnumDefinition: + case *db.EnumDefinition: generate(file, "CppEnumReader", definition) generate(file, "CppEnumWriter", definition) } @@ -104,9 +104,9 @@ func emitSerializer(definitions []parse.TypeDefinition, outputFolder string) { generate(file, "HppHeader", nil) for _, definition := range definitions { switch definition.(type) { - case *parse.StructDefinition: + case *db.StructDefinition: generate(file, "HppStruct", definition) - case *parse.EnumDefinition: + case *db.EnumDefinition: generate(file, "HppEnum", definition) } } diff --git a/tools/beamsplitter/serializer.template b/tools/beamsplitter/emitters/serializer.template similarity index 100% rename from tools/beamsplitter/serializer.template rename to tools/beamsplitter/emitters/serializer.template diff --git a/tools/beamsplitter/main.go b/tools/beamsplitter/main.go index 02f60d971e..ec273ec1ee 100644 --- a/tools/beamsplitter/main.go +++ b/tools/beamsplitter/main.go @@ -23,10 +23,12 @@ import ( "path/filepath" "runtime" + db "beamsplitter/database" + "beamsplitter/emitters" "beamsplitter/parse" ) -const kCodelineMarker = "The remainder of this file is generated by beamsplitter" +var CHECK_API_SYNTAX = false func findFilamentRoot() string { var ( @@ -43,36 +45,25 @@ func main() { log.SetPrefix(sourceFilename + ":") root := findFilamentRoot() sourcePath := filepath.Join(root, "filament", "include", "filament", sourceFilename) - definitions := parse.Parse(sourcePath) - // For diagnostic purposes, this dumps out the database that was gathered from - // the parsing phase. - if len(os.Args) > 1 && os.Args[1] == "--verbose" { - for _, defn := range definitions { - switch concrete := defn.(type) { - case *parse.StructDefinition: - fmt.Println("STRUCT:", concrete.QualifiedName()) - for _, field := range concrete.Fields { - fmt.Println("\t", field.TypeString, "...", field.Name, "...", field.DefaultValue) - } - case *parse.EnumDefinition: - fmt.Println(" ENUM:", concrete.QualifiedName()) - for _, value := range concrete.Values { - fmt.Println("\t", value) - } - } - } + data, err := os.ReadFile(sourcePath) + if err != nil { + log.Fatal(err) } - emitSerializer(definitions, filepath.Join(root, "libs", "viewer", "src")) + contents := string(data) + ast := parse.Parse(contents) + definitions := db.Create(ast, contents) + + emitters.EmitSerializer(definitions, filepath.Join(root, "libs", "viewer", "src")) jsfolder := filepath.Join(root, "web", "filament-js") - emitJavaScript(definitions, "View", jsfolder) - editTypeScript(definitions, "View", jsfolder) + emitters.EmitJavaScript(definitions, "View", jsfolder) + emitters.EditTypeScript(definitions, "View", jsfolder) javafolder := filepath.FromSlash("com/google/android/filament") javafolder = filepath.Join(root, "android/filament-android/src/main/java", javafolder) - editJava(definitions, "View", javafolder) + emitters.EditJava(definitions, "View", javafolder) fmt.Print(` Note that this tool does not generate bindings for setter methods on @@ -88,4 +79,22 @@ will likely need to manually modify the following files: - android/filament-android/src/main/java/.../View.java - android/filament-android/src/main/cpp/View.cpp `) + + if CHECK_API_SYNTAX { + sources := []string{} + apiPath := filepath.Join(root, "filament", "include", "filament") + entries, err := os.ReadDir(apiPath) + for _, entry := range entries { + sources = append(sources, filepath.Join(apiPath, entry.Name())) + } + for _, source := range sources { + log.SetPrefix(filepath.Base(source) + ":") + data, err = os.ReadFile(source) + if err != nil { + log.Fatal(err) + } + contents = string(data) + parse.Parse(contents) + } + } } diff --git a/tools/beamsplitter/parse/ast.go b/tools/beamsplitter/parse/ast.go new file mode 100644 index 0000000000..0f88f95e32 --- /dev/null +++ b/tools/beamsplitter/parse/ast.go @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package parse + +type Node interface{} + +type RootNode struct { + Line int + Child *NamespaceNode +} + +type NamespaceNode struct { + Line int + Name string + Children []Node +} + +type ClassNode struct { + Line int + Name string + Members []Node + IsTemplate bool +} + +type StructNode struct { + Line int + Name string + Members []Node + IsTemplate bool +} + +type EnumNode struct { + Line int + Name string + Values []string + ValueLines []int // used to find docstring for each enum value +} + +type UsingNode struct { + Line int + Name string + Rhs string +} + +type AccessSpecifierNode struct { + Line int + Access string +} + +type MethodNode struct { + Line int + Name string + ReturnType string + Arguments string + Body string + IsTemplate bool +} + +type FieldNode struct { + Line int + Name string + Type string + Rhs string + ArrayLength int +} diff --git a/tools/beamsplitter/parse/lex.go b/tools/beamsplitter/parse/lex.go new file mode 100644 index 0000000000..40c2df5523 --- /dev/null +++ b/tools/beamsplitter/parse/lex.go @@ -0,0 +1,802 @@ +/* + * Copyright (C) 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package parse + +import ( + "errors" + "fmt" + "log" + "strings" + "unicode" + "unicode/utf8" +) + +var VERBOSE_LEXER = false +var KNOWN_MACROS = []string{"UTILS_PUBLIC"} + +// item represents a token or text string returned from the scanner. +type item struct { + typ itemType // The type of this item. + pos int // The starting position, in bytes, of this item in the input string. + val string // The value of this item. + line int // The line number at the start of this item. +} + +func (i item) String() string { + dbg := strings.ReplaceAll(i.val, "\n", "\\n") + switch { + case i.typ == itemEOF: + return "EOF" + case i.typ == itemError: + return i.val + case i.typ > itemKeywords_: + return dbg + case len(i.val) > 30: + return dbg[:10] + "..." + dbg[len(dbg)-10:] + } + return dbg +} + +// itemType identifies the type of lex items. +type itemType int + +const ( + itemError itemType = iota // error occurred; value is text of error + itemSimpleType // examples: `Texture* const`, `uint8_t`, `BlendMode` + itemMethodBody // unparsed blob, includes outermost { and } + itemMethodArgs // unparsed blob, includes outermost ( and ) + itemTemplateArgs // unparsed blob, includes outermost < and > + itemDefaultValue // the unparsed RHS of an expression + itemIdentifier // legal C++ identifier + itemEOF + + itemCloseBrace + itemColon + itemComma + itemEquals + itemOpenBrace + itemSemicolon + itemOpenBracket + itemCloseBracket + itemArrayLength + + itemKeywords_ // unused enum separator + itemClass + itemConst + itemConstexpr + itemEnum + itemFriend + itemNamespace + itemNoexcept + itemPrivate + itemProtected + itemPublic + itemStruct + itemTemplate + itemUsing +) + +const eof = -1 + +// lexer holds the state of the scanner. +type lexer struct { + input string // the entire contents of the file being scanned + items chan item // channel of scanned items + line int // 1+number of newlines seen + startLine int // start line of this item + pos int // current position in the input + start int // start position of this item + atEOF bool // we have hit the end of input + lookahead []item +} + +// next returns the next rune in the input. +func (l *lexer) next() rune { + if int(l.pos) >= len(l.input) { + l.atEOF = true + return eof + } + r, w := utf8.DecodeRuneInString(l.input[l.pos:]) + l.pos += w + if r == '\n' { + l.line++ + } + return r +} + +// backup steps back one rune. +func (l *lexer) backup() { + if !l.atEOF && l.pos > 0 { + r, w := utf8.DecodeLastRuneInString(l.input[:l.pos]) + l.pos -= w + // Correct newline count. + if r == '\n' { + l.line-- + } + } +} + +func (lex *lexer) backupMultiple(count int) { + for i := 0; i < count; i++ { + lex.backup() + } +} + +func (lex *lexer) backupKeyword() { + lex.backup() + for unicode.IsLetter(rune(lex.input[lex.pos])) { + lex.backup() + } + lex.next() +} + +// Passes an item back to the parser and moves the start pointer to catch up to the cursor. +// The start pointer marks the beginning of the next item. +// It helps to imagine a crawling worm whose tail catches up with its head in one movement. +func (l *lexer) emit(t itemType) { + text := strings.TrimSpace(l.input[l.start:l.pos]) + if text == "" { + column := l.pos - l.findLineStartPos() + // Can occur when calling emit() twice in a row without "accepting" anything in between. + log.Fatalf("%d: internal error at column %d\n", l.line, column) + } + token := item{t, l.start, text, l.startLine} + if l.lookahead != nil { + if VERBOSE_LEXER { + fmt.Printf("%03d Stashing %s\n", token.line, token.String()) + } + l.lookahead = append(l.lookahead, token) + } else { + if VERBOSE_LEXER { + fmt.Printf("%03d Emitting %s\n", token.line, token.String()) + } + l.items <- token + } + l.start = l.pos + l.startLine = l.line + l.discardOptionalSpace() +} + +// Moves the start pointer to catch up to the cursor but does not pass the item to the parser. +// This function is like emit, except that it throws the item in the trash. +func (l *lexer) discard() { + if VERBOSE_LEXER { + dbg := l.input[l.start:l.pos] + dbg = strings.ReplaceAll(dbg, "\n", "\\n") + if len(dbg) > 30 { + dbg = dbg[:10] + "..." + dbg[len(dbg)-10:] + } + fmt.Printf("%03d Trashing [[%s]]\n", l.line, dbg) + } + l.start = l.pos + l.startLine = l.line +} + +func (l *lexer) acceptAny(valid string) bool { + if strings.ContainsRune(valid, l.next()) { + return true + } + l.backup() + return false +} + +func (lex *lexer) acceptAlphaNumeric() bool { + next := lex.next() + if isAlphaNumeric(next) { + return true + } + lex.backup() + return false +} + +func (lex *lexer) acceptPositiveInteger() bool { + next := lex.next() + if !unicode.IsDigit(next) || next == '0' { + lex.backup() + return false + } + lex.acceptRun("123456789") + return true +} + +// Consumes a run of runes from the valid set. +func (l *lexer) acceptRun(valid string) { + for strings.ContainsRune(valid, l.next()) { + } + l.backup() +} + +func (lex *lexer) acceptSpace() bool { + return lex.acceptAny(" \t\n") +} + +func (l *lexer) acceptRune(expected rune) bool { + if l.next() == expected { + return true + } + l.backup() + return false +} + +func (lex *lexer) acceptString(expectedString string) bool { + for i, c := range expectedString { + if lex.next() != c { + lex.backupMultiple(i + 1) + return false + } + } + return true +} + +func (lex *lexer) acceptIdentifier() bool { + next := lex.next() + if next != '_' && !unicode.IsLetter(next) { + lex.backup() + return false + } + for isAlphaNumeric(lex.next()) { + } + lex.backup() + return true +} + +func (lex *lexer) acceptKeyword(keyword string) bool { + start := lex.pos + if !lex.acceptString(keyword) { + return false + } + if lex.eof() { + return true + } + if lex.acceptAlphaNumeric() { + lex.backupMultiple(lex.pos - start) + return false + } + return true +} + +func (lex *lexer) acceptKeywords(keywords []string) bool { + for _, keyword := range keywords { + if lex.acceptKeyword(keyword) { + return true + } + } + return false +} + +// Accepts a blob of unparsed text up until the given terminator, which is not included. +func (lex *lexer) acceptTerminatedBlob(term rune) bool { + previous := lex.pos + for { + if lex.next() == term { + lex.backup() + return true + } + if lex.atEOF { + lex.backupMultiple(lex.pos - previous) + return false + } + } +} + +// Accepts a blob of unparsed text with the given delimiters, which can be nested. +func (lex *lexer) acceptDelimitedBlob(start rune, stop rune) bool { + previous := lex.pos + next := lex.next() + if next != start { + lex.backup() + return false + } + for depth := 1; next != eof; { + switch lex.next() { + case start: + depth++ + case stop: + depth-- + if depth == 0 { + return true + } + } + } + lex.backupMultiple(lex.pos - previous) + return false +} + +// Discards whitespace, linefeeds, comments, and C preprocessor directives. +func (lex *lexer) discardOptionalSpace() { + for { + for _, macro := range KNOWN_MACROS { + if lex.acceptKeyword(macro) { + lex.discard() + } + } + switch { + case lex.acceptSpace(): + lex.acceptRun(" \n\t") + lex.discard() + case lex.acceptString("/*"): + for !lex.acceptString("*/") { + lex.next() + } + lex.discard() + case lex.acceptString("//") || lex.acceptRune('#'): + for !lex.acceptRune('\n') { + lex.next() + } + lex.discard() + default: + return + } + } +} + +// Returns the next item from the input. +// Called by the parser, not in the lexing goroutine. +func (l *lexer) nextItem() item { + return <-l.items +} + +// Drains the output so the lexing goroutine will exit. +// Called by the parser, not in the lexing goroutine. +func (l *lexer) drain() { + for range l.items { + } +} + +func (lex *lexer) eof() bool { + return lex.pos >= len(lex.input) +} + +// Creates a new scanner for the input string. +func createLexer(input string) *lexer { + l := &lexer{ + input: input, + items: make(chan item), + line: 1, + startLine: 1, + } + go l.run() + return l +} + +func (lex *lexer) run() { + if err := lexRoot(lex); err != nil { + column := lex.pos - lex.findLineStartPos() + log.Fatalf("%d:%d: lexer expected %s", lex.line, column, err.Error()) + } + close(lex.items) +} + +func (lex *lexer) findLineStartPos() int { + lineNo := 1 + for pos, c := range lex.input { + if c != '\n' { + continue + } + lineNo++ + if lineNo == lex.line { + return pos + } + } + return 0 +} + +func isAlphaNumeric(r rune) bool { + return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) +} + +// Causes the lexer to enter a special "lookahead state" in which it stashes +// the lexer state and buffers all emitted items. +func lookahead(lex *lexer, cb func(*lexer) error) bool { + if lex.lookahead != nil { + log.Fatal("Nested lookahead.") + } + lex.lookahead = make([]item, 0) + previousStart := lex.start + previousStartLine := lex.startLine + previousPos := lex.pos + previousLine := lex.line + if err := cb(lex); err == nil { + for _, item := range lex.lookahead { + if VERBOSE_LEXER { + fmt.Printf("%03d Emitting %s\n", item.line, item.String()) + } + lex.items <- item + } + lex.lookahead = nil + return true + } + lex.lookahead = nil + lex.start = previousStart + lex.startLine = previousStartLine + lex.line = previousLine + lex.pos = previousPos + return false +} + +// ------------------------------------------------------------------------------------------------- +// The remainder of this file has one function for each nonterminal in the BNF. +// The ordering should be consistent with the grammar in the README. +// These functions are all prefixed with "lex" and return an error or nil. +// Error strings are automatically prefixed with the line number and "lexer expected ". +// In lookahead situations, the returned error might be intentionally discarded. +// ------------------------------------------------------------------------------------------------- + +func lexRoot(lex *lexer) error { + if lex.eof() { + return nil + } + lex.discardOptionalSpace() + if lex.acceptKeyword("namespace") { + return lexNamespace(lex) + } + return errors.New("namespace") +} + +// Assumptions: cursor is just past the namespace keyword +// Directly emits: keyword, the optional identifier, opening and closing braces +// Indirectly emits: content of the namespace +func lexNamespace(lex *lexer) error { + lex.emit(itemNamespace) + if lex.acceptIdentifier() { + lex.emit(itemIdentifier) + } + if !lex.acceptRune('{') { + return errors.New("{") + } + lex.emit(itemOpenBrace) + for !lex.acceptRune('}') { + if err := lexBlock(lex); err != nil { + return err + } + } + lex.emit(itemCloseBrace) + return nil +} + +// Assumptions: cursor is just before one of the block keywords +// Directly emits: struct or class keywords +// Indirectly emits: the block keyword, its braces, contents, semicolons +func lexBlock(lex *lexer) error { + switch { + case lex.acceptKeyword("namespace"): + return lexNamespace(lex) + case lex.acceptKeyword("template"): + lex.emit(itemTemplate) + if !lex.acceptDelimitedBlob('<', '>') { + return errors.New("template arguments") + } + lex.emit(itemTemplateArgs) + if !lex.acceptKeywords([]string{"class", "struct"}) { + return errors.New("class or struct") + } + lex.backupKeyword() + return lexBlock(lex) + case lex.acceptKeyword("struct"): + lex.emit(itemStruct) + if lookahead(lex, lexForwardDeclaration) { + return nil + } + return lexStruct(lex) + case lex.acceptKeyword("class"): + lex.emit(itemClass) + if lookahead(lex, lexForwardDeclaration) { + return nil + } + return lexClass(lex) + case lex.acceptKeyword("using"): + return lexUsing(lex) + case lex.acceptKeyword("enum"): + return lexEnum(lex) + } + return errors.New("namespace, struct, class, enum, or using") +} + +func lexForwardDeclaration(lex *lexer) error { + if !lex.acceptIdentifier() { + return errors.New("identifier") + } + lex.emit(itemIdentifier) + if !lex.acceptRune(';') { + return errors.New("; after forward declaration") + } + lex.emit(itemSemicolon) + return nil +} + +// Assumptions: cursor is just past the class keyword +// Directly emits: identifier, opening and closing braces, semicolon +// Indirectly emits: content of the struct +func lexClass(lex *lexer) error { + if !lex.acceptIdentifier() { + return errors.New("does not allow anonymous classes") + } + lex.emit(itemIdentifier) + if lex.acceptRune(':') { + lex.emit(itemColon) + if lex.acceptKeyword("public") { + lex.emit(itemPublic) + } + if err := lexSimpleType(lex); err != nil { + return err + } + } + if !lex.acceptRune('{') { + return errors.New("{ before class body") + } + lex.emit(itemOpenBrace) + if err := lexStructBody(lex); err != nil { + return err + } + lex.emit(itemCloseBrace) + if !lex.acceptRune(';') { + return errors.New(": after class") + } + lex.emit(itemSemicolon) + return nil +} + +// Assumptions: cursor is just past the struct keyword +// Directly emits: the optional identifier, opening and closing braces, semicolon +// Indirect emits: content of the struct +func lexStruct(lex *lexer) error { + if lex.acceptIdentifier() { + lex.emit(itemIdentifier) + } + if !lex.acceptRune('{') { + return errors.New("{ before struct body") + } + lex.emit(itemOpenBrace) + if err := lexStructBody(lex); err != nil { + return err + } + lex.emit(itemCloseBrace) + if !lex.acceptRune(';') { + return errors.New("; after struct body") + } + lex.emit(itemSemicolon) + return nil +} + +// Assumptions: cursor is just past the enum keyword +// Directly emits: all parts of the enum, including the trailing semicolon +func lexEnum(lex *lexer) error { + lex.emit(itemEnum) + if !lex.acceptKeyword("class") { + return errors.New("class-style enum") + } + lex.emit(itemClass) + if !lex.acceptIdentifier() { + return errors.New("valid identifier (anonymous enums are not supported)") + } + lex.emit(itemIdentifier) + if lex.acceptRune(':') { + lex.emit(itemColon) + if err := lexSimpleType(lex); err != nil { + return err + } + } + if !lex.acceptRune('{') { + return errors.New("{ before enum definition") + } + lex.emit(itemOpenBrace) + + if !lex.acceptIdentifier() { + return errors.New("at least one value in the enum") + } + lex.emit(itemIdentifier) + + for !lex.acceptRune('}') { + if !lex.acceptRune(',') { + return errors.New(", between enum values") + } + lex.emit(itemComma) + if lex.acceptRune('}') { + break + } + if !lex.acceptIdentifier() { + return errors.New("valid identifier in enum") + } + lex.emit(itemIdentifier) + } + lex.emit(itemCloseBrace) + if !lex.acceptRune(';') { + return errors.New("; after enum definition") + } + lex.emit(itemSemicolon) + return nil +} + +func lexUsing(lex *lexer) error { + lex.emit(itemUsing) + if !lex.acceptIdentifier() { + return errors.New("valid identifier in type alias") + } + lex.emit(itemIdentifier) + if !lex.acceptRune('=') { + return errors.New("= in type alias") + } + lex.emit(itemEquals) + if err := lexSimpleType(lex); err != nil { + return err + } + + if !lex.acceptRune(';') { + return errors.New("; after type alias") + } + lex.emit(itemSemicolon) + return nil +} + +// Assumptions: cursor is just past the opening brace of a struct or class +// Directly emits: nothing +// Indirect emits: entire content of the struct or class, but not the outer braces +func lexStructBody(lex *lexer) error { + accessKeywords := []string{"public", "private", "protected"} + blockKeywords := []string{"class", "struct", "enum", "using", "template", "namespace"} + for { + switch { + case lex.acceptRune('}'): + return nil + case lex.acceptKeywords(accessKeywords): + lex.backupKeyword() + if err := lexAccessSpecifier(lex); err != nil { + return err + } + case lex.acceptKeywords(blockKeywords): + lex.backupKeyword() + if err := lexBlock(lex); err != nil { + return err + } + default: + if lookahead(lex, lexMethod) { + continue + } + if err := lexField(lex); err != nil { + return err + } + } + } +} + +// Assumptions: cursor is just before one of the access keywords +// Directly emits: the access keyword and the colon +func lexAccessSpecifier(lex *lexer) error { + switch { + case lex.acceptKeyword("public"): + lex.emit(itemPublic) + case lex.acceptKeyword("protected"): + lex.emit(itemProtected) + case lex.acceptKeyword("private"): + lex.emit(itemPrivate) + default: + return errors.New("legal access specifier") + } + if !lex.acceptRune(':') { + return errors.New(": after access specifier") + } + lex.emit(itemColon) + return nil +} + +// Assumptions: cursor is just before a method declaration or implementation +// Directly emits: entire content of the method declaration or implementation +func lexMethod(lex *lexer) error { + if lex.acceptKeyword("template") { + lex.emit(itemTemplate) + if !lex.acceptDelimitedBlob('<', '>') { + return errors.New("template arguments") + } + lex.emit(itemTemplateArgs) + } + if lex.acceptKeyword("friend") { + lex.emit(itemFriend) + } + if lex.acceptKeyword("constexpr") { + lex.emit(itemConstexpr) + } + if err := lexSimpleType(lex); err != nil { + return err + } + if !lex.acceptIdentifier() { + return errors.New("valid identifier") + } + lex.emit(itemIdentifier) + if !lex.acceptDelimitedBlob('(', ')') { + return errors.New("function arguments") + } + lex.emit(itemMethodArgs) + if lex.acceptKeyword("const") { + lex.emit(itemConst) + } + if lex.acceptKeyword("noexcept") { + lex.emit(itemNoexcept) + } + if lex.acceptRune(';') { + lex.emit(itemSemicolon) + return nil + } + if !lex.acceptDelimitedBlob('{', '}') { + return errors.New("function body or ;") + } + lex.emit(itemMethodBody) + return nil +} + +// Assumptions: cursor is just before a data field declaration in a class or struct +// Directly emits: entire content of the method declaration or implementation +func lexField(lex *lexer) error { + if err := lexSimpleType(lex); err != nil { + return err + } + if !lex.acceptIdentifier() { + return errors.New("valid identifier") + } + lex.emit(itemIdentifier) + if lex.acceptRune('[') { + lex.emit(itemOpenBracket) + if !lex.acceptPositiveInteger() { + return errors.New("positive integer") + } + lex.emit(itemArrayLength) + if !lex.acceptRune(']') { + return errors.New("valid array length") + } + lex.emit(itemCloseBracket) + } + if lex.acceptRune('=') { + lex.emit(itemEquals) + if !lex.acceptTerminatedBlob(';') { + return errors.New("right-hand side of assignment terminated by ;") + } + lex.emit(itemDefaultValue) + } + if !lex.acceptRune(';') { + return errors.New("; after field") + } + lex.emit(itemSemicolon) + return nil +} + +// For now, SimpleType is a very restrictive subset of the C++ type expression language. It should +// not contain parentheses or commas, so C callbacks are not allowed unless you alias them first. +// Basically, a type is a bag of tokens that must include exactly one valid C identifier, along with +// a mix of spaces, "*", "&", "::", "const", "<", ">". +// +// Assumptions: cursor is just before a type identifier +// Directly emits: itemSimpleType +func lexSimpleType(lex *lexer) error { + encounteredIdentifier := false + for { + switch { + case lex.acceptString("::"), lex.acceptRune('<'): + encounteredIdentifier = false + continue + case lex.acceptAny("*& >\t\n"): + continue + case lex.acceptKeyword("const"): + continue + case encounteredIdentifier: + lex.emit(itemSimpleType) + return nil + case lex.acceptIdentifier(): + encounteredIdentifier = true + default: + return errors.New("valid type identifier") + } + } +} diff --git a/tools/beamsplitter/parse/parse.go b/tools/beamsplitter/parse/parse.go index cf84b354fc..efa76c2346 100644 --- a/tools/beamsplitter/parse/parse.go +++ b/tools/beamsplitter/parse/parse.go @@ -17,447 +17,291 @@ package parse import ( - "bufio" - "io" "log" - "os" - "regexp" - "strings" + "strconv" ) -// Consumes a C++ header file and produces a type database. -func Parse(sourcePath string) []TypeDefinition { - sourceFile, err := os.Open(sourcePath) - if err != nil { - log.Fatal(err) - } - defer sourceFile.Close() - - // In the first pass, gather all block-style comments. - context := parserContext{} - context.commentBlocks = gatherCommentBlocks(sourceFile) - sourceFile.Seek(0, 0) - - // In the second pass, pry apart each C++ codeline. - lineScanner := bufio.NewScanner(sourceFile) - for lineNumber := 1; lineScanner.Scan(); lineNumber++ { - context.scanCppCodeline(lineScanner.Text(), lineNumber) - } - if err := lineScanner.Err(); err != nil { - log.Fatal(err) - } - context.addTypeQualifiers() - return context.definitions +// Consumes the entire content of a C++ header file and produces an abstract syntax tree. +func Parse(contents string) *RootNode { + lexer := createLexer(contents) + return parseRoot(lexer) } -type TypeDefinition interface { - BaseName() string - QualifiedName() string - Parent() TypeDefinition -} - -type StructField struct { - TypeString string - Name string - DefaultValue string - Description string - LineNumber int - EmitterFlags map[string]struct{} - CustomType TypeDefinition -} - -type StructDefinition struct { - name string - qualifier string - Fields []StructField - Description string - parent TypeDefinition -} - -func (defn StructDefinition) BaseName() string { return defn.name } -func (defn StructDefinition) QualifiedName() string { return defn.qualifier + defn.name } -func (defn StructDefinition) Parent() TypeDefinition { return defn.parent } - -type EnumValue struct { - Description string - Name string -} - -type EnumDefinition struct { - name string - qualifier string - Values []EnumValue - Description string - parent TypeDefinition -} - -func (defn EnumDefinition) BaseName() string { return defn.name } -func (defn EnumDefinition) QualifiedName() string { return defn.qualifier + defn.name } -func (defn EnumDefinition) Parent() TypeDefinition { return defn.parent } - -type Documented interface{ GetDoc() string } - -func (defn EnumDefinition) GetDoc() string { return defn.Description } -func (defn StructDefinition) GetDoc() string { return defn.Description } -func (field StructField) GetDoc() string { return field.Description } -func (value EnumValue) GetDoc() string { return value.Description } - -type generalScope struct{} - -type scope interface { - BaseName() string - QualifiedName() string - Parent() TypeDefinition -} - -func (defn generalScope) BaseName() string { return "" } -func (defn generalScope) QualifiedName() string { return "" } -func (defn generalScope) Parent() TypeDefinition { return nil } - -type parserContext struct { - definitions []TypeDefinition - stack []scope - insideComment bool - commentBlocks map[int]string - cppTokenizer *regexp.Regexp - floatMatcher *regexp.Regexp - vectorMatcher *regexp.Regexp - fieldParser *regexp.Regexp - fieldDescParser *regexp.Regexp - customFlagFinder *regexp.Regexp -} - -// https://github.com/google/re2/wiki/Syntax -func (context *parserContext) compileRegexps() { - context.cppTokenizer = regexp.MustCompile(`((?:/\*)|(?:\*/)|(?:;)|(?://)|(?:\})|(?:\{))`) - context.floatMatcher = regexp.MustCompile(`(\-?[0-9]+\.[0-9]*)f?`) - context.vectorMatcher = regexp.MustCompile(`\{(\s*\-?[0-9\.]+\s*(,\s*\-?[0-9\.]+\s*){1,})\}`) - context.customFlagFinder = regexp.MustCompile(`\s*\%codegen_([a-zA-Z0-9_]+)\%\s*`) - - const kFieldType = `(?P.*)` - const kFieldName = `(?P[A-Za-z0-9_]+)` - const kFieldValue = `(?P(.*?))` - const kFieldDesc = `(?://\s*\!\<\s*(?P.*))?` - context.fieldParser = regexp.MustCompile( - `^\s*` + kFieldType + `\s+` + kFieldName + `\s*=\s*` + kFieldValue + `\s*;\s*` + kFieldDesc) - - context.fieldDescParser = regexp.MustCompile(`(?://\s*\!\<\s*(.*))`) -} - -// Creates a mapping from line numbers to strings, where the strings are entire block comments -// and the line numbers correspond to the last line of each block comment. -func gatherCommentBlocks(sourceFile io.Reader) map[int]string { - comments := make(map[int]string) - scanner := bufio.NewScanner(sourceFile) - var comment = "" - var indention = 0 - for lineNumber := 1; scanner.Scan(); lineNumber++ { - codeline := scanner.Text() - if strings.Contains(codeline, `/**`) { - indention = strings.Index(codeline, `/**`) - if strings.Contains(codeline, `*/`) { - comments[lineNumber] = codeline[indention:] + "\n" - continue - } - comment = codeline[indention:] + "\n" - continue - } - if comment != "" { - if len(codeline) > indention { - codeline = codeline[indention:] - } - comment += codeline + "\n" - if strings.Contains(codeline, `*/`) { - comments[lineNumber] = comment - comment = "" - } - } - } - if err := scanner.Err(); err != nil { - log.Fatal(err) - } - return comments -} - -func (context parserContext) generateQualifier() string { - qualifier := "" - for _, defn := range context.stack { - if defn.BaseName() != "" { - qualifier = qualifier + defn.BaseName() + "::" - } - } - return qualifier -} - -func (context parserContext) findParent() TypeDefinition { - for i := len(context.stack) - 1; i >= 0; i-- { - switch defn := context.stack[i].(type) { - case *StructDefinition, *EnumDefinition: - var result TypeDefinition = defn - return result - } - } - return nil -} - -// Annotates struct fields that have custom types (i.e. enums or structs). -func (context parserContext) addTypeQualifiers() { - typeMap := make(map[string]TypeDefinition) - for _, defn := range context.definitions { - typeMap[defn.QualifiedName()] = defn - } - for _, defn := range context.definitions { - structDefn, isStruct := defn.(*StructDefinition) - if !isStruct { - continue - } - for fieldIndex, field := range structDefn.Fields { - // Extract the namespace prefix (if any) explicitly specified for this field type. - var namespace string - localTypeName := field.TypeString - if index := strings.LastIndex(field.TypeString, "::"); index > -1 { - namespace = field.TypeString[:index] - localTypeName = field.TypeString[index+2:] - } - - // Prepend additional qualifiers to the type string by searching upward through - // the current namespace hierarchy, and looking for a match. - mutable := &structDefn.Fields[fieldIndex] - for ancestor := defn; ; ancestor = ancestor.Parent() { - var qualified string - if namespace != "" && strings.HasSuffix(ancestor.QualifiedName(), namespace) { - qualified = ancestor.QualifiedName() + "::" + localTypeName - } else { - qualified = ancestor.QualifiedName() + "::" + field.TypeString - } - if fieldType, found := typeMap[qualified]; found { - mutable.TypeString = qualified - mutable.CustomType = fieldType - break - } - if ancestor.Parent() == nil { - if fieldType, found := typeMap[field.TypeString]; found { - mutable.CustomType = fieldType - } - break - } - } - - if mutable.CustomType == nil { - continue - } - - // Prepend additional qualifiers to the value string if it is a known enum. - var fieldType TypeDefinition = structDefn.Fields[fieldIndex].CustomType - enumDefn, isEnum := fieldType.(*EnumDefinition) - if isEnum { - selectedEnum := field.DefaultValue - if index := strings.LastIndex(field.DefaultValue, "::"); index > -1 { - selectedEnum = field.DefaultValue[index+2:] - } - mutable.DefaultValue = enumDefn.QualifiedName() + "::" + selectedEnum - } - } - } -} - -// Validates and transforms the RHS of an assignment. -// For vectors, this converts curly braces into square brackets. -func (context parserContext) distillValue(cppvalue string, lineNumber int) string { - cppvalue = strings.TrimSpace(cppvalue) - - // Remove trailing "f" from floats, which isn't allowed in JavaScript. - if context.floatMatcher.MatchString(cppvalue) { - cppvalue = context.floatMatcher.ReplaceAllString(cppvalue, "$1") +// Assumes that we have just consumed the open brace from the lexer. +// This consumes all lexemes in the entire struct, including the outer end brace. +// It does not consume anything after the end brace. +func parseStructBody(lex *lexer) []Node { + var members []Node + append := func(node Node) { + members = append(members, node) } - // There are many ways to declare vector values (multi-arg constructor, single-arg - // constructor, curly braces with type, curly braces without type), so just poop out if - // the syntax is anything other than "curly braces without type". - if strings.Contains(cppvalue, "math::") || strings.Contains(cppvalue, "Color") { - log.Fatalf("%d: vectors must have the form {x, y ...}", lineNumber) + // Assumes that we have just consumed the end paren in the argument list. + parseMethod := func(name, returns, args item, isTemplate bool) *MethodNode { + item := lex.nextItem() + for item.typ == itemConst || item.typ == itemNoexcept { + item = lex.nextItem() + } + for item.typ == itemConst || item.typ == itemNoexcept { + item = lex.nextItem() + } + method := &MethodNode{ + Line: item.line, + Name: name.val, + ReturnType: returns.val, + Arguments: args.val, + Body: "", + IsTemplate: isTemplate, + } + switch item.typ { + case itemMethodBody: + method.Body = item.val + case itemSemicolon: + default: + panic(lex, item) + return nil + } + return method } - - // Assume it's a vector if there's a curly brace. - if strings.Contains(cppvalue, "{") { - if context.vectorMatcher.MatchString(cppvalue) { - cppvalue = context.vectorMatcher.ReplaceAllString(cppvalue, "[$1]") - } else { - log.Fatalf("%d: vectors must have the form {x, y ...}", lineNumber) - } - } - - if len(cppvalue) == 0 { - log.Fatalf("%d: empty value specified", lineNumber) - } - - return cppvalue -} - -func (context *parserContext) scanCppCodeline(codeline string, lineNumber int) { - if context.cppTokenizer == nil { - context.compileRegexps() - } - - // For "group begin" or "group end" comments inside a struct, emit a faux field. - commentBlock := context.commentBlocks[lineNumber] - if strings.Contains(commentBlock, "@{") || strings.Contains(commentBlock, "@}") { - depth := len(context.stack) - 1 - if depth > 0 { - _, insideStruct := context.stack[depth].(*StructDefinition) - if insideStruct { - // TODO: support group comments - log.Printf("%d: Grouping comments are ignored.\n", lineNumber) - } - } - } - - codeline = context.cppTokenizer.ReplaceAllString(codeline, " $1 ") - scanner := bufio.NewScanner(strings.NewReader(codeline)) - scanner.Split(bufio.ScanWords) - inPlaceDefinition := "" - - scanStructField := func(defn *StructDefinition, firstWord string) { - var field = StructField{ - LineNumber: lineNumber, - } - - // Extract all custom flags into a string set. These are special backend-specific directives - // delimited by percent signs, e.g. %codegen_skip_javascript% - if matches := context.customFlagFinder.FindAllStringSubmatch(codeline, -1); matches != nil { - field.EmitterFlags = make(map[string]struct{}, len(matches)) - for _, flag := range matches { - field.EmitterFlags[flag[1]] = struct{}{} - } - } - codeline = context.customFlagFinder.ReplaceAllString(codeline, "") - - // Normally when we're inside a struct, the first word on each codeline is the field type, - // and the second word is the field name. However if a nested struct is defined, then the - // type is potentially anonymous and the first word is the field name. - if !scanner.Scan() { - log.Fatalf("%d: bad struct field", lineNumber) - } - - // Check if this field type has an in place definition. For example: - // struct OuterType { - // int foo; - // struct Baz { int bar } baz; - // }; - // In the above example, inPlaceDefinition == Baz. - if inPlaceDefinition != "" { - field.TypeString = inPlaceDefinition - field.Name = firstWord - defn.Fields = append(defn.Fields, field) - return - } - - if !context.fieldParser.MatchString(codeline) { - log.Fatalf("%d: unexpected form in struct field declaration", lineNumber) - } - - // To make the regex usage somewhat readable, extract the named subgroups into a map rather - // than referring to each result by index. - subexpList := context.fieldParser.FindStringSubmatch(codeline) - subexpMap := make(map[string]string) - for i, name := range context.fieldParser.SubexpNames() { - if i != 0 && name != "" { - subexpMap[name] = subexpList[i] - } - } - - field.TypeString = subexpMap["type"] - field.Name = subexpMap["name"] - field.Description = subexpMap["description"] - field.DefaultValue = context.distillValue(subexpMap["value"], lineNumber) - - if commentBlock := context.commentBlocks[lineNumber-1]; commentBlock != "" { - field.Description = commentBlock - } - - defn.Fields = append(defn.Fields, field) - } - - for scanner.Scan() { - depth := len(context.stack) - 1 - token := scanner.Text() + for item := lex.nextItem(); item.typ != itemCloseBrace; item = lex.nextItem() { switch { - case token == "//": - return - case token == "/*": - context.insideComment = true - case token == "*/": - if !context.insideComment { - log.Fatalf("%d: strange comment", lineNumber) + case item.val == "constexpr", item.val == "friend": + // do nothing for these annotations + case item.val == "enum": + append(parseEnum(lex)) + case item.val == "struct": + append(parseStruct(lex)) + case item.val == "class": + append(parseClass(lex)) + case item.val == "namespace": + append(parseNamespace(lex)) + case item.val == "using": + append(parseUsing(lex)) + case item.val == "public", item.val == "private", item.val == "protected": + expect(lex, itemColon) + append(&AccessSpecifierNode{ + Line: item.line, + Access: item.val, + }) + case item.typ == itemCloseBrace: + break + case item.typ == itemSimpleType: + name := expect(lex, itemIdentifier) + nextItem := lex.nextItem() + arrayLength := 0 + if nextItem.typ == itemOpenBracket { + arrayLength, _ = strconv.Atoi(expect(lex, itemArrayLength).val) + expect(lex, itemCloseBracket) + nextItem = lex.nextItem() } - context.insideComment = false - case context.insideComment: - // Do nothing. - case token == ";": - // Do nothing. - case token == "{": - context.stack = append(context.stack, &generalScope{}) - case token == "}": - if depth < 0 { - log.Fatalf("%d: bizarre nesting", lineNumber) + switch nextItem.typ { + case itemMethodArgs: + parseMethod(name, item, nextItem, false) + case itemSemicolon: + append(&FieldNode{ + Line: item.line, + Name: name.val, + Type: item.val, + Rhs: "", + ArrayLength: arrayLength, + }) + case itemEquals: + rhs := expect(lex, itemDefaultValue) + expect(lex, itemSemicolon) + append(&FieldNode{ + Line: item.line, + Name: name.val, + Type: item.val, + Rhs: rhs.val, + ArrayLength: arrayLength, + }) } - switch defn := context.stack[depth].(type) { - case *StructDefinition, *EnumDefinition: - inPlaceDefinition = defn.BaseName() - context.definitions = append(context.definitions, defn) - } - context.stack = context.stack[:depth] - case token == "struct": - if !scanner.Scan() { - log.Fatalf("%d: bizarre struct", lineNumber) - } - if !strings.Contains(codeline, "{") || strings.Contains(codeline, "}") { - log.Fatalf("%d: bad formatting", lineNumber) - } - stackEntry := StructDefinition{ - name: scanner.Text(), - qualifier: context.generateQualifier(), - Description: context.commentBlocks[lineNumber-1], - parent: context.findParent(), - } - context.stack = append(context.stack, &stackEntry) - return - case token == "enum": - if !scanner.Scan() || scanner.Text() != "class" || !scanner.Scan() { - log.Fatalf("%d: bad enum", lineNumber) - } - if !strings.Contains(codeline, "{") || strings.Contains(codeline, "}") { - log.Fatalf("%d: bad formatting", lineNumber) - } - stackEntry := EnumDefinition{ - name: scanner.Text(), - qualifier: context.generateQualifier(), - Description: context.commentBlocks[lineNumber-1], - parent: context.findParent(), - } - context.stack = append(context.stack, &stackEntry) - return - case depth > 0: - switch defn := context.stack[depth].(type) { - case *StructDefinition: - scanStructField(defn, token) + case item.typ == itemTemplate: + expect(lex, itemTemplateArgs) + returns := expect(lex, itemSimpleType) + name := expect(lex, itemIdentifier) + args := expect(lex, itemMethodArgs) + append(parseMethod(name, returns, args, true)) + default: + panic(lex, item) + } + } + return members +} + +// Assumes that we have just consumed the "class" keyword from the lexer. +// Consumes everything up to (and including) the trailing semicolon. +func parseClass(lex *lexer) *ClassNode { + name := expect(lex, itemIdentifier) + item := lex.nextItem() + if item.typ == itemSemicolon { + // We don't have an AST node for forward declarations, just skip it. + return nil + } + if item.typ == itemColon { + // Only one base class is allowed. + item = lex.nextItem() + if item.typ == itemPublic { + item = lex.nextItem() + } + expect(lex, itemSimpleType) + item = lex.nextItem() + } + if item.typ != itemOpenBrace { + panic(lex, item) + } + members := parseStructBody(lex) + expect(lex, itemSemicolon) + return &ClassNode{ + Line: name.line, + Name: name.val, + Members: members, + } +} + +// Assumes that we have just consumed the "struct" keyword from the lexer. +// Consumes everything up to (and including) the trailing semicolon. +func parseStruct(lex *lexer) *StructNode { + name := expect(lex, itemIdentifier) + item := lex.nextItem() + if item.typ == itemSemicolon { + // We don't have an AST node for forward declarations, just skip it. + return nil + } + if item.typ != itemOpenBrace { + panic(lex, item) + } + members := parseStructBody(lex) + expect(lex, itemSemicolon) + return &StructNode{ + Line: name.line, + Name: name.val, + Members: members, + } +} + +// Assumes that we have just consumed the "enum" keyword from the lexer. +// Consumes everything up to (and including) the trailing semicolon. +func parseEnum(lex *lexer) *EnumNode { + expect(lex, itemClass) + name := expect(lex, itemIdentifier) + item := lex.nextItem() + if item.typ == itemColon { + expect(lex, itemSimpleType) + item = lex.nextItem() + } + if item.typ != itemOpenBrace { + panic(lex, item) + } + firstVal := expect(lex, itemIdentifier) + node := &EnumNode{ + Name: name.val, + Line: name.line, + Values: []string{firstVal.val}, + ValueLines: []int{firstVal.line}, + } + for item = lex.nextItem(); item.typ != itemCloseBrace; { + if item.typ != itemComma { + panic(lex, item) + } + item = lex.nextItem() + if item.typ == itemCloseBrace { + break + } + if item.typ != itemIdentifier { + panic(lex, item) + } + node.Values = append(node.Values, item.val) + node.ValueLines = append(node.ValueLines, item.line) + item = lex.nextItem() + } + expect(lex, itemSemicolon) + return node +} + +// Assumes that we have just consumed the "using" keyword from the lexer. +// Consumes everything up to (and including) the trailing semicolon. +func parseUsing(lex *lexer) *UsingNode { + name := expect(lex, itemIdentifier) + expect(lex, itemEquals) + rhs := expect(lex, itemSimpleType) + expect(lex, itemSemicolon) + return &UsingNode{name.line, name.val, rhs.val} +} + +// Assumes that we have just consumed the "namespace" keyword from the lexer. +// Consumes everything up to (and including) the closing brace. +func parseNamespace(lex *lexer) *NamespaceNode { + name := expect(lex, itemIdentifier) + expect(lex, itemOpenBrace) + ns := &NamespaceNode{name.line, name.val, nil} + item := lex.nextItem() + + // Filter out nil nodes (e.g. forward declarations) + // Note that checking for nil is tricky due to a classic Go gotcha. + append := func(child Node) { + switch concrete := child.(type) { + case *StructNode: + if concrete == nil { return - case *EnumDefinition: - if strings.Contains(codeline, "=") { - log.Fatalf("%d: custom values are not allowed", lineNumber) - } - value := EnumValue{ - Name: strings.Trim(token, ","), - } - if matches := context.fieldDescParser.FindStringSubmatch(codeline); matches != nil { - value.Description = matches[1] - } - defn.Values = append(defn.Values, value) + } + case *ClassNode: + if concrete == nil { return } } + ns.Children = append(ns.Children, child) } - if err := scanner.Err(); err != nil { - log.Fatalf("%d: %s", lineNumber, err) + + for ; item.typ != itemCloseBrace; item = lex.nextItem() { + switch item.typ { + case itemTemplate: + expect(lex, itemTemplateArgs) + switch lex.nextItem().typ { + case itemClass: + node := parseClass(lex) + node.IsTemplate = true + append(node) + case itemStruct: + node := parseStruct(lex) + node.IsTemplate = true + append(node) + default: + panic(lex, item) + } + case itemClass: + append(parseClass(lex)) + case itemStruct: + append(parseStruct(lex)) + case itemEnum: + append(parseEnum(lex)) + case itemUsing: + append(parseUsing(lex)) + case itemNamespace: + append(parseNamespace(lex)) + default: + panic(lex, item) + } } + return ns +} + +func parseRoot(lex *lexer) *RootNode { + expect(lex, itemNamespace) + ns := parseNamespace(lex) + return &RootNode{0, ns} +} + +func expect(lex *lexer, expectedType itemType) item { + item := lex.nextItem() + if item.typ != expectedType { + panic(lex, item) + } + return item +} + +func panic(lex *lexer, unexpected item) { + lex.drain() + // Very useful local hack: change this to Panicf to see a call stack. + log.Fatalf("%d: parser sees unexpected lexeme %s", unexpected.line, unexpected.String()) }