diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,41 @@ # Revision history for zeolite-lang +## 0.24.1.0 -- 2024-01-07 + +### Language + +* **[fix]** Fixes parsing of `<` expressions when the left side is an explicit + type conversion, e.g., `"123"?String < "456"`. Previously, the `<` was assumed + to be the start of a param substitution. + + Note that this shouldn't ever be needed because `<` only supports built-in + `concrete` types. (Any value with a union/intersection type convertible to one + of those types already simplifies to that type, making the union/intersection + superfluous.) + +* **[new]** Adds the `<||` operator to use the left value unless it's `empty`. + +```text +optional Int value <- empty +// value is preferred, but 123 is used because value is empty. +Int other <- value <|| 123 +``` + +* **[behavior]** Adds source context to traces for crashes during initialization + of `@category` members. + +### Libraries + +* **[fix]** Fixes `CheckSequence:using` (in `lib/testing`) when using an empty + list. + +* **[new]** Adds `AppendH` helpers to `lib/util`. + +### Examples + +* Adds `example/highlighter`, as both a code example and a highlighted-HTML + generator for Zeolite source files. + ## 0.24.0.1 -- 2023-12-10 ### Language diff --git a/base/category-source.hpp b/base/category-source.hpp --- a/base/category-source.hpp +++ b/base/category-source.hpp @@ -190,7 +190,12 @@ BoxedValue::Present(result) \ ? TypeValue::Call(result, func, args) \ : ReturnTuple(count); \ - }) + }) + +#define TYPE_VALUE_LEFT_UNLESS_EMPTY(left, right) ({ \ + const BoxedValue result = left; \ + BoxedValue::Present(result) ? result : (right); \ + }) class TypeValue { public: diff --git a/example/highlighter/README.md b/example/highlighter/README.md --- a/example/highlighter/README.md +++ b/example/highlighter/README.md @@ -1,10 +1,12 @@ # Zeolite Code Highlighter -This example is a work in progress and doesn't do anything yet. Eventually it -will generate syntax-highlighted HTML from Zeolite source files. +This example is a fully-functional syntax highlighter for Zeolite source files. -The module should still build and the tests should pass, however: +**You can see the highlighting of the code in this example +[here](https://ta0kira.github.io/zeolite/example/highlighter/).** +To build and test the example code: + ```shell # This is just to locate the example code. Not for normal use! ZEOLITE_PATH=$(zeolite --get-path) @@ -14,4 +16,10 @@ # Run the unit tests. zeolite -p "$ZEOLITE_PATH" -t example/highlighter +``` + +To generate an HTML file for Zeolite source files (after building the example): + +```shell +example/highlighter/ZeoliteHighlight "Your Page Title" your-source-file.0rx ``` diff --git a/example/highlighter/main.0rx b/example/highlighter/main.0rx --- a/example/highlighter/main.0rx +++ b/example/highlighter/main.0rx @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- -Copyright 2023 Kevin P. Barry +Copyright 2023-2024 Kevin P. Barry Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,7 +21,134 @@ } define ZeoliteHighlight { + $ReadOnlyExcept[]$ + + @category ZeoliteParseContext zeoliteSourceContext <- ZeoliteParseContext.new() + .include() + .include() + .include() + .include() + // Must be before ZeoliteLowerSymbol. + .include() + // Must be before ZeoliteOperator. (Due to : in labels.) + .include() + .include() + // Must be before ZeoliteExtras. (Due to \ in escapes.) + .include() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + run () { - // TODO + if (Argv.global().size() < 3) { + \ BasicOutput.stderr() + .append(Argv.global().readAt(0)) + .append(" [page title] [.0r? input files]\n") + exit(1) + } + \ writeHtmlStart(title: Argv.global().readAt(1)) + traverse (Counter.builder().start(2).limit(Argv.global().size()).done() -> Int i) { + String filename <- Argv.global().readAt(i) + $ReadOnly[filename]$ + scoped { + optional String content <- loadFile(filename: filename) + } in if (`present` content) { + \ processFile(filename: filename, content: `require` content) + } + } + \ writeHtmlEnd() + } + + @type writeHtmlStart (String title:) -> () + writeHtmlStart (title) { + Formatted escapedTitle <- HtmlContent.escape(content: title) + // TODO: Use HtmlContent to format this stuff? + \ BasicOutput.stdout() + .append("") + .append("") + .append("") + .append("") + .append("") + .append("").append(escapedTitle).append("") + .append("") + .append("") + .append("

").append(escapedTitle).append("

\n") + \ writeCommand() + \ BasicOutput.stdout().append("
") + } + + @type writeCommand () -> () + writeCommand () { + \ BasicOutput.stdout() + .append("

This file was generated by ZeoliteHighlight.

") + } + + @type writeHtmlEnd () -> () + writeHtmlEnd () { + \ BasicOutput.stdout() + .append("") + .append("") + } + + @type processFile (String filename:, String content:) -> () + processFile (filename, content) { + TextStream input <- TextStream.new(content) + $Hidden[content]$ + DefaultOrder output <- defer + \ StreamTokenizer:new(context: zeoliteSourceContext, tokenizer: zeoliteSourceContext.defaultTokenizer()) + .tokenizeAll(input, (output <- Vector.new())) + + if (input.atEnd()) { + \ BasicOutput.stderr() + .append("Successfully parsed file ") + .append(filename) + .append(":\n") + } else { + \ BasicOutput.stderr() + .append("Incomplete parse of file ") + .append(filename) + .append(" at line ") + .append(input.currentLine()) + .append(" char ") + .append(input.currentChar()) + .append(":\n") + } + + ZeoliteHtmlFormatter formatter <- ZeoliteHtmlFormatter.new() + \ BasicOutput.stdout().append("

").append(HtmlContent.escape(content: filename)).append("

\n") + \ BasicOutput.stdout().append("
\n")
+    traverse (output.defaultOrder() -> ZeoliteParsed parsed) {
+      \ BasicOutput.stdout().append(parsed.formatWith(formatter))
+    }
+    \ BasicOutput.stdout().append("
\n") + } + + @type loadFile (String filename:) -> (optional String) + loadFile (filename) { + \ BasicOutput.stderr() + .append("Reading file ") + .append(filename) + .append("...\n") + scoped { + RawFileReader input <- RawFileReader.open(filename) + } cleanup { + \ input.freeResource() + } in if (`present` input.getFileError()) { + \ BasicOutput.stderr() + .append("Error reading file ") + .append(filename) + .append(": ") + .append(`require` input.getFileError()) + .append("\n") + return empty + } else { + return TextReader.readAll(input) + } } } diff --git a/example/highlighter/src/html.0rp b/example/highlighter/src/html.0rp new file mode 100644 --- /dev/null +++ b/example/highlighter/src/html.0rp @@ -0,0 +1,34 @@ +/* ----------------------------------------------------------------------------- +Copyright 2024 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +$ModuleOnly$ + +concrete HtmlContent { + refines Formatted + + @type section (String open:, String close:) -> ([Append & Build]) + @type tag (String name:) -> (HtmlContent) + @type escape (String content:) -> (HtmlContent) + @type raw (String verified:) -> (HtmlContent) +} + +concrete ZeoliteHtmlFormatter { + refines TokenFormatter + + @type new () -> (ZeoliteHtmlFormatter) +} diff --git a/example/highlighter/src/html.0rx b/example/highlighter/src/html.0rx new file mode 100644 --- /dev/null +++ b/example/highlighter/src/html.0rx @@ -0,0 +1,206 @@ +/* ----------------------------------------------------------------------------- +Copyright 2024 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +define HtmlContent { + $ReadOnlyExcept[]$ + + @category KVReader charEscapes <- HashedMap.new() + .set('"', """) + .set('#', "#") + .set('$', "$") + .set('%', "%") + .set('&', "&") + .set('\'', "'") + .set('<', "<") + .set('>', ">") + + @value String content + + formatted () { + return content + } + + section (open, close) { + return delegate -> `HtmlSectionBuilder.new` + } + + tag (name) { + return #self{ String.builder() + .append("<") + .append(name) + .append(" />") + .build() } + } + + escape (content) { + [Append & Build] builder <- String.builder() + traverse (content.defaultOrder() -> Char c) { + scoped { + optional String replacement <- charEscapes.get(c) + } in if (`present` replacement) { + \ builder.append(`require` replacement) + } else { + \ builder.append(c) + } + } + return #self{ builder.build() } + } + + raw (verified) { + return #self{ verified } + } +} + +define ZeoliteHtmlFormatter { + $ReadOnlyExcept[]$ + + @category KVReader labelToStyle <- HashedMap.new() + .set("ZeoliteError", "color:red; font-weight:bold; text-decoration:underline;") + + .set("ZeoliteBlockComment", "color:DarkGray; font-style:italic;") + .set("ZeoliteLineComment", "color:DarkGray; font-style:italic;") + .set("ZeolitePragma", "color:DarkSalmon; font-style:italic;") + .set("ZeolitePragmaName", "color:DarkSalmon; font-style:italic; font-weight:bold;") + .set("ZeolitePragmaArg", "color:DarkSalmon; font-style:normal;") + + .set("ZeoliteArgLabel", "color:green;") + .set("ZeoliteAssignment", "color:blue;") + .set("ZeoliteDiscard", "color:blue;") + .set("ZeoliteFunctionCall", "color:blue;") + .set("ZeoliteTick", "color:blue;") + + .set("ZeoliteBuiltinCategory", "color:DarkBlue; font-weight:bold;") + .set("ZeoliteBuiltinConstant", "color:DarkCyan; font-weight:bold;") + .set("ZeoliteIgnore", "color:DarkCyan; font-weight:bold;") + .set("ZeoliteBuiltinFunction", "color:DarkGreen; font-weight:bold;") + .set("ZeoliteBuiltinParam", "color:DarkBlue; font-weight:bold;") + .set("ZeoliteCategoryName", "color:DarkBlue;") + .set("ZeoliteParamName", "color:DarkBlue;") + + .set("ZeoliteCharLiteral", "color:DarkViolet;") + .set("ZeoliteStringLiteral", "color:DeepPink;") + .set("ZeoliteEscapedChar", "color:DarkMagenta;") + .set("ZeoliteOperator", "color:BlueViolet;") + + .set("ZeoliteNumber", "color:GoldenRod;") + .set("ZeoliteEscapedNumber", "color:DarkMagenta;") + + .set("ZeoliteContainKeyword", "color:Indigo; font-weight:bold;") + .set("ZeoliteScopeQualifier", "color:Indigo; font-weight:bold;") + .set("ZeoliteTypeKeyword", "color:Crimson;") + .set("ZeoliteControlKeyword", "font-weight:bold;") + .set("ZeoliteStorageKeyword", "font-style:italic;") + + .set("ZeoliteTestcase", "background:PaleTurquoise;") + .set("ZeoliteTestcaseKeyword", "color:green;") + + @category SetReader labelsUsingDefault <- HashedSet.new() + .append("ZeoliteBraceSection") + .append("ZeoliteCloseDelim") + .append("ZeoliteComma") + .append("ZeoliteDoubleQuote") + .append("ZeoliteFunctionOrVariableName") + .append("ZeoliteOpenDelim") + .append("ZeoliteParenSection") + .append("ZeolitePragmaArgClose") + .append("ZeolitePragmaArgOpen") + .append("ZeolitePragmaDelim") + .append("ZeoliteQuotedChars") + .append("ZeoliteSingleQuote") + .append("ZeoliteSquareSection") + .append("ZeoliteTestcaseSection") + .append("ZeoliteWhitespace") + + new () { + return delegate -> #self + } + + formatLeaf (label, value) { + scoped { + optional String style <- getStyle(label) + } in if (`present` style) { + return HtmlContent + .section(open: buildSpanTag(`require` style), close: "") + .append(HtmlContent.escape(content: value)) + .build() + } else { + return HtmlContent.escape(content: value) + } + } + + createSection (label) { + scoped { + optional String style <- getStyle(label) + } in if (`present` style) { + return HtmlContent.section(open: buildSpanTag(`require` style), close: "") + } else { + return HtmlContent.section(open: "", close: "") + } + } + + @type getStyle (String) -> (optional String) + getStyle (label) (style) { + style <- labelToStyle.get(label) + if (! `present` style && ! labelsUsingDefault.member(label)) { + \ BasicOutput.stderr() + .append("HTML style for label \"") + .append(label) + .append("\" not found!\n") + } + } + + @type buildSpanTag (String) -> (String) + buildSpanTag (style) { + return String.builder() + .append("") + .build() + } +} + +concrete HtmlSectionBuilder { + refines Append + refines Build + + @type new (String open:, String close:) -> (HtmlSectionBuilder) +} + +define HtmlSectionBuilder { + $ReadOnlyExcept[]$ + + @value [Append & Build] content + @value String close + + new (open, close) { + return #self{ String.builder().append(open), close } + } + + append (newContent) { + \ delegate -> `content.append` + return self + } + + build () { + // We need a new builder so that we don't append close multiple times. + return HtmlContent.raw(verified: String.builder() + .append(content.build()) + .append(close) + .build()) + } +} diff --git a/example/highlighter/src/tokenizer.0rp b/example/highlighter/src/tokenizer.0rp --- a/example/highlighter/src/tokenizer.0rp +++ b/example/highlighter/src/tokenizer.0rp @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- -Copyright 2023 Kevin P. Barry +Copyright 2023-2024 Kevin P. Barry Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,55 +18,69 @@ $ModuleOnly$ -@value interface TextToken<|#l> { - immutable +// Tokenizes with context #c and generates values of type #v. +@value interface Tokenizer<#c|#v> { + tokenize (TextStream, #c) -> (optional #v) +} - exportWith<#f> (TextFormatter<#l, #f>) -> (#f) +// Supports generic formatting based on label type #l and content type #x. +@value interface FormattedToken<|#l, #x> { + formatWith<#f> (TokenFormatter<#l, #x, #f>) -> (#f) } -@value interface TextFormatter<#l|#f> { - format (String content:, #l label:) -> (#f) +// Generates #f values by formatting with label type #l and content type #x. +@value interface TokenFormatter<#l, #x|#f|> { + formatLeaf (#l label:, #x value:) -> (#f) + createSection (#l label:) -> ([Append<#f> & Build<#f>]) } +// Input adapter for parsing text input. concrete TextStream { @type new (String) -> (TextStream) - @value current () -> (optional Char) - @value forward () -> (TextStream) - @value reset () -> (TextStream) - @value take () -> (String) - @value preview () -> (String) - @value atEnd () -> (Bool) -} + @value current () -> (optional Char) + @value tokenSize () -> (Int) + @value preview () -> (String) + @value currentLine () -> (Int) + @value currentChar () -> (Int) + @value atEnd () -> (Bool) -@value interface Tokenizer<#t, #v> { - tokenize (TextStream, KVReader<#t, Tokenizer<#t, #v>>) -> (optional #v, optional #t, Int) + @value forward () -> (TextStream) + @value reset () -> (TextStream) + @value take () -> (String) } -concrete MultiTokenizer<#t, #v> { - refines Tokenizer<#t, #v> - - visibility Tokenizer<#t, #v> - - @type new () -> (#self) +// Tokenizes all content in a TextStream. +concrete StreamTokenizer<#c|#v> { + @category new<#c, #v> (#c context:, Tokenizer<#c, #v> tokenizer:) -> (StreamTokenizer<#c, #v>) - @value include (#t) -> (#self) + @value tokenizeAll (TextStream, Append<#v>) -> (#self) } -concrete StreamTokenizer<#t, #v> { - @category new<#t, #v> - #v requires TextToken - (KVReader<#t, Tokenizer<#t, #v>> tokenizers:, Tokenizer<#t, #v> default:) -> (StreamTokenizer<#t, #v>) +// Serializes parsed data without formatting. +concrete UnformattedFormatter { + refines TokenFormatter - @value tokenizeAll (TextStream, Append<#v>) -> (#self) + @type new () -> (#self) } +// Helpers for reasoning about Char values. concrete CharType { - @type lower (Char) -> (Bool) - @type upper (Char) -> (Bool) - @type digit (Char) -> (Bool) - @type alphaNum (Char) -> (Bool) + @type lower (Char) -> (Bool) + @type upper (Char) -> (Bool) + @type digit (Char) -> (Bool) + @type binDigit (Char) -> (Bool) + @type octDigit (Char) -> (Bool) + @type hexDigit (Char) -> (Bool) + @type alphaNum (Char) -> (Bool) @type whitespace (Char) -> (Bool) - @type oneOf (Char, String) -> (Bool) + + @type binChars () -> (SetReader) + @type octChars () -> (SetReader) + @type decChars () -> (SetReader) + @type hexChars () -> (SetReader) + + @type oneOf (Char, DefaultOrder) -> (Bool) + @type escapeBreaks (DefaultOrder) -> (String) } diff --git a/example/highlighter/src/tokenizer.0rx b/example/highlighter/src/tokenizer.0rx --- a/example/highlighter/src/tokenizer.0rx +++ b/example/highlighter/src/tokenizer.0rx @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- -Copyright 2023 Kevin P. Barry +Copyright 2023-2024 Kevin P. Barry Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,9 +22,11 @@ @value String source @value Int tokenStart @value Int tokenEnd + @value Int currentLine + @value Int currentChar new (source) { - return TextStream{ source, 0, 0 } + return TextStream{ source, 0, 0, 1, 1 } } current () (char) { @@ -34,6 +36,10 @@ } } + tokenSize () { + return tokenEnd-tokenStart + } + forward () { if (tokenEnd < source.size()) { tokenEnd <- tokenEnd+1 @@ -48,72 +54,66 @@ take () { cleanup { + while (tokenStart < tokenEnd) { + if (source.readAt(tokenStart) == '\n') { + currentChar <- 1 + currentLine <- currentLine+1 + } else { + currentChar <- currentChar+1 + } + } update { + tokenStart <- tokenStart+1 + } tokenStart <- tokenEnd } in return preview() } preview () { - return source.subSequence(tokenStart, tokenEnd-tokenStart) + return source.subSequence(tokenStart, tokenSize()) } + currentLine () { + return currentLine + } + + currentChar () { + return currentChar + } + atEnd () { return tokenStart >= source.size() } } -define MultiTokenizer { - $ReadOnlyExcept[]$ - - @value [DefaultOrder<#t> & Append<#t>] types - +define UnformattedFormatter { new () { - return #self{ Vector<#t>.new() } + return delegate -> #self } - include (type) { - \ types.append(type) - return self + formatLeaf (_, value) { + return value.formatted() } - tokenize (input, tokenizers) (token, nextType, popCount) { - token <- empty - nextType <- empty - popCount <- 0 - traverse (types.defaultOrder() -> #t type) { - scoped { - optional Tokenizer<#t, #v> tokenizer <- tokenizers.get(type) - } in if (`present` tokenizer) { - token, nextType, popCount <- require(tokenizer).tokenize(input, tokenizers) - } - } update { - if (`present` token) { - break - } - } + createSection (_) { + return String.builder() } } define StreamTokenizer { $ReadOnlyExcept[]$ - @value KVReader<#t, Tokenizer<#t, #v>> tokenizers - @value Tokenizer<#t, #v> defaultTokenizer + @value #c context + @value Tokenizer<#c, #v> tokenizer - new (tokenizers, defaultTokenizer) { - return delegate -> StreamTokenizer<#t, #v> + new (context, tokenizer) { + return delegate -> StreamTokenizer<#c, #v> } tokenizeAll (input, output) { scoped { - Stack> tokenizerStack <- Vector>.new().push(defaultTokenizer) - Tokenizer<#t, #v> tokenizer <- defaultTokenizer optional #v token <- empty - optional #t nextType <- empty - Int popCount <- 0 } in while (!input.atEnd()) { - $Hidden[tokenizerStack,defaultTokenizer]$ - $ReadOnly[tokenizer]$ - token, nextType, popCount <- tokenizer.tokenize(input, tokenizers) + token <- input `tokenizer.tokenize` context } update { // Process new token. if (`present` token) { @@ -122,26 +122,6 @@ // Tokenizing failed => let the caller deal with the rest of the input. break } - // Pop tokenizers. - while (popCount > 0 && tokenizerStack.size() > 0) { - tokenizer <- tokenizerStack.pop() - } update { - popCount <- popCount-1 - } - if (popCount > 0) { - tokenizer <- defaultTokenizer - popCount <- 0 - } - // Set new tokenizer. - scoped { - optional Tokenizer<#t, #v> nextTokenizer <- empty - if (`present` nextType) { - nextTokenizer <- tokenizers.get(`require` nextType) - } - } in if (`present` nextTokenizer) { - \ tokenizerStack.push(tokenizer) - tokenizer <- `require` nextTokenizer - } } return self } @@ -155,7 +135,18 @@ .set('\n', "\\n") .set('\r', "\\r") .set('\"', "\\\"") + .set('\\', "\\\\") + @category SetReader binChars <- HashedSet.new() `AppendH:from` "01".defaultOrder() + + @category SetReader octChars <- HashedSet.new() `AppendH:from` "01234567".defaultOrder() + + @category SetReader decChars <- HashedSet.new() `AppendH:from` "0123456789".defaultOrder() + + @category SetReader hexChars <- HashedSet.new() `AppendH:from` "0123456789abcdefABCDEF".defaultOrder() + + @category SetReader whitespaceChars <- HashedSet.new() `AppendH:from` "\n\t\r ".defaultOrder() + lower (c) { return c >= 'a' && c <= 'z' } @@ -165,15 +156,43 @@ } digit (c) { - return c >= '0' && c <= '9' + return `decChars.member` c } + binDigit (c) { + return `binChars.member` c + } + + octDigit (c) { + return `octChars.member` c + } + + hexDigit (c) { + return `hexChars.member` c + } + alphaNum (c) { return lower(c) || upper(c) || digit(c) } whitespace (c) { - return c `oneOf` "\n\t\r " + return `whitespaceChars.member` c + } + + binChars () { + return binChars + } + + octChars () { + return octChars + } + + decChars () { + return decChars + } + + hexChars () { + return hexChars } oneOf (c, allowed) (match) { diff --git a/example/highlighter/src/zeolite-parsing.0rp b/example/highlighter/src/zeolite-parsing.0rp new file mode 100644 --- /dev/null +++ b/example/highlighter/src/zeolite-parsing.0rp @@ -0,0 +1,94 @@ +/* ----------------------------------------------------------------------------- +Copyright 2023 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +$ModuleOnly$ + +// Hierarchical parsed data from Zeolite source files. +concrete ZeoliteParsed { + refines Formatted + refines FormattedToken + refines TestCompare + + visibility Tokenizer + + @type leaf (String label:, String content:) -> (ZeoliteParsed) + @type section (String label:, DefaultOrder) -> (ZeoliteParsed) +} + +// Singleton tokenizer with a name. +@type interface NamedTokenizer { + tokenizer () -> (Tokenizer) + tokenizerName () -> (String) +} + +// Context used when parsing Zeolite source files. +concrete ZeoliteParseContext { + refines KVReader> + + @type new () -> (ZeoliteParseContext) + + @value defaultTokenizer () -> (Tokenizer) + + @value include<#t> + #t defines NamedTokenizer + () -> (ZeoliteParseContext) +} + +// Delegate to a tokenizer by name. +concrete UseNamedTokenizer { + @type new<#t> + #t defines NamedTokenizer + () -> (Tokenizer) +} + +// Attempts multiple tokenizers in order. +concrete TokenAlternatives { + refines Append> + refines Tokenizer + + @type new () -> (TokenAlternatives) +} + +// Zeolite symbol parsing. +concrete ZeoliteSymbol { + // Parses symbol and returns the stream forwarded to the end of the symbol. + + @type parseUpperSymbol (TextStream) -> (optional TextStream) + @type parseLowerSymbol (TextStream) -> (optional TextStream) + @type parseParamName (TextStream) -> (optional TextStream) + @type parseScopeQualifier (TextStream) -> (optional TextStream) + @type parseOperatorSymbols (TextStream) -> (optional TextStream) + + // Caller must check for \ first. Stream isn't reset! + @type parseEscapedChar (TextStream) -> (optional TextStream) + + // Returns a type label if the token is in the given set. + + @type tryControlKeyword (String) -> (optional String) + @type tryTypeKeyword (String) -> (optional String) + @type tryContainKeyword (String) -> (optional String) + @type tryStorageKeyword (String) -> (optional String) + @type tryScopeQualifier (String) -> (optional String) + @type tryTestcaseKeyword (String) -> (optional String) + @type tryBuiltinCategory (String) -> (optional String) + @type tryBuiltinFunction (String) -> (optional String) + @type tryBuiltinConstant (String) -> (optional String) + @type tryBuiltinParam (String) -> (optional String) + @type tryAssignment (String) -> (optional String) + @type tryFunctionCall (String) -> (optional String) +} diff --git a/example/highlighter/src/zeolite-parsing.0rx b/example/highlighter/src/zeolite-parsing.0rx new file mode 100644 --- /dev/null +++ b/example/highlighter/src/zeolite-parsing.0rx @@ -0,0 +1,467 @@ +/* ----------------------------------------------------------------------------- +Copyright 2023-2024 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +define ZeoliteParsed { + $ReadOnlyExcept[]$ + + @value String label + @value optional String content + @value DefaultOrder subsections + + leaf (label, content) { + return #self{ label, content, AlwaysEmpty.default() } + } + + section (label, subsections) { + return #self{ label, empty, subsections } + } + + testCompare (actual, report) { + \ MultiChecker.new(report) + .check( + title: "label", + actual.label(), + CheckValue:equals(label)) + &.tryCheck( + title: "content", + actual.content(), + CheckValue:equals(content)) + &.tryCheck( + title: "subsections", + actual.subsections(), + CheckSequence:matches(CheckSequence:using(subsections))) + } + + formatted () { + [Append & Build] builder <- String.builder() + .append("ZeoliteParsed{label:") + .append(label) + if (`present` content) { + \ builder + .append(",content:\"") + .append(CharType.escapeBreaks(`require` content)) + .append("\"") + } else { + traverse (subsections.defaultOrder() -> Formatted subsection) { + \ builder + .append(",") + .append(subsection) + } + } + \ builder.append("}") + return builder.build() + } + + formatWith (formatter) { + if (`present` content) { + return formatter.formatLeaf(label: label, value: `require` content) + } else { + [Append<#f> & Build<#f>] builder <- formatter.createSection(label: label) + traverse (subsections.defaultOrder() -> ZeoliteParsed subsection) { + \ builder.append(subsection.formatWith(formatter)) + } + return builder.build() + } + } + + @value label () -> (String) + label () { + return label + } + + @value content () -> (optional String) + content () { + return content + } + + @value subsections () -> (DefaultOrder) + subsections () { + return subsections + } +} + +define UseNamedTokenizer { + $ReadOnlyExcept[]$ + + refines Tokenizer + + @value String name + + new () { + return #self{ #t.tokenizerName() } + } + + tokenize (input, context) { + return delegate -> `context.get(name)&.tokenize` + } +} + +define TokenAlternatives { + $ReadOnlyExcept[]$ + + @value Vector> tokenizers + + new () { + return #self{ Vector>.new() } + } + + append (tokenizer) { + \ delegate -> `tokenizers.append` + return self + } + + tokenize (input, context) (token) { + token <- empty + traverse (tokenizers.defaultOrder() -> Tokenizer tokenizer) { + token <- tokenizer.tokenize(input, context) + } update { + if (`present` token) { + break + } + } + } +} + +define ZeoliteParseContext { + $ReadOnlyExcept[]$ + + @value TokenAlternatives defaultTokenizer + @value HashedMap> tokenizers + + new () { + return #self{ TokenAlternatives.new(), HashedMap>.new() } + } + + defaultTokenizer () { + return defaultTokenizer + } + + include () { + \ #t.tokenizerName() `tokenizers.set` #t.tokenizer() + \ defaultTokenizer.append(UseNamedTokenizer.new<#t>()) + return self + } + + get (k) { + return delegate -> `tokenizers.get` + } +} + +define ZeoliteSymbol { + $ReadOnlyExcept[]$ + + @category SetReader controlKeywords <- HashedSet.new() + .append("break") + .append("cleanup") + .append("continue") + .append("defer") + .append("delegate") + .append("elif") + .append("else") + .append("if") + .append("in") + .append("return") + .append("scoped") + .append("traverse") + .append("update") + .append("while") + + @category SetReader typeKeywords <- HashedSet.new() + .append("allows") + .append("defines") + .append("immutable") + .append("refines") + .append("requires") + .append("visibility") + + @category SetReader containKeywords <- HashedSet.new() + .append("concrete") + .append("interface") + .append("define") + .append("testcase") + + @category SetReader storageKeywords <- HashedSet.new() + .append("optional") + .append("weak") + + @category SetReader scopeQualifiers <- HashedSet.new() + .append("@category") + .append("@type") + .append("@value") + .append("unittest") + + @category SetReader testcaseKeywords <- HashedSet.new() + .append("args") + .append("compiler") + .append("compiles") + .append("error") + .append("exclude") + .append("failure") + .append("require") + .append("stderr") + .append("stdout") + .append("success") + .append("timeout") + + @category SetReader builtinCategries <- HashedSet.new() + .append("all") + .append("any") + .append("Append") + .append("AsBool") + .append("AsChar") + .append("AsFloat") + .append("AsInt") + .append("Bool") + .append("Bounded") + .append("Build") + .append("Char") + .append("CharBuffer") + .append("Container") + .append("Default") + .append("Duplicate") + .append("Equals") + .append("Float") + .append("Formatted") + .append("Hashed") + .append("Identifier") + .append("Int") + .append("LessThan") + .append("Order") + .append("Pointer") + .append("ReadAt") + .append("String") + .append("SubSequence") + .append("Testcase") + .append("WriteAt") + + @category SetReader builtinFunctions <- HashedSet.new() + .append("exit") + .append("fail") + .append("identify") + .append("present") + .append("reduce") + .append("require") + .append("strong") + .append("typename") + + @category SetReader builtinConstants <- HashedSet.new() + .append("empty") + .append("false") + .append("true") + .append("self") + + @category SetReader builtinParams <- HashedSet.new() + .append("#self") + + @category SetReader assignmentOperators <- HashedSet.new() + .append("<-") + .append("<-|") + .append("<->") + .append("->") + + @category SetReader functionCallOperators <- HashedSet.new() + .append(":") + .append(".") + .append("&.") + .append("?") + + @category SetReader operatorChars <- HashedSet.new() `AppendH:from` ":.!%^&*-+|<>?=".defaultOrder() + + @category SetReader escapedChars <- HashedSet.new() `AppendH:from` "'\"?\\abfnrtv".defaultOrder() + + parseUpperSymbol (input) { + \ input.reset() + optional Char current <- input.current() + if (! `present` current || !CharType.upper(`require` current)) { + return empty + } + while (`present` current && CharType.alphaNum(`require` current)) { + current <- input.forward().current() + } + return input + } + + parseLowerSymbol (input) { + \ input.reset() + optional Char current <- input.current() + if (! `present` current || !CharType.lower(`require` current)) { + return empty + } + while (`present` current && CharType.alphaNum(`require` current)) { + current <- input.forward().current() + } + return input + } + + parseParamName (input) { + \ input.reset() + optional Char current <- input.current() + if (! `present` current || `require` current != '#') { + return empty + } + current <- input.forward().current() + if (! `present` current || !CharType.lower(`require` current)) { + return empty + } + while (`present` current && CharType.alphaNum(`require` current)) { + current <- input.forward().current() + } + return input + } + + parseScopeQualifier (input) { + \ input.reset() + optional Char current <- input.current() + if (! `present` current || `require` current != '@') { + return empty + } + current <- input.forward().current() + while (`present` current && CharType.alphaNum(`require` current)) { + current <- input.forward().current() + } + return input + } + + parseOperatorSymbols (input) (output) { + \ input.reset() + output <- empty + optional Char current <- input.current() + while (`present` current && `operatorChars.member` `require` current) { + current <- input.forward().current() + output <- input + } + } + + parseEscapedChar (input) { + scoped { + optional Char current <- input.current() + } in if (! `present` current) { + return empty + } elif (`require` current == 'x') { + // Hex escaped char. + traverse (Counter.zeroIndexed(2) -> _) { + current <- input.forward().current() + if (! `present` current || !CharType.hexDigit(`require` current)) { + return empty + } + } + \ input.forward() + } elif (CharType.octDigit(`require` current)) { + // Oct escaped char. + traverse (Counter.zeroIndexed(2) -> _) { + current <- input.forward().current() + if (! `present` current || !CharType.octDigit(`require` current)) { + return empty + } + } + \ input.forward() + } elif (`escapedChars.member` `require` current) { + // Single escaped char. + \ input.forward() + } else { + // Invalid escaped char. + \ input.forward() + return empty + } + return input + } + + tryControlKeyword (value) (type) { + type <- empty + if (`controlKeywords.member` value) { + type <- "ZeoliteControlKeyword" + } + } + + tryTypeKeyword (value) (type) { + type <- empty + if (`typeKeywords.member` value) { + type <- "ZeoliteTypeKeyword" + } + } + + tryContainKeyword (value) (type) { + type <- empty + if (`containKeywords.member` value) { + type <- "ZeoliteContainKeyword" + } + } + + tryStorageKeyword (value) (type) { + type <- empty + if (`storageKeywords.member` value) { + type <- "ZeoliteStorageKeyword" + } + } + + tryScopeQualifier (value) (type) { + type <- empty + if (`scopeQualifiers.member` value) { + type <- "ZeoliteScopeQualifier" + } + } + + tryTestcaseKeyword (value) (type) { + type <- empty + if (`testcaseKeywords.member` value) { + type <- "ZeoliteTestcaseKeyword" + } + } + + tryBuiltinCategory (value) (type) { + type <- empty + if (`builtinCategries.member` value) { + type <- "ZeoliteBuiltinCategory" + } + } + + tryBuiltinFunction (value) (type) { + type <- empty + if (`builtinFunctions.member` value) { + type <- "ZeoliteBuiltinFunction" + } + } + + tryBuiltinConstant (value) (type) { + type <- empty + if (`builtinConstants.member` value) { + type <- "ZeoliteBuiltinConstant" + } + } + + tryAssignment (value) (type) { + type <- empty + if (`assignmentOperators.member` value) { + type <- "ZeoliteAssignment" + } + } + + tryBuiltinParam (value) (type) { + type <- empty + if (`builtinParams.member` value) { + type <- "ZeoliteBuiltinParam" + } + } + + tryFunctionCall (value) (type) { + type <- empty + if (`functionCallOperators.member` value) { + type <- "ZeoliteFunctionCall" + } + } +} diff --git a/example/highlighter/src/zeolite-tokenizing.0rp b/example/highlighter/src/zeolite-tokenizing.0rp new file mode 100644 --- /dev/null +++ b/example/highlighter/src/zeolite-tokenizing.0rp @@ -0,0 +1,94 @@ +/* ----------------------------------------------------------------------------- +Copyright 2023-2024 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +$ModuleOnly$ + +concrete ZeoliteWhitespace { + defines NamedTokenizer +} + +concrete ZeoliteLineComment { + defines NamedTokenizer +} + +concrete ZeoliteBlockComment { + defines NamedTokenizer +} + +concrete ZeoliteUpperSymbol { + defines NamedTokenizer +} + +concrete ZeoliteLowerSymbol { + defines NamedTokenizer +} + +concrete ZeoliteScopeQualifier { + defines NamedTokenizer +} + +concrete ZeoliteOperator { + defines NamedTokenizer +} + +concrete ZeoliteParamName { + defines NamedTokenizer +} + +concrete ZeoliteStringLiteral { + defines NamedTokenizer +} + +concrete ZeoliteCharLiteral { + defines NamedTokenizer +} + +concrete ZeoliteNumber { + defines NamedTokenizer +} + +concrete ZeoliteBraceSection { + defines NamedTokenizer +} + +concrete ZeoliteParenSection { + defines NamedTokenizer +} + +concrete ZeoliteSquareSection { + defines NamedTokenizer +} + +concrete ZeolitePragma { + defines NamedTokenizer +} + +concrete ZeoliteExtras { + defines NamedTokenizer +} + +// NOTE: Uses the following if available: +// - ZeoliteStringLiteral (required) +// - ZeoliteLowerSymbol (required) +// - ZeoliteSquareSection +// - ZeoliteUpperSymbol +// - ZeoliteNumber +// - ZeoliteOperator +concrete ZeoliteTestcase { + defines NamedTokenizer +} diff --git a/example/highlighter/src/zeolite-tokenizing.0rx b/example/highlighter/src/zeolite-tokenizing.0rx new file mode 100644 --- /dev/null +++ b/example/highlighter/src/zeolite-tokenizing.0rx @@ -0,0 +1,688 @@ +/* ----------------------------------------------------------------------------- +Copyright 2023-2024 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +define ZeoliteWhitespace { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + optional Char current <- input.current() + while (`present` current && CharType.whitespace(`require` current)) { + current <- input.forward().current() + } + scoped { + String content <- input.take() + } in if (content.size() > 0) { + token <- ZeoliteParsed.leaf(label: tokenizerName(), content: content) + } + } +} + +define ZeoliteLineComment { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + \ input.forward().forward() + if (input.preview() != "//") { + return _ + } + optional Char current <- input.current() + while (`present` current && !CharType.oneOf(`require` current, "\r\n")) { + current <- input.forward().current() + } + token <- ZeoliteParsed.leaf(label: tokenizerName(), content: input.take()) + } +} + +define ZeoliteBlockComment { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + \ input.forward().forward() + if (input.preview() != "/*") { + return _ + } + scoped { + optional Char current <- empty + Bool starFound <- false + } in while (`present` (current <- input.current())) { + if (starFound && `require` current == '/') { + \ input.forward() + break + } else { + starFound <- `require` current == '*' + } + } update { + \ input.forward() + } + token <- ZeoliteParsed.leaf(label: tokenizerName(), content: input.take()) + } +} + +define ZeoliteUpperSymbol { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, _) { + optional String content <- ZeoliteSymbol.parseUpperSymbol(input)&.take() + if (! `present` content) { + return empty + } else { + String type <- ZeoliteSymbol.tryBuiltinCategory(`require` content) + <|| "ZeoliteCategoryName" + return ZeoliteParsed.leaf(label: type, content: `require` content) + } + } +} + +define ZeoliteLowerSymbol { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, _) { + optional String content <- ZeoliteSymbol.parseLowerSymbol(input)&.preview() + if (! `present` content) { + return empty + } else { + String type <- ZeoliteSymbol.tryControlKeyword(`require` content) + <|| ZeoliteSymbol.tryTypeKeyword(`require` content) + <|| ZeoliteSymbol.tryContainKeyword(`require` content) + <|| ZeoliteSymbol.tryStorageKeyword(`require` content) + <|| ZeoliteSymbol.tryScopeQualifier(`require` content) + <|| ZeoliteSymbol.tryBuiltinCategory(`require` content) + <|| ZeoliteSymbol.tryBuiltinFunction(`require` content) + <|| ZeoliteSymbol.tryBuiltinConstant(`require` content) + <|| tryLabel(input) + <|| "ZeoliteFunctionOrVariableName" + $Hidden[content]$ + return ZeoliteParsed.leaf(label: type, content: input.take()) + } + } + + @type tryLabel (TextStream) -> (optional String) + tryLabel (input) { + if (`present` input.current() && `require` input.current() == ':') { + \ input.forward() + return "ZeoliteArgLabel" + } else { + return empty + } + } +} + +define ZeoliteScopeQualifier { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, _) { + optional String content <- ZeoliteSymbol.parseScopeQualifier(input)&.take() + if (! `present` content) { + return empty + } else { + String type <- ZeoliteSymbol.tryScopeQualifier(`require` content) + <|| "ZeoliteError" + return ZeoliteParsed.leaf(label: type, content: `require` content) + } + } +} + +define ZeoliteOperator { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, _) { + optional String content <- ZeoliteSymbol.parseOperatorSymbols(input)&.take() + if (! `present` content) { + return empty + } else { + String type <- ZeoliteSymbol.tryAssignment(`require` content) + <|| ZeoliteSymbol.tryFunctionCall(`require` content) + <|| "ZeoliteOperator" + return ZeoliteParsed.leaf(label: type, content: `require` content) + } + } +} + +define ZeoliteParamName { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) { + optional String content <- ZeoliteSymbol.parseParamName(input)&.take() + if (! `present` content) { + return empty + } else { + String type <- ZeoliteSymbol.tryBuiltinParam(`require` content) + <|| "ZeoliteParamName" + return ZeoliteParsed.leaf(label: type, content: `require` content) + } + } +} + +concrete ZeoliteTestcaseKeyword { + defines Default + refines Tokenizer +} + +define ZeoliteTestcaseKeyword { + default () { + return delegate -> #self + } + + tokenize (input, context) (token) { + token <- empty + scoped { + optional String content <- ZeoliteSymbol.parseLowerSymbol(input)&.preview() + } in if (! `present` content) { + return empty + } else { + optional String type <- ZeoliteSymbol.tryTestcaseKeyword(`require` content) + $Hidden[content]$ + if (`present` type) { + token <- ZeoliteParsed.leaf(label: `require` type, content: input.take()) + } + } + } +} + +define ZeoliteStringLiteral { + $ReadOnlyExcept[]$ + + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + if (input.atEnd() || `require` input.current() != '"') { + return _ + } + Vector subsections <- Vector.new() + // Setting this early still allows building it incrementally while also + // allowing an early return if something is missing. + token <- ZeoliteParsed.section(label: tokenizerName(), subsections) + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: input.forward().take())) + + scoped { + optional Char current <- empty + } in while (!input.atEnd() && `present` (current <- input.current())) { + if (`require` current == '\"') { + \ addCurrentChars(input, subsections) + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: input.forward().take())) + return _ + } elif (`require` current == '\\') { + \ addCurrentChars(input, subsections) + if (`present` ZeoliteSymbol.parseEscapedChar(input.forward())) { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: input.take())) + } else { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteError", content: input.take())) + } + } else { + \ input.forward() + } + } + \ addCurrentChars(input, subsections) + } + + @type addCurrentChars (TextStream, Append) -> () + addCurrentChars (input, output) { + if (input.tokenSize() > 0) { + \ output.append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: input.take())) + } + } +} + +define ZeoliteCharLiteral { + $ReadOnlyExcept[]$ + + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + if (input.atEnd() || `require` input.current() != '\'') { + return _ + } + Vector subsections <- Vector.new() + // Setting this early still allows building it incrementally while also + // allowing an early return if something is missing. + token <- ZeoliteParsed.section(label: tokenizerName(), subsections) + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: input.forward().take())) + + optional Char current <- input.current() + if (! `present` current) { + return _ + } elif (`require` current == '\\') { + if (`present` ZeoliteSymbol.parseEscapedChar(input.forward())) { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: input.take())) + } else { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteError", content: input.take())) + } + } elif (`require` current == '\'') { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteError", content: input.forward().take())) + } else { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: input.forward().take())) + } + if (`present` (current <- input.current())) { + if (`require` current == '\'') { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: input.forward().take())) + } else { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteError", content: input.forward().take())) + } + } + } +} + +define ZeoliteNumber { + $ReadOnlyExcept[]$ + + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + optional Char current <- input.current() + if (! `present` current) { + return _ + } elif (CharType.digit(`require` current)) { + token <- parseNumber(input, escaped: false, CharType.decChars()) + } elif (`require` current `CharType.oneOf` "-+") { + \ input.forward() + token <- parseNumber(input, escaped: false, CharType.decChars()) + } elif (`require` current != '\\') { + return _ + } else { + current <- input.forward().current() + if (! `present` current) { + return _ + } elif (`require` current `CharType.oneOf` "bB") { + \ input.forward() + token <- parseNumber(input, escaped: true, CharType.binChars()) + } elif (`require` current `CharType.oneOf` "oO") { + \ input.forward() + token <- parseNumber(input, escaped: true, CharType.octChars()) + } elif (`require` current `CharType.oneOf` "dD") { + \ input.forward() + token <- parseNumber(input, escaped: true, CharType.decChars()) + } elif (`require` current `CharType.oneOf` "xX") { + \ input.forward() + token <- parseNumber(input, escaped: true, CharType.hexChars()) + } else { + return _ + } + } + } + + @type parseNumber (TextStream, Bool escaped:, SetReader) -> (optional ZeoliteParsed) + parseNumber (input, escaped, allowed) { + Bool isEmpty <- true + optional Char current <- empty + while (`present` (current <- input.current()) && `allowed.member` `require` current) { + \ input.forward() + isEmpty <- false + } + if (isEmpty) { + return empty + } elif (`present` (current <- input.current()) && `require` current == '.') { + \ input.forward() + while (`present` (current <- input.current()) && `allowed.member` `require` current) { + \ input.forward() + } + } + if (escaped) { + return ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: input.take()) + } elif (`present` current && `require` current `CharType.oneOf` "eE") { + \ input.forward() + if (`present` (current <- input.current()) && `require` current `CharType.oneOf` "-+") { + \ input.forward() + } + while (`present` (current <- input.current()) && `allowed.member` `require` current) { + \ input.forward() + } + } + return ZeoliteParsed.leaf(label: "ZeoliteNumber", content: input.take()) + } +} + +define ZeoliteBraceSection { + tokenizer () { + return ZeoliteDelimSection.new(name: tokenizerName(), open: '{', close: '}', empty) + } + + tokenizerName () { + return typename<#self>().formatted() + } +} + +define ZeoliteParenSection { + tokenizer () { + return ZeoliteDelimSection.new(name: tokenizerName(), open: '(', close: ')', empty) + } + + tokenizerName () { + return typename<#self>().formatted() + } +} + +define ZeoliteSquareSection { + tokenizer () { + return ZeoliteDelimSection.new(name: tokenizerName(), open: '[', close: ']', empty) + } + + tokenizerName () { + return typename<#self>().formatted() + } +} + +define ZeolitePragma { + $ReadOnlyExcept[]$ + + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + if (input.atEnd() || `require` input.current() != '$') { + return _ + } + Vector subsections <- Vector.new() + token <- ZeoliteParsed.section(label: tokenizerName(), subsections) + \ subsections.append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: input.forward().take())) + + if (! `present` ZeoliteSymbol.parseUpperSymbol(input)) { + return _ + } else { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeolitePragmaName", content: input.take())) + } + + optional Char current <- input.current() + if (! `present` current) { + return _ + } elif (`require` current == '[') { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeolitePragmaArgOpen", content: input.forward().take())) + while (`present` (current <- input.current()) && `require` current != ']') { + \ input.forward() + } + \ subsections.append(ZeoliteParsed.leaf(label: "ZeolitePragmaArg", content: input.take())) + if (`present` (current <- input.current())) { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeolitePragmaArgClose", content: input.forward().take())) + } else { + return _ + } + } + + if (`present` (current <- input.current())) { + if (`require` current == '$') { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: input.forward().take())) + } else { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteError", content: input.forward().take())) + } + } + } +} + +define ZeoliteExtras { + refines Tokenizer + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + \ input.reset() + optional Char current <- input.current() + if (! `present` current) { + return empty + } elif (`require` current == '_') { + \ input.forward() + return ZeoliteParsed.leaf(label: "ZeoliteIgnore", content: input.take()) + } elif (`require` current == '\\') { + \ input.forward() + return ZeoliteParsed.leaf(label: "ZeoliteDiscard", content: input.take()) + } elif (`require` current == '`') { + \ input.forward() + return ZeoliteParsed.leaf(label: "ZeoliteTick", content: input.take()) + } elif (`require` current == ',') { + \ input.forward() + return ZeoliteParsed.leaf(label: "ZeoliteComma", content: input.take()) + } elif (`require` current `CharType.oneOf` ";#@") { + \ input.forward() + return ZeoliteParsed.leaf(label: "ZeoliteError", content: input.take()) + } else { + return empty + } + } +} + +define ZeoliteTestcase { + $ReadOnlyExcept[]$ + + refines Tokenizer + + @category Tokenizer testcaseSpecs <- TokenAlternatives.new() + .append(ZeoliteOptionalSeparator.default()) + .append(UseNamedTokenizer.new()) + .append(UseNamedTokenizer.new()) + .append(ZeoliteTestcaseKeyword.default()) + .append(UseNamedTokenizer.new()) + .append(UseNamedTokenizer.new()) + .append(UseNamedTokenizer.new()) + .append(UseNamedTokenizer.new()) + + tokenizer () { + return delegate -> #self + } + + tokenizerName () { + return typename<#self>().formatted() + } + + tokenize (input, context) (token) { + scoped { + optional String content <- ZeoliteSymbol.parseLowerSymbol(input)&.preview() + } in if (! `present` content || require(content) != "testcase") { + return empty + } + + Vector subsections <- Vector.new() + // Setting this early still allows building it incrementally while also + // allowing an early return if something is missing. + token <- ZeoliteParsed.section(label: tokenizerName(), subsections) + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteContainKeyword", content: input.take())) + + \ ZeoliteOptionalSeparator.parseAny(input, context, subsections) + // Testcase description string. + scoped { + optional ZeoliteParsed name <- delegate -> `ZeoliteStringLiteral.tokenizer().tokenize` + } in if (`present` name) { + \ subsections.append(`require` name) + } else { + return _ + } + \ ZeoliteOptionalSeparator.parseAny(input, context, subsections) + + // Testcase specs. + \ StreamTokenizer:new(context: context, tokenizer: ZeoliteDelimSection.new(name: "ZeoliteTestcaseSection", open: '{', close: '}', testcaseSpecs)) + .tokenizeAll(input, subsections) + } +} + +concrete ZeoliteOptionalSeparator { + defines Default + refines Tokenizer + + visibility Tokenizer + + @type parseAny (TextStream, ZeoliteParseContext, Append) -> () +} + +define ZeoliteOptionalSeparator { + $ReadOnlyExcept[]$ + + @category Tokenizer tokenizer <- TokenAlternatives.new() + .append(UseNamedTokenizer.new()) + .append(UseNamedTokenizer.new()) + .append(UseNamedTokenizer.new()) + + parseAny (input, context, output) { + \ StreamTokenizer:new(context: context, tokenizer: default()).tokenizeAll(input, output) + } + + default () { + return delegate -> #self + } + + tokenize (input, output) { + return delegate -> `tokenizer.tokenize` + } +} + +concrete ZeoliteDelimSection { + @type new (String name:, Char open:, Char close:, optional Tokenizer) -> (Tokenizer) +} + +define ZeoliteDelimSection { + $ReadOnlyExcept[]$ + + refines Tokenizer + + @value String name + @value Char open + @value Char close + @value optional Tokenizer tokenizer + + new (name, open, close, tokenizer) { + return delegate -> #self + } + + tokenize (input, context) (token) { + \ input.reset() + token <- empty + if (input.atEnd() || `require` input.current() != open) { + return _ + } + Vector subsections <- Vector.new() + token <- ZeoliteParsed.section(label: name, subsections) + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: input.forward().take())) + + \ StreamTokenizer:new(context: context, tokenizer: tokenizer <|| context.defaultTokenizer()).tokenizeAll(input, subsections) + \ input.reset() + if (!input.atEnd() && `require` input.current() == close) { + \ subsections.append(ZeoliteParsed.leaf(label: "ZeoliteCloseDelim", content: input.forward().take())) + } + } +} diff --git a/example/highlighter/src/zeolite.0rp b/example/highlighter/src/zeolite.0rp deleted file mode 100644 --- a/example/highlighter/src/zeolite.0rp +++ /dev/null @@ -1,72 +0,0 @@ -/* ----------------------------------------------------------------------------- -Copyright 2023 Kevin P. Barry - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ------------------------------------------------------------------------------ */ - -// Author: Kevin P. Barry [ta0kira@gmail.com] - -$ModuleOnly$ - -concrete ZeoliteToken { - refines Formatted - refines TestCompare - refines TextToken - - visibility Tokenizer - - @type new (String content:, String label:) -> (optional ZeoliteToken) -} - -@type interface NamedTokenizer { - tokenizer () -> (Tokenizer) - tokenizerName () -> (String) -} - -concrete ZeoliteTokenizers { - refines KVReader> - - @type new () -> (ZeoliteTokenizers) - - @value include<#t> - #t defines NamedTokenizer - () -> (ZeoliteTokenizers) -} - -concrete ZeoliteSelectFrom { - refines Tokenizer - - visibility Tokenizer - - @type new () -> (ZeoliteSelectFrom) - - @value include<#t> - #t defines NamedTokenizer - () -> (ZeoliteSelectFrom) -} - -concrete ZeoliteWhitespace { - defines NamedTokenizer -} - -concrete ZeoliteLineComment { - defines NamedTokenizer -} - -concrete ZeoliteCategoryName { - defines NamedTokenizer -} - -concrete ZeoliteParamName { - defines NamedTokenizer -} diff --git a/example/highlighter/src/zeolite.0rx b/example/highlighter/src/zeolite.0rx deleted file mode 100644 --- a/example/highlighter/src/zeolite.0rx +++ /dev/null @@ -1,214 +0,0 @@ -/* ----------------------------------------------------------------------------- -Copyright 2023 Kevin P. Barry - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ------------------------------------------------------------------------------ */ - -// Author: Kevin P. Barry [ta0kira@gmail.com] - -define ZeoliteToken { - @value String content - @value String label - - new (content, label) { - if (content.size() < 1) { - return empty - } else { - return delegate -> #self - } - } - - exportWith (formatter) { - return formatter.format(content: content, label: label) - } - - testCompare (actual, report) { - \ MultiChecker.new(report) - .tryCheck( - title: "content", - actual.content(), - CheckValue:equals(content)) - .tryCheck( - title: "label", - actual.label(), - CheckValue:equals(label)) - } - - formatted () { - return String.builder() - .append("ZeoliteToken{\"") - .append(CharType.escapeBreaks(content)) - .append("\",") - .append(label) - .append("}") - .build() - } - - @value content () -> (String) - content () { - return content - } - - @value label () -> (String) - label () { - return label - } -} - -define ZeoliteTokenizers { - $ReadOnlyExcept[]$ - - @value HashedMap> tokenizers - - new () { - return #self{ HashedMap>.new() } - } - - include () { - \ #t.tokenizerName() `tokenizers.set` #t.tokenizer() - return self - } - - get (k) { - return delegate -> `tokenizers.get` - } -} - -define ZeoliteSelectFrom { - $ReadOnlyExcept[]$ - - @value MultiTokenizer selector - - new () { - return #self{ MultiTokenizer.new() } - } - - include () { - \ selector.include(#t.tokenizerName()) - return self - } - - tokenize (input, tokenizers) { - return delegate -> `selector.tokenize` - } -} - -define ZeoliteWhitespace { - refines Tokenizer - - tokenizer () { - return #self{ } - } - - tokenizerName () { - return typename<#self>().formatted() - } - - tokenize (input, tokenizers) (token, nextType, popCount) { - \ input.reset() - token <- empty - nextType <- empty - popCount <- 0 - optional Char current <- input.current() - while (`present` current && CharType.whitespace(`require` current)) { - current <- input.forward().current() - } - token <- ZeoliteToken.new(content: input.take(), label: tokenizerName()) - } -} - -define ZeoliteLineComment { - refines Tokenizer - - tokenizer () { - return #self{ } - } - - tokenizerName () { - return typename<#self>().formatted() - } - - tokenize (input, tokenizers) (token, nextType, popCount) { - \ input.reset() - token <- empty - nextType <- empty - popCount <- 0 - \ input.forward().forward() - if (input.preview() != "//") { - return _ - } - optional Char current <- input.current() - while (`present` current && !CharType.oneOf(`require` current, "\r\n")) { - current <- input.forward().current() - } - token <- ZeoliteToken.new(content: input.take(), label: tokenizerName()) - } -} - -define ZeoliteCategoryName { - refines Tokenizer - - tokenizer () { - return #self{ } - } - - tokenizerName () { - return typename<#self>().formatted() - } - - tokenize (input, tokenizers) (token, nextType, popCount) { - \ input.reset() - token <- empty - nextType <- empty - popCount <- 0 - optional Char current <- input.current() - if (! `present` current || !CharType.upper(`require` current)) { - return _ - } - while (`present` current && CharType.alphaNum(`require` current)) { - current <- input.forward().current() - } - token <- ZeoliteToken.new(content: input.take(), label: tokenizerName()) - } -} - -define ZeoliteParamName { - refines Tokenizer - - tokenizer () { - return #self{ } - } - - tokenizerName () { - return typename<#self>().formatted() - } - - tokenize (input, tokenizers) (token, nextType, popCount) { - \ input.reset() - token <- empty - nextType <- empty - popCount <- 0 - optional Char current <- input.current() - if (! `present` current || `require` current != '#') { - return _ - } - current <- input.forward().current() - if (! `present` current || !CharType.lower(`require` current)) { - return _ - } - while (`present` current && CharType.alphaNum(`require` current)) { - current <- input.forward().current() - } - token <- ZeoliteToken.new(content: input.take(), label: tokenizerName()) - } -} diff --git a/example/highlighter/test/tokenizer.0rt b/example/highlighter/test/tokenizer.0rt new file mode 100644 --- /dev/null +++ b/example/highlighter/test/tokenizer.0rt @@ -0,0 +1,67 @@ +/* ----------------------------------------------------------------------------- +Copyright 2024 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +testcase "input tests" { + success TestChecker +} + +unittest integration { + TextStream input <- TextStream.new("123\n4") + + \ input.atEnd() `Matches:with` CheckValue:equals(false) + \ input.current() `Matches:with` CheckValue:equals('1') + \ input.preview() `Matches:with` CheckValue:equals("") + \ input.currentLine() `Matches:with` CheckValue:equals(1) + \ input.currentChar() `Matches:with` CheckValue:equals(1) + + \ input.forward().forward() + \ input.atEnd() `Matches:with` CheckValue:equals(false) + \ input.current() `Matches:with` CheckValue:equals('3') + \ input.preview() `Matches:with` CheckValue:equals("12") + \ input.currentLine() `Matches:with` CheckValue:equals(1) + \ input.currentChar() `Matches:with` CheckValue:equals(1) + + \ input.take() `Matches:with` CheckValue:equals("12") + \ input.atEnd() `Matches:with` CheckValue:equals(false) + \ input.current() `Matches:with` CheckValue:equals('3') + \ input.preview() `Matches:with` CheckValue:equals("") + \ input.currentLine() `Matches:with` CheckValue:equals(1) + \ input.currentChar() `Matches:with` CheckValue:equals(3) + + \ input.forward().forward() + \ input.atEnd() `Matches:with` CheckValue:equals(false) + \ input.current() `Matches:with` CheckValue:equals('4') + \ input.preview() `Matches:with` CheckValue:equals("3\n") + + \ input.reset() + \ input.atEnd() `Matches:with` CheckValue:equals(false) + \ input.current() `Matches:with` CheckValue:equals('3') + \ input.preview() `Matches:with` CheckValue:equals("") + \ input.currentLine() `Matches:with` CheckValue:equals(1) + \ input.currentChar() `Matches:with` CheckValue:equals(3) + + \ input.forward().forward().forward() + \ input.atEnd() `Matches:with` CheckValue:equals(false) + \ input.current() `Matches:with` CheckValue:equals(empty?Char) + \ input.preview() `Matches:with` CheckValue:equals("3\n4") + + \ input.take() `Matches:with` CheckValue:equals("3\n4") + \ input.atEnd() `Matches:with` CheckValue:equals(true) + \ input.currentLine() `Matches:with` CheckValue:equals(2) + \ input.currentChar() `Matches:with` CheckValue:equals(2) +} diff --git a/example/highlighter/test/zeolite-tokenizing.0rt b/example/highlighter/test/zeolite-tokenizing.0rt new file mode 100644 --- /dev/null +++ b/example/highlighter/test/zeolite-tokenizing.0rt @@ -0,0 +1,568 @@ +/* ----------------------------------------------------------------------------- +Copyright 2023-2024 Kevin P. Barry + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +----------------------------------------------------------------------------- */ + +// Author: Kevin P. Barry [ta0kira@gmail.com] + +testcase "parsing tests" { + success TestChecker +} + +concrete ParsingTester { + @type new (ZeoliteParseContext) -> (ParsingTester) + + @value checkParsesAs (String, DefaultOrder) -> () +} + +unittest parseZeoliteWhitespace { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ " \t \r\n" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " \t \r\n")) + + \ " test" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) +} + +unittest parseZeoliteLineComment { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include() + .include()) + + \ "//12345\n//54321" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteLineComment", content: "//12345")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: "\n")) + .append(ZeoliteParsed.leaf(label: "ZeoliteLineComment", content: "//54321")) + + \ "/ /" `tester.checkParsesAs` AlwaysEmpty.default() +} + +unittest parseZeoliteBlockComment { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include() + .include()) + + \ "/*/12345\n54321* */" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*/12345\n54321* */")) + + \ "/*comment*/ " `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*comment*/")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + \ "/*forever " `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*forever ")) + + \ "/ *" `tester.checkParsesAs` AlwaysEmpty.default() +} + +unittest parseZeoliteUpperSymbol { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "Hello123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteCategoryName", content: "Hello123")) + \ "String" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBuiltinCategory", content: "String")) + + \ "Hello_123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteCategoryName", content: "Hello")) + + \ "hello" `tester.checkParsesAs` AlwaysEmpty.default() + \ "123Hello" `tester.checkParsesAs` AlwaysEmpty.default() +} + +unittest parseZeoliteLowerSymbol { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "hello123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionOrVariableName", content: "hello123")) + \ "hello123:" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteArgLabel", content: "hello123:")) + \ "cleanup" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteControlKeyword", content: "cleanup")) + \ "immutable" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteTypeKeyword", content: "immutable")) + \ "concrete" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteContainKeyword", content: "concrete")) + \ "optional" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteStorageKeyword", content: "optional")) + \ "unittest" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteScopeQualifier", content: "unittest")) + \ "any" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBuiltinCategory", content: "any")) + \ "identify" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBuiltinFunction", content: "identify")) + \ "empty" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBuiltinConstant", content: "empty")) + + \ "hello_123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionOrVariableName", content: "hello")) + // Only has a special label in testcase. + \ "compiler" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionOrVariableName", content: "compiler")) + \ "empty123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionOrVariableName", content: "empty123")) + \ "identify:" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBuiltinFunction", content: "identify")) + + \ "Hello" `tester.checkParsesAs` AlwaysEmpty.default() + \ "123hello" `tester.checkParsesAs` AlwaysEmpty.default() +} + +unittest parseZeoliteScopeQualifier { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "@value" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteScopeQualifier", content: "@value")) + \ "@type" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteScopeQualifier", content: "@type")) + \ "@category" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteScopeQualifier", content: "@category")) + + \ "@value something" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteScopeQualifier", content: "@value")) + \ "@local" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "@local")) +} + +unittest parseZeoliteOperator { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "<-" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteAssignment", content: "<-")) + \ "<-|" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteAssignment", content: "<-|")) + \ "<->" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteAssignment", content: "<->")) + \ "->" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteAssignment", content: "->")) + \ ":" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionCall", content: ":")) + \ "." `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionCall", content: ".")) + \ "&." `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionCall", content: "&.")) + \ "?" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionCall", content: "?")) + \ "<<" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOperator", content: "<<")) + \ ":.!%^&*-+|<>?=" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOperator", content: ":.!%^&*-+|<>?=")) + + \ "<-||" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOperator", content: "<-||")) + \ "&.call()" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteFunctionCall", content: "&.")) +} + +unittest parseZeoliteParamName { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "#param123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteParamName", content: "#param123")) + \ "#self" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteBuiltinParam", content: "#self")) + + \ "#param_123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteParamName", content: "#param")) + \ "#self123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteParamName", content: "#self123")) + + \ "param" `tester.checkParsesAs` AlwaysEmpty.default() + \ "#Param" `tester.checkParsesAs` AlwaysEmpty.default() + \ "#123param" `tester.checkParsesAs` AlwaysEmpty.default() +} + +unittest parseZeoliteStringLiteral { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "\"hello\\n\\t\\0123\\xABC\\Q \\09c goodbye\"" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteStringLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "hello")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\n")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\t")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\012")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "3")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\xAB")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "C")) + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "\\Q")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "\\0")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "9c goodbye")) + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")))) + + \ "\"forever" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteStringLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "forever")))) +} + +unittest parseZeoliteCharLiteral { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "'a'" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteCharLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "a")) + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")))) + \ "'\\''" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteCharLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")))) + \ "'\\n'" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteCharLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\n")) + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")))) + \ "'\\xAB'" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteCharLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\xAB")) + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")))) + \ "'\\Q'" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteCharLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "\\Q")) + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")))) + + \ "''" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteCharLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "'")))) + \ "'forever" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteCharLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteSingleQuote", content: "'")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "f")) + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "o")))) +} + +unittest parseZeoliteNumber { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "123")) + \ "123.456" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "123.456")) + \ "123.456E123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "123.456E123")) + \ "+123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "+123")) + \ "-123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "-123")) + \ "123.456E+123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "123.456E+123")) + \ "123.456E-123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "123.456E-123")) + + \ "\\b101.010" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\b101.010")) + \ "\\o123.456" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\o123.456")) + \ "\\d123.456" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\d123.456")) + \ "\\x123.abc" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\x123.abc")) + + \ "123A" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "123")) + \ "\\b0123" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\b01")) + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "23")) + \ "\\o12389" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\o123")) + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "89")) + \ "\\d123A" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\d123")) + \ "\\xABCqq" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedNumber", content: "\\xABC")) + + \ "-" `tester.checkParsesAs` AlwaysEmpty.default() + \ "+" `tester.checkParsesAs` AlwaysEmpty.default() + \ "\\" `tester.checkParsesAs` AlwaysEmpty.default() + \ "\\b" `tester.checkParsesAs` AlwaysEmpty.default() + \ "\\o" `tester.checkParsesAs` AlwaysEmpty.default() + \ "\\d" `tester.checkParsesAs` AlwaysEmpty.default() + \ "\\h" `tester.checkParsesAs` AlwaysEmpty.default() +} + +unittest parseZeoliteBraceSection { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include() + .include() + .include()) + + \ "{ /*hello*/ }" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteBraceSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "{")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*hello*/")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCloseDelim", content: "}")))) + + \ "{ /*forever*/" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteBraceSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "{")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*forever*/")))) +} + +unittest parseZeoliteParenSection { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include() + .include() + .include()) + + \ "( /*hello*/ )" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteParenSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "(")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*hello*/")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCloseDelim", content: ")")))) + + \ "( /*forever*/" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteParenSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "(")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*forever*/")))) +} + +unittest parseZeoliteSquareSection { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include() + .include() + .include()) + + \ "[ /*hello*/ ]" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteSquareSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "[")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*hello*/")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCloseDelim", content: "]")))) + + \ "[ /*forever*/" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteSquareSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "[")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*forever*/")))) +} + +unittest parseZeoliteTestcase { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + // ZeoliteTestcase also uses the above if available. + .include() + // These should be after ZeoliteTestcase. + .include() + .include()) + + String data <- "testcase \"my test\" { + success Checker<[A&B]> // <- enables lib/testing support + require compiler \"something\" + timeout 30 + }" + + \ data `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteTestcase", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteContainKeyword", content: "testcase")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.section(label: "ZeoliteStringLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "my test")) + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")))) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.section(label: "ZeoliteTestcaseSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "{")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: "\n ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteTestcaseKeyword", content: "success")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCategoryName", content: "Checker")) + .append(ZeoliteParsed.leaf(label: "ZeoliteOperator", content: "<")) + .append(ZeoliteParsed.section(label: "ZeoliteSquareSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "[")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCategoryName", content: "A")) + .append(ZeoliteParsed.leaf(label: "ZeoliteOperator", content: "&")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCategoryName", content: "B")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCloseDelim", content: "]")))) + .append(ZeoliteParsed.leaf(label: "ZeoliteOperator", content: ">")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteLineComment", content: "// <- enables lib/testing support")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: "\n ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteTestcaseKeyword", content: "require")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteTestcaseKeyword", content: "compiler")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.section(label: "ZeoliteStringLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "something")) + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")))) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: "\n ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteTestcaseKeyword", content: "timeout")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteNumber", content: "30")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: "\n ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCloseDelim", content: "}")))))) + + \ "testcase { }" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteTestcase", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteContainKeyword", content: "testcase")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")))) +} + +unittest parseZeolitePragma { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "$NoArgs$" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeolitePragma", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaName", content: "NoArgs")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")))) + \ "$SomeArgs[a, b, 123]$" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeolitePragma", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaName", content: "SomeArgs")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaArgOpen", content: "[")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaArg", content: "a, b, 123")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaArgClose", content: "]")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")))) + + \ "$NoArgs" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeolitePragma", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaName", content: "NoArgs")))) + \ "$foo$" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeolitePragma", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")))) + \ "$NoArgs#" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeolitePragma", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaName", content: "NoArgs")) + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "#")))) + \ "$SomeArgs[args" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeolitePragma", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaDelim", content: "$")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaName", content: "SomeArgs")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaArgOpen", content: "[")) + .append(ZeoliteParsed.leaf(label: "ZeolitePragmaArg", content: "args")))) +} + +unittest parseZeoliteExtras { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include()) + + \ "_" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteIgnore", content: "_")) + \ "\\" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteDiscard", content: "\\")) + \ "`" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteTick", content: "`")) + \ "," `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteComma", content: ",")) + \ ";" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: ";")) + \ "#" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "#")) + \ "@" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteError", content: "@")) +} + +unittest integration { + ParsingTester tester <- ParsingTester.new(ZeoliteParseContext.new() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + .include() + .include()) + + \ "{Category #param}\"string\\n\" /*/ done*/\n" `tester.checkParsesAs` Vector.new() + .append(ZeoliteParsed.section(label: "ZeoliteBraceSection", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteOpenDelim", content: "{")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCategoryName", content: "Category")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteParamName", content: "#param")) + .append(ZeoliteParsed.leaf(label: "ZeoliteCloseDelim", content: "}")))) + .append(ZeoliteParsed.section(label: "ZeoliteStringLiteral", Vector.new() + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")) + .append(ZeoliteParsed.leaf(label: "ZeoliteQuotedChars", content: "string")) + .append(ZeoliteParsed.leaf(label: "ZeoliteEscapedChar", content: "\\n")) + .append(ZeoliteParsed.leaf(label: "ZeoliteDoubleQuote", content: "\"")))) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: " ")) + .append(ZeoliteParsed.leaf(label: "ZeoliteBlockComment", content: "/*/ done*/")) + .append(ZeoliteParsed.leaf(label: "ZeoliteWhitespace", content: "\n")) +} + +define ParsingTester { + $ReadOnlyExcept[]$ + + @value ZeoliteParseContext context + + new (context) { + return delegate -> #self + } + + checkParsesAs (originalData, expected) { + TextStream input <- TextStream.new(originalData) + DefaultOrder output <- defer + \ StreamTokenizer:new(context: context, tokenizer: context.defaultTokenizer()) + .tokenizeAll(input, (output <- Vector.new())) + + // Match output. + \ output `Matches:tryWith` CheckSequence:matches(CheckSequence:using(expected)) + + // Serialize output. + [Append & Build] builder <- String.builder() + UnformattedFormatter formatter <- UnformattedFormatter.new() + traverse (output.defaultOrder() -> ZeoliteParsed parsed) { + \ builder.append(parsed.formatWith(formatter)) + } + + // Append unparsed remainder. + while (`present` input.current()) { + \ input.forward() + } + // This will be included in the output of failing tests. + \ BasicOutput.stderr() + .append("Remaining content: \"") + .append(CharType.escapeBreaks(input.preview())) + .append("\"\n") + \ builder.append(input.take()) + + // Check that all data is accounted for. + \ builder.build() `Matches:tryWith` CheckValue:equals(originalData) + } +} diff --git a/example/highlighter/test/zeolite.0rt b/example/highlighter/test/zeolite.0rt deleted file mode 100644 --- a/example/highlighter/test/zeolite.0rt +++ /dev/null @@ -1,49 +0,0 @@ -/* ----------------------------------------------------------------------------- -Copyright 2023 Kevin P. Barry - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ------------------------------------------------------------------------------ */ - -// Author: Kevin P. Barry [ta0kira@gmail.com] - -testcase "working tests" { - success TestChecker -} - -unittest test { - ZeoliteTokenizers allTokenizers <- ZeoliteTokenizers.new() - .include() - .include() - .include() - .include() - ZeoliteSelectFrom categoryOrParam <- ZeoliteSelectFrom.new() - .include() - .include() - .include() - .include() - - TextStream input <- TextStream.new("Category #param // done") - Vector output <- Vector.new() - - \ StreamTokenizer:new(tokenizers: allTokenizers, default: categoryOrParam).tokenizeAll(input, output) - - DefaultOrder expected <- Vector.new() - .append(`require` ZeoliteToken.new(content: "Category", label: "ZeoliteCategoryName")) - .append(`require` ZeoliteToken.new(content: " ", label: "ZeoliteWhitespace")) - .append(`require` ZeoliteToken.new(content: "#param", label: "ZeoliteParamName")) - .append(`require` ZeoliteToken.new(content: " ", label: "ZeoliteWhitespace")) - .append(`require` ZeoliteToken.new(content: "// done", label: "ZeoliteLineComment")) - - \ input.atEnd() `Matches:with` CheckValue:equals(true) - \ output `Matches:with` CheckSequence:matches(CheckSequence:using(expected)) -} diff --git a/lib/testing/src/check-sequence.0rx b/lib/testing/src/check-sequence.0rx --- a/lib/testing/src/check-sequence.0rx +++ b/lib/testing/src/check-sequence.0rx @@ -54,7 +54,13 @@ } defaultOrder () { - return #self{ original, original.defaultOrder() } + scoped { + optional Order<#x> order <- original.defaultOrder() + } in if (`present` order) { + return #self{ original, order } + } else { + return empty + } } get () { diff --git a/lib/testing/test/check-sequence.0rt b/lib/testing/test/check-sequence.0rt --- a/lib/testing/test/check-sequence.0rt +++ b/lib/testing/test/check-sequence.0rt @@ -176,13 +176,19 @@ success TestChecker } -unittest test { +unittest matchingValues { ValueList actual <- ValueList.new() .append(TestValue.default()) .append(TestValue.new(456, "something", empty)) ValueList expected <- ValueList.new() .append(TestValue.default()) .append(TestValue.new(456, "something", empty)) + \ actual `Matches:with` CheckSequence:matches(CheckSequence:using(expected)) +} + +unittest bothEmpty { + ValueList actual <- ValueList.new() + ValueList expected <- ValueList.new() \ actual `Matches:with` CheckSequence:matches(CheckSequence:using(expected)) } diff --git a/lib/util/helpers.0rp b/lib/util/helpers.0rp --- a/lib/util/helpers.0rp +++ b/lib/util/helpers.0rp @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- -Copyright 2021 Kevin P. Barry +Copyright 2021,2024 Kevin P. Barry Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -239,4 +239,12 @@ #xx defines LessThan<#x> @type new () -> (LessThan2<#x>) +} + +// Helpers for Append<#x>. +concrete AppendH { + // Appends all items in order. + @category from<#x, #a> + #a requires Append<#x> + (#a, optional Order<#x>) -> (#a) } diff --git a/lib/util/src/helpers.0rx b/lib/util/src/helpers.0rx --- a/lib/util/src/helpers.0rx +++ b/lib/util/src/helpers.0rx @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- -Copyright 2021 Kevin P. Barry +Copyright 2021,2024 Kevin P. Barry Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -232,5 +232,14 @@ lessThan2 (x, y) { return x `#xx.lessThan` y + } +} + +define AppendH { + from (output, input) { + traverse (input -> #x value) { + \ output.append(value) + } + return output } } diff --git a/lib/util/test/helpers.0rt b/lib/util/test/helpers.0rt --- a/lib/util/test/helpers.0rt +++ b/lib/util/test/helpers.0rt @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- -Copyright 2021,2023 Kevin P. Barry +Copyright 2021,2023-2024 Kevin P. Barry Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -212,6 +212,11 @@ expected <- expected+1 } \ expected `Matches:with` CheckValue:equals(5) +} + +unittest appendHFrom { + [Append & Build] output <- String.builder() `AppendH:from` "123".defaultOrder() `AppendH:from` "abc".defaultOrder() + \ output.build() `Matches:with` CheckValue:equals("123abc") } concrete ValueOrder { diff --git a/src/CompilerCxx/Code.hs b/src/CompilerCxx/Code.hs --- a/src/CompilerCxx/Code.hs +++ b/src/CompilerCxx/Code.hs @@ -102,6 +102,9 @@ startInitTracing :: CategoryName -> SymbolScope -> String startInitTracing t s = "TRACE_FUNCTION(" ++ show (show t ++ " " ++ show s ++ " init") ++ ")" +startLazyTracing :: CategoryName -> SymbolScope -> String +startLazyTracing t s = "TRACE_FUNCTION(" ++ show (show t ++ " " ++ show s ++ " init") ++ ")" + startTestTracing :: FunctionName -> String startTestTracing f = "TRACE_FUNCTION(" ++ show ("unittest " ++ show f) ++ ")" diff --git a/src/CompilerCxx/CxxFiles.hs b/src/CompilerCxx/CxxFiles.hs --- a/src/CompilerCxx/CxxFiles.hs +++ b/src/CompilerCxx/CxxFiles.hs @@ -593,7 +593,7 @@ inlineCategoryConstructor t d tm em = do ctx <- getContextForInit testing tm em t d CategoryScope - initMembers <- runDataCompiler (sequence $ map compileLazyInit members) ctx + initMembers <- runDataCompiler (sequence $ map (compileLazyInit $ getCategoryName t) members) ctx let initMembersStr = intercalate ", " $ cdOutput initMembers let initColon = if null initMembersStr then "" else " : " return $ mconcat [ diff --git a/src/CompilerCxx/Procedure.hs b/src/CompilerCxx/Procedure.hs --- a/src/CompilerCxx/Procedure.hs +++ b/src/CompilerCxx/Procedure.hs @@ -46,6 +46,7 @@ import Base.CompilerError import Base.GeneralType +import Base.Mergeable (mergeAny) import Base.MergeTree import Base.Positional import Compilation.CompilerState @@ -413,9 +414,9 @@ compileLazyInit :: (Ord c, Show c, CollectErrorsM m, CompilerContext c m [String] a) => - DefinedMember c -> CompilerState a m () -compileLazyInit (DefinedMember _ _ _ _ Nothing) = return () -compileLazyInit (DefinedMember c _ t1 n (Just e)) = resetBackgroundM $ do + CategoryName -> DefinedMember c -> CompilerState a m () +compileLazyInit _ (DefinedMember _ _ _ _ Nothing) = return () +compileLazyInit t0 (DefinedMember c _ t1 n (Just e)) = resetBackgroundM $ do (ts,e') <- compileExpression e when (length (pValues ts) /= 1) $ compilerErrorM $ "Expected single return in initializer" ++ formatFullContextBrace (getExpressionContext e) @@ -424,7 +425,14 @@ let Positional [t2] = ts lift $ (checkValueAssignment r fa t2 t1) " " ++ v + _ -> "" + -- NOTE: This needs to be on one line, due to how multiple member inits are concatenated. + csWrite [variableName n ++ "([this]() { " ++ startInitTracing t0 CategoryScope ++ trace ++ + " return " ++ writeStoredVariable t1 e' ++ "; })" + ] compileVoidExpression :: (Ord c, Show c, CollectErrorsM m, CompilerContext c m [String] a) => @@ -706,6 +714,12 @@ bind t1' t2' where bind t1 t2 + | o == "<||" = do + when (not $ isOptionalValue t1) $ do + compilerErrorM $ "<|| requires the left expression to be optional but got " ++ show t1 + when (isWeakValue t2) $ do + compilerErrorM $ "<|| requires the right expression to be not be weak but got " ++ show t2 + compileOptionalOr (t1,e1) (t2,e2) | o `Set.member` comparison && isIdentifierRequiredValue t1 && isIdentifierRequiredValue t2 = do return (Positional [boolRequiredValue],glueInfix PrimIdentifier PrimBool e1 o e2) | t1 /= t2 = @@ -772,6 +786,11 @@ compilerErrorM $ "Function call requires one return but got " ++ formatTypes ts ++ formatFullContextBrace c2 formatTypes [] = "none" formatTypes ts = intercalate ", " (map show ts) + compileOptionalOr (t1,e1) (t2,e2) = do + let t' = combineTypes t1 t2 + let code = WrappedSingle $ "TYPE_VALUE_LEFT_UNLESS_EMPTY(" ++ useAsUnwrapped e1 ++ ", " ++ useAsUnwrapped e2 ++ ")" + return (Positional [t'], code) + combineTypes (ValueType _ t1) (ValueType s t2) = ValueType s (mergeAny [t1,t2]) forceOptionalReturns :: [c] -> ScopedFunction c -> ScopedFunction c forceOptionalReturns c0 (ScopedFunction c n t s v as rs ps fs ms) = diff --git a/src/Parser/Procedure.hs b/src/Parser/Procedure.hs --- a/src/Parser/Procedure.hs +++ b/src/Parser/Procedure.hs @@ -35,6 +35,7 @@ import qualified Data.Set as Set import Base.CompilerError +import Base.GeneralType (singleType) import Base.Positional import Parser.Common import Parser.Pragma @@ -43,6 +44,7 @@ import Parser.TypeInstance () import Types.Procedure import Types.TypeCategory +import Types.TypeInstance (TypeInstanceOrParam(..), TypeInstance(..)) instance ParseFromSource (ExecutableProcedure SourceContext) where @@ -339,7 +341,7 @@ o <- op return $ NamedOperator [c] o where op = labeled "binary operator" $ foldr (<|>) empty $ map (try . operator) ops - ops = compareInfix ++ logicalInfix ++ addInfix ++ subInfix ++ multInfix ++ divInfix ++ bitwiseInfix ++ bitshiftInfix + ops = compareInfix ++ logicalInfix ++ addInfix ++ subInfix ++ multInfix ++ divInfix ++ bitwiseInfix ++ bitshiftInfix ++ optionalOrInfix compareInfix :: [String] compareInfix = ["==","!=","<","<=",">",">="] @@ -365,11 +367,14 @@ bitshiftInfix :: [String] bitshiftInfix = [">>","<<"] +optionalOrInfix :: [String] +optionalOrInfix = ["<||"] + leftAssocInfix :: [String] leftAssocInfix = addInfix ++ subInfix ++ multInfix ++ divInfix ++ bitwiseInfix ++ bitshiftInfix rightAssocInfix :: [String] -rightAssocInfix = logicalInfix +rightAssocInfix = logicalInfix ++ optionalOrInfix nonAssocInfix :: [String] nonAssocInfix = compareInfix @@ -729,7 +734,7 @@ return $ IntegerLiteral [c] unsigned d instance ParseFromSource (ValueOperation SourceContext) where - sourceParser = valueCall <|> conversion <|> selectReturn where + sourceParser = conversion <|> valueCall <|> selectReturn where valueCall = labeled "function call" $ do c <- getSourceContext o <- (valueSymbolGet >> return AlwaysCall) <|> (valueSymbolMaybeGet >> return CallUnlessEmpty) @@ -739,8 +744,18 @@ conversion = labeled "type conversion" $ do c <- getSourceContext inferredParam - t <- sourceParser -- NOTE: Should not need try here. + t <- typeInstance <|> sourceParser return $ TypeConversion [c] t + -- This is the same as the TypeInstance ParseFromSource implementation + -- except that it uses try for params. This is necessary for < and <|| when + -- the left side is converted to a category without params. + typeInstance = labeled "type instance" $ do + n <- sourceParser + as <- labeled "type args" $ try args <|> return [] + return $ singleType $ JustTypeInstance $ TypeInstance n (Positional as) + args = between (sepAfter $ string "<") + (sepAfter $ string ">") + (sepBy sourceParser (sepAfter $ string ",")) selectReturn = labeled "return selection" $ do c <- getSourceContext sepAfter_ (string_ "{") diff --git a/src/Test/Procedure.hs b/src/Test/Procedure.hs --- a/src/Test/Procedure.hs +++ b/src/Test/Procedure.hs @@ -284,9 +284,22 @@ checkShortParseFail "x <- \\o123.", checkShortParseFail "x <- \\o123.456E1", + checkShortParseSuccess "x <-| 123", checkShortParseFail "Int x <-| 123", checkShortParseFail "_ <-| 123", checkShortParseFail "x, y <-| 123", + + checkShortParseSuccess "\\ x <|| y", + checkShortParseSuccess "\\ empty?Int <|| y", + checkShortParseSuccess "\\ 123 <|| 456", + checkShortParseSuccess "\\ empty <|| 'a'", + checkShortParseFail "Int x <|| 'a'", + checkShortParseFail "x, y <|| 'a'", + checkShortParseFail "\\ _ <|| 'a'", + + checkShortParseSuccess "\\ \"123\"?String < \"456\"", + checkShortParseSuccess "\\ \"123\"?ReadAt < Char>", + checkShortParseSuccess "\\ \"123\"?String < String.default()", checkParsesAs "'\"'" (\e -> case e of diff --git a/tests/conversions.0rt b/tests/conversions.0rt --- a/tests/conversions.0rt +++ b/tests/conversions.0rt @@ -1,5 +1,5 @@ /* ----------------------------------------------------------------------------- -Copyright 2020-2021 Kevin P. Barry +Copyright 2020-2021,2023 Kevin P. Barry Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -53,6 +53,14 @@ unittest convertedCall { Value value <- Value.create() \ value?Base1.call() +} + +unittest convertWithLessThan { + \ Testing.checkEquals("123"?String < "456", true) +} + +unittest convertWithParamSubstitution { + \ Testing.checkEquals("123"?ReadAt.readAt(1), '2') } concrete Helper { diff --git a/tests/delegation.0rt b/tests/delegation.0rt --- a/tests/delegation.0rt +++ b/tests/delegation.0rt @@ -292,3 +292,62 @@ return delegate -> Value } } + + +testcase "delegate handles optional with &." { + success +} + +concrete Helper { + @type testEmpty (Int) -> () + @type testNonEmpty (Int) -> () + @type call (Bool fake:) -> (optional String) +} + +define Helper { + testEmpty (position) { + \ Testing.checkEquals(delegate -> `Helper.call(fake: true)&.readAt`, empty) + } + + testNonEmpty (position) { + \ Testing.checkEquals(delegate -> `Helper.call(fake: false)&.readAt`, "123".readAt(position)) + } + + call (fake) { + if (fake) { + return empty + } else { + return "123" + } + } +} + +unittest testEmpty { + \ Helper.testEmpty(1) +} + +unittest testNonEmpty { + \ Helper.testNonEmpty(1) +} + + +testcase "delegate applies optional with &." { + error + require compiler "value" + require compiler "modifier" +} + +concrete Helper { + @type test (Int) -> () + @type call () -> (optional String) +} + +define Helper { + test (position) { + Char value <- delegate -> `Helper.call()&.readAt` + } + + call () { + return empty + } +} diff --git a/tests/modified-storage.0rt b/tests/modified-storage.0rt --- a/tests/modified-storage.0rt +++ b/tests/modified-storage.0rt @@ -395,6 +395,114 @@ } +testcase "optional or" { + success +} + +concrete Helper { + @type notCalled () -> (all) + @type getOnce () -> (optional Int) +} + +define Helper { + @category Bool used <- false + + notCalled () { + fail("should not be called") + } + + getOnce () { + if (used) { + fail("already called") + } + used <- true + return 123 + } +} + +unittest bothBoxedEmpty { + optional String value <- empty + \ Testing.checkEquals(value <|| "message", "message") +} + +unittest bothBoxedNotEmpty { + optional String value <- "other" + \ Testing.checkEquals(value <|| "message", "other") +} + +unittest leftUnboxedEmpty { + optional Int value <- empty + \ Testing.checkEquals((value <|| "message").formatted(), "message") +} + +unittest leftUnboxedNotEmpty { + optional Int value <- 123 + \ Testing.checkEquals((value <|| "message").formatted(), "123") +} + +unittest rightUnboxedEmpty { + optional String value <- empty + \ Testing.checkEquals((value <|| 123).formatted(), "123") +} + +unittest rightUnboxedNotEmpty { + optional String value <- "message" + \ Testing.checkEquals((value <|| 123).formatted(), "message") +} + +unittest leftEvaluatedOnce { + \ Testing.checkEquals(Helper.getOnce() <|| 456, 123) +} + +unittest shortCircuit { + optional String value <- "other" + \ Testing.checkEquals(value <|| Helper.notCalled(), "other") +} + + +testcase "optional-or required return type" { + error + require "\[Int\|String\]" + exclude "optional" +} + +unittest test { + Int value <- empty?Int <|| "message" +} + + +testcase "optional-or optional return type" { + error + require "modifier" + exclude "\[Int\|String\]" +} + +unittest test { + Formatted value <- empty?Int <|| empty?String +} + + +testcase "optional-or not allowed with left required" { + error + require "optional.+Int" +} + +unittest test { + \ 123 <|| 456 +} + + +testcase "optional-or not allowed with right weak" { + error + require "weak Int" +} + +unittest test { + weak Int value <- empty + \ empty?Int <|| value +} + + testcase "conditional assignment" { success } diff --git a/zeolite-lang.cabal b/zeolite-lang.cabal --- a/zeolite-lang.cabal +++ b/zeolite-lang.cabal @@ -1,7 +1,7 @@ cabal-version: 2.2 name: zeolite-lang -version: 0.24.0.1 +version: 0.24.1.0 synopsis: Zeolite is a statically-typed, general-purpose programming language. description: @@ -59,7 +59,7 @@ license-file: LICENSE author: Kevin P. Barry maintainer: Kevin P. Barry -copyright: (c) Kevin P. Barry 2019-2023 +copyright: (c) Kevin P. Barry 2019-2024 category: Compiler build-type: Simple