From 408ad698508515db73382425fb896ffa625d3faf Mon Sep 17 00:00:00 2001 From: Matt Vague Date: Sun, 4 Apr 2021 15:14:02 -0700 Subject: [PATCH 1/3] First pass at implementing liquid language support --- src/liquid/liquid.contribution.ts | 12 +++ src/liquid/liquid.test.ts | 115 +++++++++++++++++++++++ src/liquid/liquid.ts | 148 ++++++++++++++++++++++++++++++ src/monaco.contribution.ts | 1 + 4 files changed, 276 insertions(+) create mode 100644 src/liquid/liquid.contribution.ts create mode 100644 src/liquid/liquid.test.ts create mode 100644 src/liquid/liquid.ts diff --git a/src/liquid/liquid.contribution.ts b/src/liquid/liquid.contribution.ts new file mode 100644 index 00000000..feb7154b --- /dev/null +++ b/src/liquid/liquid.contribution.ts @@ -0,0 +1,12 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { registerLanguage } from '../_.contribution'; + +registerLanguage({ + id: 'liquid', + extensions: ['.liquid', '.liquid.html', '.liquid.css'], + loader: () => import('./liquid') +}); diff --git a/src/liquid/liquid.test.ts b/src/liquid/liquid.test.ts new file mode 100644 index 00000000..a24f567d --- /dev/null +++ b/src/liquid/liquid.test.ts @@ -0,0 +1,115 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { testTokenization } from '../test/testRunner'; + +testTokenization( + ['liquid', 'css'], + [ + // Just HTML + [ + { + line: '

liquid!

', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 3, type: 'delimiter.html' }, + { startIndex: 4, type: '' }, + { startIndex: 11, type: 'delimiter.html' }, + { startIndex: 13, type: 'tag.html' }, + { startIndex: 15, type: 'delimiter.html' } + ] + } + ], + + // Simple output + [ + { + line: '

{{ title }}

', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 3, type: 'delimiter.html' }, + { startIndex: 4, type: 'delimiter.liquid' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'variable.liquid' }, + { startIndex: 12, type: '' }, + { startIndex: 13, type: 'delimiter.liquid' }, + { startIndex: 15, type: 'delimiter.html' }, + { startIndex: 17, type: 'tag.html' }, + { startIndex: 19, type: 'delimiter.html' } + ] + } + ], + + // // Output filter + [ + { + line: '

{{ 3.14159265 | round | default: "pi" }}

', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 3, type: 'delimiter.html' }, + { startIndex: 4, type: 'delimiter.liquid' }, + { startIndex: 6, type: '' }, + { startIndex: 7, type: 'number.liquid' }, + { startIndex: 17, type: '' }, + { startIndex: 20, type: 'variable.liquid' }, + { startIndex: 25, type: '' }, + { startIndex: 28, type: 'variable.liquid' }, + { startIndex: 36, type: '' }, + { startIndex: 37, type: 'string.liquid' }, + { startIndex: 41, type: '' }, + { startIndex: 43, type: 'delimiter.liquid' }, + { startIndex: 45, type: 'delimiter.html' }, + { startIndex: 47, type: 'tag.html' }, + { startIndex: 49, type: 'delimiter.html' } + ] + } + ], + + // Tag + [ + { + line: '
{% render "files/file123.html" %}
', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 4, type: 'delimiter.html' }, + { startIndex: 5, type: 'delimiter.output.liquid' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'variable.liquid' }, + { startIndex: 14, type: '' }, + { startIndex: 15, type: 'string.liquid' }, + { startIndex: 35, type: '' }, + { startIndex: 36, type: 'delimiter.liquid' }, + { startIndex: 38, type: 'delimiter.html' }, + { startIndex: 40, type: 'tag.html' }, + { startIndex: 43, type: 'delimiter.html' } + ] + } + ], + + // Handlebars comment + [ + { + line: '
Anything you put between {% comment %} and {% endcomment %} tags
', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 4, type: 'delimiter.html' }, + { startIndex: 5, type: '' }, + { startIndex: 30, type: 'comment.start.liquid' }, + { startIndex: 43, type: 'comment.content.liquid' }, + { startIndex: 48, type: 'comment.end.liquid' }, + { startIndex: 64, type: '' }, + { startIndex: 69, type: 'delimiter.html' }, + { startIndex: 71, type: 'tag.html' }, + { startIndex: 74, type: 'delimiter.html' } + ] + } + ] + ] +); diff --git a/src/liquid/liquid.ts b/src/liquid/liquid.ts new file mode 100644 index 00000000..cccb5033 --- /dev/null +++ b/src/liquid/liquid.ts @@ -0,0 +1,148 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { languages } from '../fillers/monaco-editor-core'; + +const EMPTY_ELEMENTS: string[] = [ + 'area', + 'base', + 'br', + 'col', + 'embed', + 'hr', + 'img', + 'input', + 'keygen', + 'link', + 'menuitem', + 'meta', + 'param', + 'source', + 'track', + 'wbr' +]; + +export const conf: languages.LanguageConfiguration = { + wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g, + + // comments: { + // blockComment: ['{{!--', '--}}'] + // }, + + brackets: [ + [''], + ['<', '>'], + ['{{', '}}'], + ['{%', '%}'], + ['{', '}'], + ['(', ')'] + ], + + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '%', close: '%' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, + { open: "'", close: "'" } + ], + + surroundingPairs: [ + { open: '<', close: '>' }, + { open: '"', close: '"' }, + { open: "'", close: "'" } + ], + + onEnterRules: [ + { + beforeText: new RegExp( + `<(?!(?:${EMPTY_ELEMENTS.join('|')}))(\\w[\\w\\d]*)([^/>]*(?!/)>)[^<]*$`, + 'i' + ), + afterText: /^<\/(\w[\w\d]*)\s*>$/i, + action: { + indentAction: languages.IndentAction.IndentOutdent + } + }, + { + beforeText: new RegExp( + `<(?!(?:${EMPTY_ELEMENTS.join('|')}))(\\w[\\w\\d]*)([^/>]*(?!/)>)[^<]*$`, + 'i' + ), + action: { indentAction: languages.IndentAction.Indent } + } + ] +}; + +export const language = { + defaultToken: '', + tokenPostfix: '', + // ignoreCase: true, + + // The main tokenizer for our languages + tokenizer: { + root: [ + [/\{\%\s*comment\s*\%\}/, 'comment.start.liquid', '@comment'], + [/\{\{/, { token: '@rematch', switchTo: '@liquidInSimpleState.root' }], + [/\{\%/, { token: '@rematch', switchTo: '@liquidInSimpleState.root' }], + [/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']], + [/(<)([:\w]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]], + [/(<\/)(\w+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]], + [//, 'delimiter.html', '@pop'], + [/"([^"]*)"/, 'attribute.value'], + [/'([^']*)'/, 'attribute.value'], + [/[\w\-]+/, 'attribute.name'], + [/=/, 'delimiter'], + [/[ \t\r\n]+/] // whitespace + ], + + liquidInSimpleState: [ + [/\{\{/, 'delimiter.liquid'], + [/\}\}/, { token: 'delimiter.liquid', switchTo: '@$S2.$S3' }], + [/\{\%/, 'delimiter.output.liquid'], + [/\%\}/, { token: 'delimiter.liquid', switchTo: '@$S2.$S3' }], + { include: 'liquidRoot' } + ], + + liquidInTagState: [ + [/%\}/, { token: 'delimiter.output.liquid', switchTo: '@$S2.$S3' }], + // { include: 'liquidRoot' }, + [/[^%]/, 'wut'] + ], + + liquidRoot: [ + [/\d+(\.\d+)?/, 'number.liquid'], + [/"[^"]*"/, 'string.liquid'], + [/'[^']*'/, 'string.liquid'], + [/[\s]+/], + [/[^}|%]/, 'variable.liquid'] + ] + } +}; diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index cf90f679..0ab45b34 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -31,6 +31,7 @@ import './kotlin/kotlin.contribution'; import './less/less.contribution'; import './lexon/lexon.contribution'; import './lua/lua.contribution'; +import './liquid/liquid.contribution'; import './m3/m3.contribution'; import './markdown/markdown.contribution'; import './mips/mips.contribution'; From 5e4b3f14ae584570df162f1b32c2a3626f9272fb Mon Sep 17 00:00:00 2001 From: Matt Vague Date: Sun, 4 Apr 2021 16:53:29 -0700 Subject: [PATCH 2/3] Implement constants, keywords, raw, builtin tags, etc --- src/liquid/liquid.test.ts | 118 ++++++++++++++++++++++++++++--- src/liquid/liquid.ts | 145 +++++++++++++++++++++++++++++++++----- 2 files changed, 233 insertions(+), 30 deletions(-) diff --git a/src/liquid/liquid.test.ts b/src/liquid/liquid.test.ts index a24f567d..9b6cfedb 100644 --- a/src/liquid/liquid.test.ts +++ b/src/liquid/liquid.test.ts @@ -32,11 +32,11 @@ testTokenization( { startIndex: 0, type: 'delimiter.html' }, { startIndex: 1, type: 'tag.html' }, { startIndex: 3, type: 'delimiter.html' }, - { startIndex: 4, type: 'delimiter.liquid' }, + { startIndex: 4, type: 'delimiter.output.liquid' }, { startIndex: 6, type: '' }, { startIndex: 7, type: 'variable.liquid' }, { startIndex: 12, type: '' }, - { startIndex: 13, type: 'delimiter.liquid' }, + { startIndex: 13, type: 'delimiter.output.liquid' }, { startIndex: 15, type: 'delimiter.html' }, { startIndex: 17, type: 'tag.html' }, { startIndex: 19, type: 'delimiter.html' } @@ -52,17 +52,18 @@ testTokenization( { startIndex: 0, type: 'delimiter.html' }, { startIndex: 1, type: 'tag.html' }, { startIndex: 3, type: 'delimiter.html' }, - { startIndex: 4, type: 'delimiter.liquid' }, + { startIndex: 4, type: 'delimiter.output.liquid' }, { startIndex: 6, type: '' }, { startIndex: 7, type: 'number.liquid' }, { startIndex: 17, type: '' }, - { startIndex: 20, type: 'variable.liquid' }, + { startIndex: 20, type: 'predefined.liquid' }, { startIndex: 25, type: '' }, - { startIndex: 28, type: 'variable.liquid' }, + { startIndex: 28, type: 'predefined.liquid' }, + { startIndex: 35, type: 'variable.liquid' }, { startIndex: 36, type: '' }, { startIndex: 37, type: 'string.liquid' }, { startIndex: 41, type: '' }, - { startIndex: 43, type: 'delimiter.liquid' }, + { startIndex: 43, type: 'delimiter.output.liquid' }, { startIndex: 45, type: 'delimiter.html' }, { startIndex: 47, type: 'tag.html' }, { startIndex: 49, type: 'delimiter.html' } @@ -70,7 +71,7 @@ testTokenization( } ], - // Tag + // Simple Tag [ { line: '
{% render "files/file123.html" %}
', @@ -78,13 +79,13 @@ testTokenization( { startIndex: 0, type: 'delimiter.html' }, { startIndex: 1, type: 'tag.html' }, { startIndex: 4, type: 'delimiter.html' }, - { startIndex: 5, type: 'delimiter.output.liquid' }, + { startIndex: 5, type: 'delimiter.tag.liquid' }, { startIndex: 7, type: '' }, - { startIndex: 8, type: 'variable.liquid' }, + { startIndex: 8, type: 'predefined.liquid' }, { startIndex: 14, type: '' }, { startIndex: 15, type: 'string.liquid' }, { startIndex: 35, type: '' }, - { startIndex: 36, type: 'delimiter.liquid' }, + { startIndex: 36, type: 'delimiter.tag.liquid' }, { startIndex: 38, type: 'delimiter.html' }, { startIndex: 40, type: 'tag.html' }, { startIndex: 43, type: 'delimiter.html' } @@ -92,7 +93,78 @@ testTokenization( } ], - // Handlebars comment + // Tag with drop + [ + { + line: '
{{ thing.other_thing }}
', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 4, type: 'delimiter.html' }, + { startIndex: 5, type: 'delimiter.output.liquid' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'variable.liquid' }, + { startIndex: 13, type: '' }, + { startIndex: 14, type: 'variable.liquid' }, + { startIndex: 25, type: '' }, + { startIndex: 26, type: 'delimiter.output.liquid' }, + { startIndex: 28, type: 'delimiter.html' }, + { startIndex: 30, type: 'tag.html' }, + { startIndex: 33, type: 'delimiter.html' } + ] + } + ], + + // If tag / keywords / block style tags + [ + { + line: + '
{% if true=false %}
True
{% else %}
False
{% endif %}
', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 4, type: 'delimiter.html' }, + { startIndex: 5, type: 'delimiter.tag.liquid' }, + { startIndex: 7, type: '' }, + { startIndex: 8, type: 'predefined.liquid' }, + { startIndex: 10, type: '' }, + { startIndex: 11, type: 'keyword.liquid' }, + { startIndex: 15, type: '' }, + { startIndex: 16, type: 'keyword.liquid' }, + { startIndex: 21, type: '' }, + { startIndex: 22, type: 'delimiter.tag.liquid' }, + { startIndex: 24, type: 'delimiter.html' }, + { startIndex: 25, type: 'tag.html' }, + { startIndex: 28, type: 'delimiter.html' }, + { startIndex: 29, type: '' }, + { startIndex: 33, type: 'delimiter.html' }, + { startIndex: 35, type: 'tag.html' }, + { startIndex: 38, type: 'delimiter.html' }, + { startIndex: 39, type: 'delimiter.tag.liquid' }, + { startIndex: 41, type: '' }, + { startIndex: 42, type: 'predefined.liquid' }, + { startIndex: 46, type: '' }, + { startIndex: 47, type: 'delimiter.tag.liquid' }, + { startIndex: 49, type: 'delimiter.html' }, + { startIndex: 50, type: 'tag.html' }, + { startIndex: 53, type: 'delimiter.html' }, + { startIndex: 54, type: '' }, + { startIndex: 59, type: 'delimiter.html' }, + { startIndex: 61, type: 'tag.html' }, + { startIndex: 64, type: 'delimiter.html' }, + { startIndex: 65, type: 'delimiter.tag.liquid' }, + { startIndex: 67, type: '' }, + { startIndex: 68, type: 'predefined.liquid' }, + { startIndex: 73, type: '' }, + { startIndex: 74, type: 'delimiter.tag.liquid' }, + { startIndex: 76, type: 'delimiter.html' }, + { startIndex: 78, type: 'tag.html' }, + { startIndex: 81, type: 'delimiter.html' } + ] + } + ], + + // Comment tag [ { line: '
Anything you put between {% comment %} and {% endcomment %} tags
', @@ -110,6 +182,30 @@ testTokenization( { startIndex: 74, type: 'delimiter.html' } ] } + ], + + // Raw tag + [ + { + line: + '
Everything here should be escaped {% raw %} In Handlebars, {{ this }} will be HTML-escaped, but {{{ that }}} will not. {% endraw %}
', + tokens: [ + { startIndex: 0, type: 'delimiter.html' }, + { startIndex: 1, type: 'tag.html' }, + { startIndex: 4, type: 'delimiter.html' }, + { startIndex: 5, type: '' }, + { startIndex: 39, type: 'delimiter.tag.liquid' }, + { startIndex: 41, type: '' }, + { startIndex: 42, type: 'delimiter.tag.liquid' }, + { startIndex: 48, type: '' }, + { startIndex: 124, type: 'delimiter.tag.liquid' }, + { startIndex: 126, type: '' }, + { startIndex: 134, type: 'delimiter.tag.liquid' }, + { startIndex: 136, type: 'delimiter.html' }, + { startIndex: 138, type: 'tag.html' }, + { startIndex: 141, type: 'delimiter.html' } + ] + } ] ] ); diff --git a/src/liquid/liquid.ts b/src/liquid/liquid.ts index cccb5033..62c287d6 100644 --- a/src/liquid/liquid.ts +++ b/src/liquid/liquid.ts @@ -27,9 +27,7 @@ const EMPTY_ELEMENTS: string[] = [ export const conf: languages.LanguageConfiguration = { wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g, - // comments: { - // blockComment: ['{{!--', '--}}'] - // }, + // TODO support if,else,elseif,for,in and other built in keywords brackets: [ [''], @@ -79,14 +77,100 @@ export const conf: languages.LanguageConfiguration = { export const language = { defaultToken: '', tokenPostfix: '', - // ignoreCase: true, + + builtinTags: [ + 'if', + 'else', + 'elseif', + 'endif', + 'render', + 'assign', + 'capture', + 'endcapture', + 'case', + 'endcase', + 'comment', + 'endcomment', + 'cycle', + 'decrement', + 'for', + 'endfor', + 'include', + 'increment', + 'layout', + 'raw', + 'endraw', + 'render', + 'tablerow', + 'endtablerow', + 'unless', + 'endunless' + ], + + builtinFilters: [ + 'abs', + 'append', + 'at_least', + 'at_most', + 'capitalize', + 'ceil', + 'compact', + 'date', + 'default', + 'divided_by', + 'downcase', + 'escape', + 'escape_once', + 'first', + 'floor', + 'join', + 'json', + 'last', + 'lstrip', + 'map', + 'minus', + 'modulo', + 'newline_to_br', + 'plus', + 'prepend', + 'remove', + 'remove_first', + 'replace', + 'replace_first', + 'reverse', + 'round', + 'rstrip', + 'size', + 'slice', + 'sort', + 'sort_natural', + 'split', + 'strip', + 'strip_html', + 'strip_newlines', + 'times', + 'truncate', + 'truncatewords', + 'uniq', + 'upcase', + 'url_decode', + 'url_encode', + 'where' + ], + + constants: ['true', 'false'], + operators: ['==', '!=', '>', '<', '>=', '<='], + + symbol: /[=>)/, ['delimiter.html', 'tag.html', 'delimiter.html']], [/(<)([:\w]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]], [/(<\/)(\w+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]], @@ -105,14 +189,14 @@ export const language = { /\{\{/, { token: '@rematch', - switchTo: '@liquidInSimpleState.otherTag' + switchTo: '@liquidState.otherTag' } ], [ - /\{%/, + /\{\%/, { token: '@rematch', - switchTo: '@liquidInSimpleState.otherTag' + switchTo: '@liquidState.otherTag' } ], [/\/?>/, 'delimiter.html', '@pop'], @@ -123,25 +207,48 @@ export const language = { [/[ \t\r\n]+/] // whitespace ], - liquidInSimpleState: [ - [/\{\{/, 'delimiter.liquid'], - [/\}\}/, { token: 'delimiter.liquid', switchTo: '@$S2.$S3' }], - [/\{\%/, 'delimiter.output.liquid'], - [/\%\}/, { token: 'delimiter.liquid', switchTo: '@$S2.$S3' }], + liquidState: [ + [/\{\{/, 'delimiter.output.liquid'], + [/\}\}/, { token: 'delimiter.output.liquid', switchTo: '@$S2.$S3' }], + [/\{\%/, 'delimiter.tag.liquid'], + [/raw\s*\%\}/, 'delimiter.tag.liquid', '@liquidRaw'], + [/\%\}/, { token: 'delimiter.tag.liquid', switchTo: '@$S2.$S3' }], { include: 'liquidRoot' } ], - liquidInTagState: [ - [/%\}/, { token: 'delimiter.output.liquid', switchTo: '@$S2.$S3' }], - // { include: 'liquidRoot' }, - [/[^%]/, 'wut'] + liquidRaw: [ + [/^(?!\{\%\s*endraw\s*\%\}).+/], + [/\{\%/, 'delimiter.tag.liquid'], + [/@identifier/], + [/\%\}/, { token: 'delimiter.tag.liquid', next: '@root' }], ], liquidRoot: [ [/\d+(\.\d+)?/, 'number.liquid'], [/"[^"]*"/, 'string.liquid'], [/'[^']*'/, 'string.liquid'], - [/[\s]+/], + [/\s+/], + [ + /@symbol/, + { + cases: { + '@operators': 'operator.liquid', + '@default': '' + } + } + ], + [/\./], + [ + /@identifier/, + { + cases: { + '@constants': 'keyword.liquid', + '@builtinFilters': 'predefined.liquid', + '@builtinTags': 'predefined.liquid', + '@default': 'variable.liquid' + } + } + ], [/[^}|%]/, 'variable.liquid'] ] } From 9928c69e43241712e68e777bc109dbaf47a7fa93 Mon Sep 17 00:00:00 2001 From: Matt Vague Date: Sun, 4 Apr 2021 18:19:14 -0700 Subject: [PATCH 3/3] Clean up --- src/liquid/liquid.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/liquid/liquid.ts b/src/liquid/liquid.ts index 62c287d6..9bad701b 100644 --- a/src/liquid/liquid.ts +++ b/src/liquid/liquid.ts @@ -27,8 +27,6 @@ const EMPTY_ELEMENTS: string[] = [ export const conf: languages.LanguageConfiguration = { wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g, - // TODO support if,else,elseif,for,in and other built in keywords - brackets: [ [''], ['<', '>'], @@ -164,11 +162,9 @@ export const language = { symbol: /[=>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],