From 0a6d139688b01e43a844a24c88292c61a66fa8c5 Mon Sep 17 00:00:00 2001 From: KapitanOczywisty <44417092+KapitanOczywisty@users.noreply.github.com> Date: Sat, 5 Sep 2020 03:29:27 +0200 Subject: [PATCH 1/3] Semantic tokens provider example --- ...ices-semantic-tokens-provider-example.html | 206 ++++++++++++++++++ test/playground.generated/index.html | 1 + website/playground/new-samples/all.js | 6 + .../sample.css | 0 .../sample.html | 1 + .../sample.js | 162 ++++++++++++++ 6 files changed, 376 insertions(+) create mode 100644 test/playground.generated/extending-language-services-semantic-tokens-provider-example.html create mode 100644 website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.css create mode 100644 website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.html create mode 100644 website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js diff --git a/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html b/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html new file mode 100644 index 00000000..5157d161 --- /dev/null +++ b/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html @@ -0,0 +1,206 @@ + + + + + + + + + +[<< BACK]
+THIS IS A GENERATED FILE VIA gulp generate-test-samples + +
+ +
+
+ + +
+ + + +
+
+ + + + + + \ No newline at end of file diff --git a/test/playground.generated/index.html b/test/playground.generated/index.html index 77a2dc93..5f539f23 100644 --- a/test/playground.generated/index.html +++ b/test/playground.generated/index.html @@ -33,6 +33,7 @@ THIS IS A GENERATED FILE VIA gulp generate-test-samples

Extending Language Services > Symbols provider example
Extending Language Services > Folding provider example
Extending Language Services > Hover provider example
+Extending Language Services > Semantic tokens provider example
Extending Language Services > Configure JavaScript defaults
Extending Language Services > Configure JSON defaults diff --git a/website/playground/new-samples/all.js b/website/playground/new-samples/all.js index ec1190e3..df3f786c 100644 --- a/website/playground/new-samples/all.js +++ b/website/playground/new-samples/all.js @@ -157,6 +157,12 @@ var PLAY_SAMPLES = [ id: "extending-language-services-hover-provider-example", path: "extending-language-services/hover-provider-example" }, + { + chapter: "Extending Language Services", + name: "Semantic tokens provider example", + id: "extending-language-services-semantic-tokens-provider-example", + path: "extending-language-services/semantic-tokens-provider-example" + }, { chapter: "Extending Language Services", name: "Configure JavaScript defaults", diff --git a/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.css b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.css new file mode 100644 index 00000000..e69de29b diff --git a/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.html b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.html new file mode 100644 index 00000000..b2e43e28 --- /dev/null +++ b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.html @@ -0,0 +1 @@ +
diff --git a/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js new file mode 100644 index 00000000..5d91fd32 --- /dev/null +++ b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js @@ -0,0 +1,162 @@ +/** @type {monaco.languages.SemanticTokensLegend} */ +const legend = { + tokenTypes: [ + 'comment', 'string', 'keyword', 'number', 'regexp', 'operator', 'namespace', + 'type', 'struct', 'class', 'interface', 'enum', 'typeParameter', 'function', + 'member', 'macro', 'variable', 'parameter', 'property', 'label' + ], + tokenModifiers: [ + 'declaration', 'documentation', 'readonly', 'static', 'abstract', 'deprecated', + 'modification', 'async' + ] +}; + +/** @type {(type: string)=>number} */ +function getType(type) { + return legend.tokenTypes.indexOf(type); +} + +/** @type {(modifier: string[]|string|null)=>number} */ +function getModifier(modifiers) { + if (typeof modifiers === 'string') modifiers = [modifiers]; + if (Array.isArray(modifiers)) { + let nModifiers = 0; + for (let modifier of modifiers) { + nModifier = legend.tokenModifiers.indexOf(modifier); + if (nModifier > -1) { + nModifiers |= (1 << nModifier) >>> 0; + } + } + return nModifiers; + } else { + return 0; + } +} + +const tokenPattern = new RegExp('(?<=\\[)([a-zA-Z]+)((?:\\.[a-zA-Z]+)*)(?=\\])', 'g'); + +monaco.languages.registerDocumentSemanticTokensProvider('plaintext', { + getLegend: function () { + return legend; + }, + provideDocumentSemanticTokens: function (model, lastResultId, token) { + const lines = model.getLinesContent(); + + /** @type {number[]} */ + const data = []; + + let prevLine = 0; + let prevChar = 0; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + for (let match = null; match = tokenPattern.exec(line);) { + // translate token and modifiers to number representations + let type = getType(match[1]); + if (type === -1) continue; + let modifier = match[2].length + ? getModifier(match[2].split('.').slice(1)) + : 0; + + data.push( + // translate line to deltaLine + i - prevLine, + // for the same line, translate start to deltaStart + prevLine === i ? match.index - prevChar : match.index, + match[0].length, + type, + modifier + ); + + prevLine = i; + prevChar = match.index; + } + } + return { + data: new Uint32Array(data), + resultId: null + }; + }, + releaseDocumentSemanticTokens: function (resultId) { } +}); + +// add some missing tokens +monaco.editor.defineTheme('myCustomTheme', { + base: 'vs', + inherit: true, + rules: [ + { token: 'comment', foreground: 'aaaaaa', fontStyle: 'italic' }, + { token: 'keyword', foreground: 'ce63eb' }, + { token: 'operator', foreground: '000000' }, + { token: 'namespace', foreground: '66afce' }, + + { token: 'type', foreground: '1db010' }, + { token: 'struct', foreground: '0000ff' }, + { token: 'class', foreground: '0000ff', fontStyle: 'bold' }, + { token: 'interface', foreground: '007700', fontStyle: 'bold' }, + { token: 'enum', foreground: '0077ff', fontStyle: 'bold' }, + { token: 'typeParameter', foreground: '1db010' }, + { token: 'function', foreground: '94763a' }, + + { token: 'member', foreground: '94763a' }, + { token: 'macro', foreground: '615a60' }, + { token: 'variable', foreground: '3e5bbf' }, + { token: 'parameter', foreground: '3e5bbf' }, + { token: 'property', foreground: '3e5bbf' }, + { token: 'label', foreground: '615a60' }, + + { token: 'type.static', fontStyle: 'bold' }, + { token: 'class.static', foreground: 'ff0000', fontStyle: 'bold' } + ] +}); + +const editor = monaco.editor.create(document.getElementById("container"), { + value: `Available token types: + [comment] [string] [keyword] [number] [regexp] [operator] [namespace] + [type] [struct] [class] [interface] [enum] [typeParameter] [function] + [member] [macro] [variable] [parameter] [property] [label] + +Available token modifiers: + [type.declaration] [type.documentation] [type.member] [type.static] + [type.abstract] [type.deprecated] [type.modification] [type.async] + +Some examples: + [class.static.token] [type.static.abstract] + [class.static.token] [type.static] + + [struct] + + [function.private] + +An error case: + [notInLegend]`, + language: "plaintext", + theme: 'myCustomTheme', + // semantic tokens provider is disabled by default + 'semanticHighlighting.enabled': true +}); + +// currently there isn't builtin token handling +editor._themeService._knownThemes.forEach(function (theme) { + theme.getTokenStyleMetadata = (type, modifiers) => { + // use theme rules match + const style = theme._tokenTheme._root.match([type, ...modifiers].join('.')); + return { + foreground: style._foreground, + italic: style._fontStyle & 1, + bold: style._fontStyle & 2, + underline: style._fontStyle & 4 + }; + }; +}); + +// press F4 to change theme +editor.addCommand(monaco.KeyCode.F4, function () { + switch (editor._themeService.getTheme().themeName) { + case 'vs': monaco.editor.setTheme('vs-dark'); break; + case 'vs-dark': monaco.editor.setTheme('hc-black'); break; + case 'hc-black': monaco.editor.setTheme('myCustomTheme'); break; + case 'myCustomTheme': monaco.editor.setTheme('vs'); break; + } +}); From 51fe304f01575a9495bc4db155e7351378e36c3f Mon Sep 17 00:00:00 2001 From: KapitanOczywisty <44417092+KapitanOczywisty@users.noreply.github.com> Date: Tue, 8 Sep 2020 20:05:50 +0200 Subject: [PATCH 2/3] Remove ES6 syntax from example --- ...ices-semantic-tokens-provider-example.html | 44 ++++++++++--------- .../sample.js | 44 ++++++++++--------- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html b/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html index 5157d161..dd991a39 100644 --- a/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html +++ b/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html @@ -149,25 +149,27 @@ monaco.editor.defineTheme('myCustomTheme', { }); const editor = monaco.editor.create(document.getElementById("container"), { - value: `Available token types: - [comment] [string] [keyword] [number] [regexp] [operator] [namespace] - [type] [struct] [class] [interface] [enum] [typeParameter] [function] - [member] [macro] [variable] [parameter] [property] [label] - -Available token modifiers: - [type.declaration] [type.documentation] [type.member] [type.static] - [type.abstract] [type.deprecated] [type.modification] [type.async] - -Some examples: - [class.static.token] [type.static.abstract] - [class.static.token] [type.static] - - [struct] - - [function.private] - -An error case: - [notInLegend]`, + value: [ + 'Available token types:', + ' [comment] [string] [keyword] [number] [regexp] [operator] [namespace]', + ' [type] [struct] [class] [interface] [enum] [typeParameter] [function]', + ' [member] [macro] [variable] [parameter] [property] [label]', + '', + 'Available token modifiers:', + ' [type.declaration] [type.documentation] [type.member] [type.static]', + ' [type.abstract] [type.deprecated] [type.modification] [type.async]', + '', + 'Some examples:', + ' [class.static.token] [type.static.abstract]', + ' [class.static.token] [type.static]', + '', + ' [struct]', + '', + ' [function.private]', + '', + 'An error case:', + ' [notInLegend]' + ].join('\n'), language: "plaintext", theme: 'myCustomTheme', // semantic tokens provider is disabled by default @@ -176,9 +178,9 @@ An error case: // currently there isn't builtin token handling editor._themeService._knownThemes.forEach(function (theme) { - theme.getTokenStyleMetadata = (type, modifiers) => { + theme.getTokenStyleMetadata = function (type, modifiers) { // use theme rules match - const style = theme._tokenTheme._root.match([type, ...modifiers].join('.')); + const style = theme._tokenTheme._root.match([type].concat(modifiers).join('.')); return { foreground: style._foreground, italic: style._fontStyle & 1, diff --git a/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js index 5d91fd32..cb4bf446 100644 --- a/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js +++ b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js @@ -112,25 +112,27 @@ monaco.editor.defineTheme('myCustomTheme', { }); const editor = monaco.editor.create(document.getElementById("container"), { - value: `Available token types: - [comment] [string] [keyword] [number] [regexp] [operator] [namespace] - [type] [struct] [class] [interface] [enum] [typeParameter] [function] - [member] [macro] [variable] [parameter] [property] [label] - -Available token modifiers: - [type.declaration] [type.documentation] [type.member] [type.static] - [type.abstract] [type.deprecated] [type.modification] [type.async] - -Some examples: - [class.static.token] [type.static.abstract] - [class.static.token] [type.static] - - [struct] - - [function.private] - -An error case: - [notInLegend]`, + value: [ + 'Available token types:', + ' [comment] [string] [keyword] [number] [regexp] [operator] [namespace]', + ' [type] [struct] [class] [interface] [enum] [typeParameter] [function]', + ' [member] [macro] [variable] [parameter] [property] [label]', + '', + 'Available token modifiers:', + ' [type.declaration] [type.documentation] [type.member] [type.static]', + ' [type.abstract] [type.deprecated] [type.modification] [type.async]', + '', + 'Some examples:', + ' [class.static.token] [type.static.abstract]', + ' [class.static.token] [type.static]', + '', + ' [struct]', + '', + ' [function.private]', + '', + 'An error case:', + ' [notInLegend]' + ].join('\n'), language: "plaintext", theme: 'myCustomTheme', // semantic tokens provider is disabled by default @@ -139,9 +141,9 @@ An error case: // currently there isn't builtin token handling editor._themeService._knownThemes.forEach(function (theme) { - theme.getTokenStyleMetadata = (type, modifiers) => { + theme.getTokenStyleMetadata = function (type, modifiers) { // use theme rules match - const style = theme._tokenTheme._root.match([type, ...modifiers].join('.')); + const style = theme._tokenTheme._root.match([type].concat(modifiers).join('.')); return { foreground: style._foreground, italic: style._fontStyle & 1, From ee8392eb0c73f8ff68b36c4345ff7f71c21ea40e Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Thu, 17 Sep 2020 22:44:16 +0200 Subject: [PATCH 3/3] Avoid using internals --- ...ices-semantic-tokens-provider-example.html | 30 ++++--------------- .../sample.js | 30 ++++--------------- 2 files changed, 12 insertions(+), 48 deletions(-) diff --git a/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html b/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html index dd991a39..c680a975 100644 --- a/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html +++ b/test/playground.generated/extending-language-services-semantic-tokens-provider-example.html @@ -55,7 +55,9 @@ function getType(type) { /** @type {(modifier: string[]|string|null)=>number} */ function getModifier(modifiers) { - if (typeof modifiers === 'string') modifiers = [modifiers]; + if (typeof modifiers === 'string') { + modifiers = [modifiers]; + } if (Array.isArray(modifiers)) { let nModifiers = 0; for (let modifier of modifiers) { @@ -91,7 +93,9 @@ monaco.languages.registerDocumentSemanticTokensProvider('plaintext', { for (let match = null; match = tokenPattern.exec(line);) { // translate token and modifiers to number representations let type = getType(match[1]); - if (type === -1) continue; + if (type === -1) { + continue; + } let modifier = match[2].length ? getModifier(match[2].split('.').slice(1)) : 0; @@ -176,29 +180,7 @@ const editor = monaco.editor.create(document.getElementById("container"), { 'semanticHighlighting.enabled': true }); -// currently there isn't builtin token handling -editor._themeService._knownThemes.forEach(function (theme) { - theme.getTokenStyleMetadata = function (type, modifiers) { - // use theme rules match - const style = theme._tokenTheme._root.match([type].concat(modifiers).join('.')); - return { - foreground: style._foreground, - italic: style._fontStyle & 1, - bold: style._fontStyle & 2, - underline: style._fontStyle & 4 - }; - }; -}); -// press F4 to change theme -editor.addCommand(monaco.KeyCode.F4, function () { - switch (editor._themeService.getTheme().themeName) { - case 'vs': monaco.editor.setTheme('vs-dark'); break; - case 'vs-dark': monaco.editor.setTheme('hc-black'); break; - case 'hc-black': monaco.editor.setTheme('myCustomTheme'); break; - case 'myCustomTheme': monaco.editor.setTheme('vs'); break; - } -}); /*----------------------------------------SAMPLE JS END*/ diff --git a/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js index cb4bf446..790173ca 100644 --- a/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js +++ b/website/playground/new-samples/extending-language-services/semantic-tokens-provider-example/sample.js @@ -18,7 +18,9 @@ function getType(type) { /** @type {(modifier: string[]|string|null)=>number} */ function getModifier(modifiers) { - if (typeof modifiers === 'string') modifiers = [modifiers]; + if (typeof modifiers === 'string') { + modifiers = [modifiers]; + } if (Array.isArray(modifiers)) { let nModifiers = 0; for (let modifier of modifiers) { @@ -54,7 +56,9 @@ monaco.languages.registerDocumentSemanticTokensProvider('plaintext', { for (let match = null; match = tokenPattern.exec(line);) { // translate token and modifiers to number representations let type = getType(match[1]); - if (type === -1) continue; + if (type === -1) { + continue; + } let modifier = match[2].length ? getModifier(match[2].split('.').slice(1)) : 0; @@ -139,26 +143,4 @@ const editor = monaco.editor.create(document.getElementById("container"), { 'semanticHighlighting.enabled': true }); -// currently there isn't builtin token handling -editor._themeService._knownThemes.forEach(function (theme) { - theme.getTokenStyleMetadata = function (type, modifiers) { - // use theme rules match - const style = theme._tokenTheme._root.match([type].concat(modifiers).join('.')); - return { - foreground: style._foreground, - italic: style._fontStyle & 1, - bold: style._fontStyle & 2, - underline: style._fontStyle & 4 - }; - }; -}); -// press F4 to change theme -editor.addCommand(monaco.KeyCode.F4, function () { - switch (editor._themeService.getTheme().themeName) { - case 'vs': monaco.editor.setTheme('vs-dark'); break; - case 'vs-dark': monaco.editor.setTheme('hc-black'); break; - case 'hc-black': monaco.editor.setTheme('myCustomTheme'); break; - case 'myCustomTheme': monaco.editor.setTheme('vs'); break; - } -});