From c1c7d4b76a5a019de91d5abaee566166a63ae80f Mon Sep 17 00:00:00 2001 From: CHOE Date: Mon, 30 Dec 2019 22:31:01 +0900 Subject: [PATCH 1/2] add support for reStructuredText --- src/monaco.contribution.ts | 1 + src/rst/restructuredtext.contribution.ts | 14 + src/rst/restructuredtext.test.ts | 492 +++++++++++++++++++++++ src/rst/restructuredtext.ts | 169 ++++++++ 4 files changed, 676 insertions(+) create mode 100644 src/rst/restructuredtext.contribution.ts create mode 100644 src/rst/restructuredtext.test.ts create mode 100644 src/rst/restructuredtext.ts diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index 816788b8..3e294e71 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -63,3 +63,4 @@ import './typescript/typescript.contribution'; import './vb/vb.contribution'; import './xml/xml.contribution'; import './yaml/yaml.contribution'; +import './rst/restructuredtext.contribution'; diff --git a/src/rst/restructuredtext.contribution.ts b/src/rst/restructuredtext.contribution.ts new file mode 100644 index 00000000..c958ff8b --- /dev/null +++ b/src/rst/restructuredtext.contribution.ts @@ -0,0 +1,14 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { registerLanguage } from '../_.contribution'; + +registerLanguage({ + id: 'restructuredtext', + extensions: ['.rst'], + aliases: ['reStructuredText', 'restructuredtext'], + loader: () => import('./restructuredtext') +}); diff --git a/src/rst/restructuredtext.test.ts b/src/rst/restructuredtext.test.ts new file mode 100644 index 00000000..edf9ef6c --- /dev/null +++ b/src/rst/restructuredtext.test.ts @@ -0,0 +1,492 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { testTokenization } from '../test/testRunner'; + +testTokenization('restructuredtext', [ + [ + { + line: '#####', + tokens: [ + { startIndex: 0, type: 'keyword.rst' } + ] + } + ], + [ + { + line: 'strong **strong**', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 7, type: 'strong.rst' }, + ] + }, + { + line: 'emphasis *emphasis*', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 9, type: 'emphasis.rst' }, + ] + } + ], + [ + { + line: '.. [23] This is the footnote', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 6, type: '' }, + ] + }, + { + line: '.. [#ab] This is the footnote', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 7, type: '' }, + ] + }, + { + line: '.. [#] This is the footnote', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 5, type: '' }, + ] + }, + { + line: '.. [*] This is the footnote', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 5, type: '' }, + ] + }, + { + line: '', + tokens: [] + }, + { + line: ' .. [23] This is not the footnote', + tokens: [ + { startIndex: 0, type: '' }, + ] + } + ], + [ + { + line: '[23]_', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'string.link.rst' }, + { startIndex: 3, type: '' } + + ] + }, + { + line: '[*]_', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'string.link.rst' }, + { startIndex: 2, type: '' } + ] + }, + { + line: '[#]_', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'string.link.rst' }, + { startIndex: 2, type: '' } + ] + }, + { + line: '[#abc]_ [#]_', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'string.link.rst' }, + { startIndex: 5, type: '' }, + { startIndex: 9, type: 'string.link.rst' }, + { startIndex: 10, type: '' }, + ] + } + ], + [ + { + line: '.. [A3] This is the citation', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 6, type: '' } + ] + } + ], + [ + { + line: ' .. [A3] This is not the citation', + tokens: [ + { startIndex: 0, type: '' }, + ] + } + ], + [ + { + line: '.. [A3] This is the citation', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 6, type: '' }, + ] + }, + { + line: ' first line', + tokens: [ + { startIndex: 0, type: '' } + ] + }, + { + line: ' second line', + tokens: [ + { startIndex: 0, type: '' } + ] + }, + { + line: 'new line starts', + tokens: [ + { startIndex: 0, type: '' } + ] + } + ], + [ + { + line: '[A3]_', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 1, type: 'string.link.rst' }, + { startIndex: 3, type: '' }, + ] + } + ], + [ + { + line: 'Interpreted Text `text`', + tokens: [ + { startIndex: 0, type: '' }, + ] + }, + { + line: ' .. _`text`: paragraph', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 6, type: 'string.link.rst' }, + { startIndex: 10, type: '' }, + ] + }, + { + line: 'Interpreted Text :role:`text`', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 18, type: 'keyword.rst' }, + { startIndex: 22, type: '' }, + ] + }, + { + line: 'Interpreted Text `text`:role:', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 24, type: 'keyword.rst' }, + { startIndex: 28, type: '' }, + ] + } + ], + [ + { + line: '.. note:: This is a directive', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 3, type: 'keyword.rst' }, + { startIndex: 7, type: '' }, + ] + } + ], + [ + { + line: 'link .. _link: this is not a hyperlink', + tokens: [ + { startIndex: 0, type: '' }, + ] + }, + { + line: '.. _link: this is a hyperlink', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 8, type: '' }, + { startIndex: 10, type: 'string.link.rst' }, + ] + }, + { + line: '.. _`link`: this is a hyperlink', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 10, type: '' }, + { startIndex: 12, type: 'string.link.rst' }, + ] + }, + { + line: '.. __: this is a anonymous hyperlink', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 7, type: 'string.link.rst' }, + ] + }, + { + line: '__: this is a anonymous hyperlink', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + ] + } + ], + [ + { + line: '...... _`this is a inline internal target`', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 9, type: 'string.link.rst' }, + { startIndex: 41, type: '' }, + ] + } + ], + [ + { + line: '.. |biohazard| image:: biohazard.png', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'string.link.rst' }, + { startIndex: 13, type: '' }, + { startIndex: 15, type: 'keyword.rst' }, + { startIndex: 20, type: '' }, + ] + }, + { + line: '... |biohazard| ...', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 5, type: 'string.link.rst' }, + { startIndex: 14, type: '' }, + ] + } + ], + [ + { + line: 'ref_ `ref`_ ref__ `ref`__', + tokens: [ + { startIndex: 0, type: 'string.link.rst' }, + { startIndex: 3, type: '' }, + { startIndex: 5, type: 'string.link.rst' }, + { startIndex: 10, type: '' }, + { startIndex: 12, type: 'string.link.rst' }, + { startIndex: 15, type: '' }, + { startIndex: 18, type: 'string.link.rst' }, + { startIndex: 23, type: '' }, + ] + } + ], + [ + { + line: '.... `title `_', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 6, type: 'string.link.rst' }, + { startIndex: 12, type: '' }, + { startIndex: 13, type: 'string.link.rst' }, + { startIndex: 30, type: '' }, + ] + } + ], + [ + { + line: '::', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + ] + }, + { + line: '', + tokens: [ + ] + }, + { + line: ' first line', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 1, type: '' }, + ] + }, + { + line: ' second line', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 1, type: '' }, + ] + }, + { + line: '', + tokens: [ + ] + }, + { + line: ' paragraph', + tokens: [ + { startIndex: 0, type: '' }, + ] + }, + ], + [ + { + line: 'desc ::', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 4, type: 'keyword.rst' }, + ] + }, + { + line: '', + tokens: [ + ] + }, + { + line: '>>first line', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 2, type: '' }, + ] + }, + { + line: '>', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + ] + }, + { + line: '>second line', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 1, type: '' }, + ] + }, + { + line: '', + tokens: [ + ] + }, + { + line: ' paragraph', + tokens: [ + { startIndex: 0, type: '' }, + ] + }, + ], + [ + { + line: '.. comment', + tokens: [ + { startIndex: 0, type: '' }, + { startIndex: 2, type: 'comment.rst' }, + ] + }, + { + line: ' firstline', + tokens: [ + { startIndex: 0, type: 'comment.rst' }, + ] + }, + { + line: '', + tokens: [ + ] + }, + { + line: ' paragraph', + tokens: [ + { startIndex: 0, type: '' }, + ] + }, + ], + [ + { + line: '==', + tokens: [ + { startIndex: 0, type: '' }, + ], + }, + { + line: '===', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + ], + }, + ], + [ + { + line: '1. item', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 3, type: '' }, + ], + }, + { + line: 'a. item', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 3, type: '' }, + ], + }, + { + line: '* item', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 2, type: '' }, + ], + }, + { + line: '- item', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 2, type: '' }, + ], + }, + { + line: '1) item', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 3, type: '' }, + ], + }, + { + line: '(a) item', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + { startIndex: 4, type: '' }, + ], + }, + ], + [ + { + line: '+------------------------+------------+----------+----------+', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + ], + }, + { + line: '+========================+============+==========+==========+', + tokens: [ + { startIndex: 0, type: 'keyword.rst' }, + ], + }, + ], +]); diff --git a/src/rst/restructuredtext.ts b/src/rst/restructuredtext.ts new file mode 100644 index 00000000..241bb3f5 --- /dev/null +++ b/src/rst/restructuredtext.ts @@ -0,0 +1,169 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import IRichLanguageConfiguration = monaco.languages.LanguageConfiguration; +import ILanguage = monaco.languages.IMonarchLanguage; + +export const conf: IRichLanguageConfiguration = { + brackets: [ + ['{', '}'], + ['[', ']'], + ['(', ')'] + ], + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '<', close: '>', notIn: ['string'] } + ], + surroundingPairs: [ + { open: '(', close: ')' }, + { open: '[', close: ']' }, + { open: '`', close: '`' }, + ], + folding: { + markers: { + start: new RegExp("^\\s*"), + end: new RegExp("^\\s*") + } + } +}; + +export const language = { + defaultToken: '', + tokenPostfix: '.rst', + + control: /[\\`*_\[\]{}()#+\-\.!]/, + escapes: /\\(?:@control)/, + + empty: [ + 'area', 'base', 'basefont', 'br', 'col', 'frame', + 'hr', 'img', 'input', 'isindex', 'link', 'meta', 'param' + ], + + alphanumerics: /[A-Za-z0-9]/, + alphanumericsplus: /[A-Za-z0-9-_+:.]/, + simpleRefNameWithoutBq: /(?:@alphanumerics@alphanumericsplus*@alphanumerics)+|(?:@alphanumerics+)/, + simpleRefName: /(?:`@simpleRefNameWithoutBq`|@simpleRefNameWithoutBq)/, + phrase: /@simpleRefName(?:\s@simpleRefName)*/, + citationName: /[A-Za-z][A-Za-z0-9-_.]*/, + blockLiteralStart: /(?:[!"#$%&'()*+,-./:;<=>?@\[\]^_`{|}~]|[\s])/, + precedingChars: /(?:[ -:/'"<([{])/, + followingChars: /(?:[ -.,:;!?/'")\]}>]|$)/, + punctuation: /(=|-|~|`|#|"|\^|\+|\*|:|\.|'|_|\+)/, + tokenizer: { + root: [ + //sections + [/^(@punctuation{3,}$){1,1}?/, 'keyword'], + + //line-blocks + //No rules on it + + //bullet-lists + [/^\s*([\*\-+‣•]|[a-zA-Z0-9]+\.|\([a-zA-Z0-9]+\)|[a-zA-Z0-9]+\))\s/, 'keyword'], + + //literal-blocks + [/([ ]::)\s*$/, 'keyword', '@blankLineOfLiteralBlocks'], + [/(::)\s*$/, 'keyword', '@blankLineOfLiteralBlocks'], + + { include: '@tables' }, + + { include: '@explicitMarkupBlocks' }, + + { include: '@inlineMarkup' }, + ], + explicitMarkupBlocks: [ + //citations + { include: '@citations' }, + //footnotes + { include: '@footnotes' }, + //directives + [/^(\.\.\s)(@simpleRefName)(::\s)(.*)$/, [{ token: '', next: 'subsequentLines' }, 'keyword', '', '']], + + //hyperlink-targets + [/^(\.\.)(\s+)(_)(@simpleRefName)(:)(\s+)(.*)/, [{ token: '', next: 'hyperlinks' }, '', '', 'string.link', '', '', 'string.link']], + + //anonymous-hyperlinks + [/^((?:(?:\.\.)(?:\s+))?)(__)(:)(\s+)(.*)/, [{ token: '', next: 'subsequentLines' }, '', '', '', 'string.link']], + [/^(__\s+)(.+)/, ['', 'string.link']], + + //substitution-definitions + [/^(\.\.)( \|)([^| ]+[^|]*[^| ]*)(\| )(@simpleRefName)(:: .*)/, [{ token: '', next: 'subsequentLines' }, '', 'string.link', '', 'keyword', ''], '@rawBlocks'], + [/(\|)([^| ]+[^|]*[^| ]*)(\|_{0,2})/, ['', 'string.link', '']], + + //comments + [/^(\.\.)([ ].*)$/, [{ token: '', next: '@comments' }, 'comment']], + ], + inlineMarkup: [ + { include: '@citationsReference' }, + { include: '@footnotesReference' }, + + //hyperlink-references + [/(@simpleRefName)(_{1,2})/, ['string.link', '']], + + //embedded-uris-and-aliases + [/(`)([^<`]+\s+)(<)(.*)(>)(`)(_)/, ['', 'string.link', '', 'string.link', '', '', '']], + + //emphasis + [/\*\*([^\\*]|\*(?!\*))+\*\*/, 'strong'], + [/\*[^*]+\*/, 'emphasis'], + + //inline-literals + [/(``)((?:[^`]|\`(?!`))+)(``)/, ['', 'keyword', '']], + [/(__\s+)(.+)/, ['', 'keyword']], + + //interpreted-text + [/(:)((?:@simpleRefNameWithoutBq)?)(:`)([^`]+)(`)/, ['', 'keyword', '', '', '']], + [/(`)([^`]+)(`:)((?:@simpleRefNameWithoutBq)?)(:)/, ['', '', '', 'keyword', '']], + [/(`)([^`]+)(`)/, ''], + + //inline-internal-targets + [/(_`)(@phrase)(`)/, ['', 'string.link', '']], + ], + citations: [ + [/^(\.\.\s+\[)((?:@citationName))(\]\s+)(.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '', '']], + ], + citationsReference: [ + [/(\[)(@citationName)(\]_)/, ['', 'string.link', '']], + ], + footnotes: [ + [/^(\.\.\s+\[)((?:[0-9]+))(\]\s+.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '']], + [/^(\.\.\s+\[)((?:#@simpleRefName?))(\]\s+)(.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '', '']], + [/^(\.\.\s+\[)((?:\*))(\]\s+)(.*)/, [{ token: '', next: '@subsequentLines' }, 'string.link', '', '']], + ], + footnotesReference: [ + [/(\[)([0-9]+)(\])(_)/, ['', 'string.link', '', '']], + [/(\[)(#@simpleRefName?)(\])(_)/, ['', 'string.link', '', '']], + [/(\[)(\*)(\])(_)/, ['', 'string.link', '', '']] + ], + blankLineOfLiteralBlocks: [ + [/^$/, '', '@subsequentLinesOfLiteralBlocks'], + [/^.*$/, '', '@pop'], + ], + subsequentLinesOfLiteralBlocks: [ + [/(@blockLiteralStart+)(.*)/, ['keyword', '']], + [/^(?!blockLiteralStart)/, '', '@popall'] + ], + subsequentLines: [ + [/^[\s]+.*/, ''], + [/^(?!\s)/, '', '@pop'], + ], + hyperlinks: [ + [/^[\s]+.*/, 'string.link'], + [/^(?!\s)/, '', '@pop'], + ], + comments: [ + [/^[\s]+.*/, 'comment'], + [/^(?!\s)/, '', '@pop'], + ], + tables: [ + [/\+-[+-]+/, 'keyword'], + [/\+=[+=]+/, 'keyword'], + ], + } +}; + From 24f923b7c0c091f481d203075eb72dc091b3d5db Mon Sep 17 00:00:00 2001 From: Alexandru Dima Date: Mon, 6 Jan 2020 13:53:41 +0100 Subject: [PATCH 2/2] Follow naming patterns & sort imports --- src/monaco.contribution.ts | 2 +- src/{rst => restructuredtext}/restructuredtext.contribution.ts | 0 src/{rst => restructuredtext}/restructuredtext.test.ts | 0 src/{rst => restructuredtext}/restructuredtext.ts | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename src/{rst => restructuredtext}/restructuredtext.contribution.ts (100%) rename src/{rst => restructuredtext}/restructuredtext.test.ts (100%) rename src/{rst => restructuredtext}/restructuredtext.ts (100%) diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index 3e294e71..95c20044 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -46,6 +46,7 @@ import './r/r.contribution'; import './razor/razor.contribution'; import './redis/redis.contribution'; import './redshift/redshift.contribution'; +import './restructuredtext/restructuredtext.contribution'; import './ruby/ruby.contribution'; import './rust/rust.contribution'; import './sb/sb.contribution'; @@ -63,4 +64,3 @@ import './typescript/typescript.contribution'; import './vb/vb.contribution'; import './xml/xml.contribution'; import './yaml/yaml.contribution'; -import './rst/restructuredtext.contribution'; diff --git a/src/rst/restructuredtext.contribution.ts b/src/restructuredtext/restructuredtext.contribution.ts similarity index 100% rename from src/rst/restructuredtext.contribution.ts rename to src/restructuredtext/restructuredtext.contribution.ts diff --git a/src/rst/restructuredtext.test.ts b/src/restructuredtext/restructuredtext.test.ts similarity index 100% rename from src/rst/restructuredtext.test.ts rename to src/restructuredtext/restructuredtext.test.ts diff --git a/src/rst/restructuredtext.ts b/src/restructuredtext/restructuredtext.ts similarity index 100% rename from src/rst/restructuredtext.ts rename to src/restructuredtext/restructuredtext.ts