Merge branch 'main' into mdx

This commit is contained in:
Remco Haszing 2023-06-13 14:20:59 +02:00 committed by GitHub
commit 62e11b4b49
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 853 additions and 253 deletions

View file

@ -39,6 +39,65 @@ testTokenization('javascript', [
}
],
// identifiers
[
{
line: 'foo;',
tokens: [
{ startIndex: 0, type: 'identifier.js' },
{ startIndex: 3, type: 'delimiter.js' }
]
}
],
[
{
line: 'foo() { return 1; }',
tokens: [
{ startIndex: 0, type: 'identifier.js' },
{ startIndex: 3, type: 'delimiter.parenthesis.js' },
{ startIndex: 5, type: '' },
{ startIndex: 6, type: 'delimiter.bracket.js' },
{ startIndex: 7, type: '' },
{ startIndex: 8, type: 'keyword.js' },
{ startIndex: 14, type: '' },
{ startIndex: 15, type: 'number.js' },
{ startIndex: 16, type: 'delimiter.js' },
{ startIndex: 17, type: '' },
{ startIndex: 18, type: 'delimiter.bracket.js' }
]
}
],
[
{
line: '#foo;',
tokens: [
{ startIndex: 0, type: 'identifier.js' },
{ startIndex: 4, type: 'delimiter.js' }
]
}
],
[
{
line: '#foo() { return 1; }',
tokens: [
{ startIndex: 0, type: 'identifier.js' },
{ startIndex: 4, type: 'delimiter.parenthesis.js' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'delimiter.bracket.js' },
{ startIndex: 8, type: '' },
{ startIndex: 9, type: 'keyword.js' },
{ startIndex: 15, type: '' },
{ startIndex: 16, type: 'number.js' },
{ startIndex: 17, type: 'delimiter.js' },
{ startIndex: 18, type: '' },
{ startIndex: 19, type: 'delimiter.bracket.js' }
]
}
],
// Comments - single line
[
{

View file

@ -43,6 +43,7 @@ export const language = <languages.IMonarchLanguage>{
'null',
'return',
'set',
'static',
'super',
'switch',
'symbol',

View file

@ -39,6 +39,65 @@ testTokenization('typescript', [
}
],
// identifiers
[
{
line: 'foo;',
tokens: [
{ startIndex: 0, type: 'identifier.ts' },
{ startIndex: 3, type: 'delimiter.ts' }
]
}
],
[
{
line: 'foo() { return 1; }',
tokens: [
{ startIndex: 0, type: 'identifier.ts' },
{ startIndex: 3, type: 'delimiter.parenthesis.ts' },
{ startIndex: 5, type: '' },
{ startIndex: 6, type: 'delimiter.bracket.ts' },
{ startIndex: 7, type: '' },
{ startIndex: 8, type: 'keyword.ts' },
{ startIndex: 14, type: '' },
{ startIndex: 15, type: 'number.ts' },
{ startIndex: 16, type: 'delimiter.ts' },
{ startIndex: 17, type: '' },
{ startIndex: 18, type: 'delimiter.bracket.ts' }
]
}
],
[
{
line: '#foo;',
tokens: [
{ startIndex: 0, type: 'identifier.ts' },
{ startIndex: 4, type: 'delimiter.ts' }
]
}
],
[
{
line: '#foo() { return 1; }',
tokens: [
{ startIndex: 0, type: 'identifier.ts' },
{ startIndex: 4, type: 'delimiter.parenthesis.ts' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'delimiter.bracket.ts' },
{ startIndex: 8, type: '' },
{ startIndex: 9, type: 'keyword.ts' },
{ startIndex: 15, type: '' },
{ startIndex: 16, type: 'number.ts' },
{ startIndex: 17, type: 'delimiter.ts' },
{ startIndex: 18, type: '' },
{ startIndex: 19, type: 'delimiter.bracket.ts' }
]
}
],
// Comments - single line
[
{

View file

@ -227,7 +227,7 @@ export const language = {
common: [
// identifiers and keywords
[
/[a-z_$][\w$]*/,
/#?[a-z_$][\w$]*/,
{
cases: {
'@keywords': 'keyword',

View file

@ -377,7 +377,7 @@ export const language = <languages.IMonarchLanguage>{
predeclared_intrinsics,
operators,
symbols: /[!%&*+\-\.\/:;<=>^|_~]+/,
symbols: /[!%&*+\-\.\/:;<=>^|_~,]+/,
tokenizer: {
root: [
@ -402,8 +402,6 @@ export const language = <languages.IMonarchLanguage>{
{ include: '@commentOrSpace' },
{ include: '@numbers' },
[/;:\./, 'delimiter'],
[/,/, 'delimiter'], // Hack: Should be in previous rule
[/[{}()\[\]]/, '@brackets'],
['@', 'annotation', '@attribute'],
[