Merge pull request #159 from aaaaaa2493/main

Support Java 12-17 syntax
This commit is contained in:
Alexandru Dima 2021-10-08 07:38:50 +02:00 committed by GitHub
commit 972af31cdb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 164 additions and 1 deletions

View file

@ -728,5 +728,151 @@ testTokenization('java', [
{ startIndex: 29, type: 'keyword.private.java' } { startIndex: 29, type: 'keyword.private.java' }
] ]
} }
],
[
{
line: 'String s = """Multiline string""";',
tokens: [
{ startIndex: 0, type: 'identifier.java' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'identifier.java' },
{ startIndex: 8, type: '' },
{ startIndex: 9, type: 'delimiter.java' },
{ startIndex: 10, type: '' },
{ startIndex: 11, type: 'string.java' },
{ startIndex: 33, type: 'delimiter.java' }
]
}
],
[
{
line: 'String s = """',
tokens: [
{ startIndex: 0, type: 'identifier.java' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'identifier.java' },
{ startIndex: 8, type: '' },
{ startIndex: 9, type: 'delimiter.java' },
{ startIndex: 10, type: '' },
{ startIndex: 11, type: 'string.java' }
]
},
{
line: ' <html></html>',
tokens: [{ startIndex: 0, type: 'string.java' }]
},
{
line: ' """;',
tokens: [
{ startIndex: 0, type: 'string.java' },
{ startIndex: 8, type: 'delimiter.java' }
]
}
],
[
{
line: 'String s = """',
tokens: [
{ startIndex: 0, type: 'identifier.java' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'identifier.java' },
{ startIndex: 8, type: '' },
{ startIndex: 9, type: 'delimiter.java' },
{ startIndex: 10, type: '' },
{ startIndex: 11, type: 'string.java' }
]
},
{
line: ' \\"""Triple quoted string inside text block\\"""',
tokens: [
{ startIndex: 0, type: 'string.java' },
{ startIndex: 5, type: 'string.escape.java' },
{ startIndex: 7, type: 'string.java' },
{ startIndex: 47, type: 'string.escape.java' },
{ startIndex: 49, type: 'string.java' }
]
},
{
line: ' """;',
tokens: [
{ startIndex: 0, type: 'string.java' },
{ startIndex: 8, type: 'delimiter.java' }
]
}
],
[
{
line: 'yield "123"',
tokens: [
{ startIndex: 0, type: 'keyword.yield.java' },
{ startIndex: 5, type: '' },
{ startIndex: 6, type: 'string.java' }
]
}
],
[
{
line: 'public sealed class Shape permits Circle, Square { }',
tokens: [
{ startIndex: 0, type: 'keyword.public.java' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'keyword.sealed.java' },
{ startIndex: 13, type: '' },
{ startIndex: 14, type: 'keyword.class.java' },
{ startIndex: 19, type: '' },
{ startIndex: 20, type: 'identifier.java' },
{ startIndex: 25, type: '' },
{ startIndex: 26, type: 'keyword.permits.java' },
{ startIndex: 33, type: '' },
{ startIndex: 34, type: 'identifier.java' },
{ startIndex: 40, type: 'delimiter.java' },
{ startIndex: 41, type: '' },
{ startIndex: 42, type: 'identifier.java' },
{ startIndex: 48, type: '' },
{ startIndex: 49, type: 'delimiter.curly.java' },
{ startIndex: 50, type: '' },
{ startIndex: 51, type: 'delimiter.curly.java' }
]
}
],
[
{
line: 'public non-sealed class Shape',
tokens: [
{ startIndex: 0, type: 'keyword.public.java' },
{ startIndex: 6, type: '' },
{ startIndex: 7, type: 'keyword.non-sealed.java' },
{ startIndex: 17, type: '' },
{ startIndex: 18, type: 'keyword.class.java' },
{ startIndex: 23, type: '' },
{ startIndex: 24, type: 'identifier.java' }
]
}
],
[
{
line: 'int x = y-z;', // Make sure "y-z" is not identifier
tokens: [
{ startIndex: 0, type: 'keyword.int.java' },
{ startIndex: 3, type: '' },
{ startIndex: 4, type: 'identifier.java' },
{ startIndex: 5, type: '' },
{ startIndex: 6, type: 'delimiter.java' },
{ startIndex: 7, type: '' },
{ startIndex: 8, type: 'identifier.java' },
{ startIndex: 9, type: 'delimiter.java' },
{ startIndex: 10, type: 'identifier.java' },
{ startIndex: 11, type: 'delimiter.java' }
]
}
] ]
]); ]);

View file

@ -96,7 +96,12 @@ export const language = <languages.IMonarchLanguage>{
'super', 'super',
'while', 'while',
'true', 'true',
'false' 'false',
'yield',
'record',
'sealed',
'non-sealed',
'permits'
], ],
operators: [ operators: [
@ -150,6 +155,9 @@ export const language = <languages.IMonarchLanguage>{
// The main tokenizer for our languages // The main tokenizer for our languages
tokenizer: { tokenizer: {
root: [ root: [
// Special keyword with a dash
['non-sealed', 'keyword.non-sealed'],
// identifiers and keywords // identifiers and keywords
[ [
/[a-zA-Z_$][\w$]*/, /[a-zA-Z_$][\w$]*/,
@ -194,6 +202,7 @@ export const language = <languages.IMonarchLanguage>{
// strings // strings
[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string [/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
[/"""/, 'string', '@multistring'],
[/"/, 'string', '@string'], [/"/, 'string', '@string'],
// characters // characters
@ -230,6 +239,14 @@ export const language = <languages.IMonarchLanguage>{
[/@escapes/, 'string.escape'], [/@escapes/, 'string.escape'],
[/\\./, 'string.escape.invalid'], [/\\./, 'string.escape.invalid'],
[/"/, 'string', '@pop'] [/"/, 'string', '@pop']
],
multistring: [
[/[^\\"]+/, 'string'],
[/@escapes/, 'string.escape'],
[/\\./, 'string.escape.invalid'],
[/"""/, 'string', '@pop'],
[/./, 'string']
] ]
} }
}; };