Fix yaml string tokenization

This commit is contained in:
Grzegorz Wcisło 2018-10-03 17:59:21 +02:00
parent 66b5497f3c
commit 4b0d5c54d2
2 changed files with 38 additions and 15 deletions

View file

@ -241,6 +241,35 @@ testTokenization('yaml', [
},] },]
}], }],
// Flow Scalars
[{
line: '\'this is a single quote string\'',
tokens: [{
startIndex: 0,
type: 'string.yaml'
}]
}],
[{
line: "\"this is a double \\quote string\\n\"",
tokens: [{
startIndex: 0,
type: 'string.yaml'
}, {
startIndex: 18,
type: 'string.escape.invalid.yaml'
}, {
startIndex: 20,
type: 'string.yaml'
}, {
startIndex: 31,
type: 'string.escape.yaml'
}, {
startIndex: 33,
type: 'string.yaml'
}]
}],
// Flow Sequence - Data types // Flow Sequence - Data types
[{ [{
line: '[string,"double",\'single\',1,1.1,2002-04-28]', line: '[string,"double",\'single\',1,1.1,2002-04-28]',

View file

@ -156,19 +156,6 @@ export const language = <ILanguage>{
}] }]
], ],
// Flow Scalars (quoted strings)
string: [
[/[^\\"']+/, 'string'],
[/@escapes/, 'string.escape'],
[/\\./, 'string.escape.invalid'],
[/["']/, {
cases: {
'$#==$S2': { token: 'string', next: '@pop' },
'@default': 'string'
}
}]
],
// First line of a Block Style // First line of a Block Style
multiString: [ multiString: [
[/^( +).+$/, 'string', '@multiStringContinued.$1'] [/^( +).+$/, 'string', '@multiStringContinued.$1']
@ -202,8 +189,15 @@ export const language = <ILanguage>{
// Start Flow Scalars (quoted strings) // Start Flow Scalars (quoted strings)
flowScalars: [ flowScalars: [
[/"/, 'string', '@string."'], [/'[^']*'/, 'string'],
[/'/, 'string', '@string.\''] [/"/, 'string', '@doubleQuotedString']
],
doubleQuotedString: [
[/[^\\"]+/, 'string'],
[/@escapes/, 'string.escape'],
[/\\./, 'string.escape.invalid'],
[/"/, 'string', '@pop']
], ],
// Start Block Scalar // Start Block Scalar