Rust: Add support for highlighting raw strings and fix chars with escapes.

Rust's raw strings were previously not supported, and chars did not correctly handle escapes.

Fixes #2552
Fixes #2481

src/rust/rust.test.ts

@@ -23,6 +23,57 @@ testTokenization('rust', [
 			]
 		}
 	],
+	// Raw String
+	[
+		{
+			line: 'r"This is a raw string" ',
+			tokens: [
+				{ startIndex: 0, type: 'string.quote.rust' },
+				{ startIndex: 2, type: 'string.rust' },
+				{ startIndex: 22, type: 'string.quote.rust' },
+				{ startIndex: 23, type: 'white.rust' }
+			]
+		}
+	],
+	[
+		{
+			line: 'r#"This is a raw string"# ',
+			tokens: [
+				{ startIndex: 0, type: 'string.quote.rust' },
+				{ startIndex: 3, type: 'string.rust' },
+				{ startIndex: 23, type: 'string.quote.rust' },
+				{ startIndex: 25, type: 'white.rust' }
+			]
+		}
+	],
+	[
+		{
+			line: 'r##"This is a# raw string"## ',
+			tokens: [
+				{ startIndex: 0, type: 'string.quote.rust' },
+				{ startIndex: 4, type: 'string.rust' },
+				{ startIndex: 25, type: 'string.quote.rust' },
+				{ startIndex: 28, type: 'white.rust' }
+			]
+		}
+	],
+	[
+		{
+			line: 'r###"This is multi-line',
+			tokens: [
+				{ startIndex: 0, type: 'string.quote.rust' },
+				{ startIndex: 5, type: 'string.rust' }
+			]
+		},
+		{
+			line: 'raw "##string"### ',
+			tokens: [
+				{ startIndex: 0, type: 'string.rust' },
+				{ startIndex: 13, type: 'string.quote.rust' },
+				{ startIndex: 17, type: 'white.rust' }
+			]
+		}
+	],
 	// Byte literal
 	[
 		{
@@ -40,17 +91,16 @@ testTokenization('rust', [
 	],
 	[
 		{
-			line: "'\"'",
+			line: "'\\\"'",
 			tokens: [{ startIndex: 0, type: 'string.byteliteral.rust' }]
 		}
 	],
 	[
 		{
-			line: "'\0'",
+			line: "'\\0'",
 			tokens: [{ startIndex: 0, type: 'string.byteliteral.rust' }]
 		}
 	],
-
 	// Comment
 	[
 		{

src/rust/rust.ts

@@ -269,6 +269,8 @@ export const language = <languages.IMonarchLanguage>{
 
 	tokenizer: {
 		root: [
+			// Raw string literals
+			[/r(#*)"/, { token: 'string.quote', bracket: '@open', next: '@stringraw.$1' }],
 			[
 				/[a-zA-Z][a-zA-Z0-9_]*!?|_[a-zA-Z0-9_]+/,
 				{
@@ -287,7 +289,7 @@ export const language = <languages.IMonarchLanguage>{
 			// Lifetime annotations
 			[/'[a-zA-Z_][a-zA-Z0-9_]*(?=[^\'])/, 'identifier'],
 			// Byte literal
-			[/'\S'/, 'string.byteliteral'],
+			[/'(\S|@escapes)'/, 'string.byteliteral'],
 			// Strings
 			[/"/, { token: 'string.quote', bracket: '@open', next: '@string' }],
 			{ include: '@numbers' },
@@ -326,6 +328,21 @@ export const language = <languages.IMonarchLanguage>{
 			[/\\./, 'string.escape.invalid'],
 			[/"/, { token: 'string.quote', bracket: '@close', next: '@pop' }]
 		],
+
+		stringraw: [
+			[/[^"#]+/, { token: 'string' }],
+			[
+				/"(#*)/,
+				{
+					cases: {
+						'$1==$S2': { token: 'string.quote', bracket: '@close', next: '@pop' },
+						'@default': { token: 'string' }
+					}
+				}
+			],
+			[/["#]/, { token: 'string' }]
+		],
+
 		numbers: [
 			//Octal
 			[/(0o[0-7_]+)(@intSuffixes)?/, { token: 'number' }],