Merge pull request #3453 from jonatanklosko/jk-elixir-improvements

Update Elixir tokenizer

src/basic-languages/elixir/elixir.test.ts

@@ -383,5 +383,32 @@ testTokenization('elixir', [
 				{ startIndex: 15, type: 'delimiter.square.elixir' }
 			]
 		}
+	],
+	// Bitstrings
+	[
+		{
+			line: '<<height::32-integer, width::32-integer, data::binary>>',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.angle.special.elixir' },
+				{ startIndex: 2, type: 'identifier.elixir' },
+				{ startIndex: 8, type: 'operator.elixir' },
+				{ startIndex: 10, type: 'number.elixir' },
+				{ startIndex: 12, type: 'operator.elixir' },
+				{ startIndex: 13, type: 'identifier.elixir' },
+				{ startIndex: 20, type: 'punctuation.elixir' },
+				{ startIndex: 21, type: 'white.elixir' },
+				{ startIndex: 22, type: 'identifier.elixir' },
+				{ startIndex: 27, type: 'operator.elixir' },
+				{ startIndex: 29, type: 'number.elixir' },
+				{ startIndex: 31, type: 'operator.elixir' },
+				{ startIndex: 32, type: 'identifier.elixir' },
+				{ startIndex: 39, type: 'punctuation.elixir' },
+				{ startIndex: 40, type: 'white.elixir' },
+				{ startIndex: 41, type: 'identifier.elixir' },
+				{ startIndex: 45, type: 'operator.elixir' },
+				{ startIndex: 47, type: 'identifier.elixir' },
+				{ startIndex: 53, type: 'delimiter.angle.special.elixir' }
+			]
+		}
 	]
 ]);

src/basic-languages/elixir/elixir.ts

@@ -167,7 +167,7 @@ export const language = <languages.IMonarchLanguage>{
 		// Keyword list shorthand
 
 		keywordsShorthand: [
-			[/(@atomName)(:)/, ['constant', 'constant.punctuation']],
+			[/(@atomName)(:)(\s+)/, ['constant', 'constant.punctuation', 'white']],
 			// Use positive look-ahead to ensure the string is followed by :
 			// and should be considered a keyword.
 			[
@@ -532,6 +532,13 @@ export const language = <languages.IMonarchLanguage>{
 					next: '@doubleQuotedHeredocDocstring'
 				}
 			],
+			[
+				/\@(module|type)?doc (~[sS])?'''/,
+				{
+					token: 'comment.block.documentation',
+					next: '@singleQuotedHeredocDocstring'
+				}
+			],
 			[
 				/\@(module|type)?doc (~[sS])?"/,
 				{
@@ -539,6 +546,13 @@ export const language = <languages.IMonarchLanguage>{
 					next: '@doubleQuotedStringDocstring'
 				}
 			],
+			[
+				/\@(module|type)?doc (~[sS])?'/,
+				{
+					token: 'comment.block.documentation',
+					next: '@singleQuotedStringDocstring'
+				}
+			],
 			[/\@(module|type)?doc false/, 'comment.block.documentation'],
 			// Module attributes
 			[/\@(@variableName)/, 'variable']
@@ -549,11 +563,21 @@ export const language = <languages.IMonarchLanguage>{
 			{ include: '@docstringContent' }
 		],
 
+		singleQuotedHeredocDocstring: [
+			[/'''/, { token: 'comment.block.documentation', next: '@pop' }],
+			{ include: '@docstringContent' }
+		],
+
 		doubleQuotedStringDocstring: [
 			[/"/, { token: 'comment.block.documentation', next: '@pop' }],
 			{ include: '@docstringContent' }
 		],
 
+		singleQuotedStringDocstring: [
+			[/'/, { token: 'comment.block.documentation', next: '@pop' }],
+			{ include: '@docstringContent' }
+		],
+
 		// Operators, punctuation, brackets
 
 		symbols: [