Merge pull request #50 from goober99/hacktoberfest

Added support for Perl quote-like operators to fix #1101

src/perl/perl.test.ts

@@ -310,14 +310,23 @@ testTokenization('perl', [
 	[
 		{
 			line: 'm/abc/',
-			tokens: [{ startIndex: 0, type: 'regexp.perl' }],
+			tokens: [
+				{ startIndex: 0, type: 'regexp.delim.perl' },
+				{ startIndex: 2, type: 'regexp.perl' },
+				{ startIndex: 5, type: 'regexp.delim.perl' },
+			],
 		},
 	],
 
 	[
 		{
 			line: 'm/[abc]+/e',
-			tokens: [{ startIndex: 0, type: 'regexp.perl' }],
+			tokens: [
+				{ startIndex: 0, type: 'regexp.delim.perl' },
+				{ startIndex: 2, type: 'regexp.perl' },
+				{ startIndex: 8, type: 'regexp.delim.perl' },
+        { startIndex: 9, type: 'regexp.modifier.perl' },
+			],
 		},
 	],
 
@@ -429,4 +438,39 @@ testTokenization('perl', [
 			],
 		},
 	],
+
+  // Quoted constructs
+	[
+		{
+			line: "m!can't!",
+			tokens: [
+				{ startIndex: 0, type: 'regexp.delim.perl' },
+				{ startIndex: 2, type: 'regexp.perl' },
+				{ startIndex: 7, type: 'regexp.delim.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: 'q XfooX',
+			tokens: [
+				{ startIndex: 0, type: 'string.delim.perl' },
+				{ startIndex: 3, type: 'string.perl' },
+				{ startIndex: 6, type: 'string.delim.perl' },
+			],
+		},
+	],
+
+	[
+		{
+			line: 'qq(test $foo)',
+			tokens: [
+				{ startIndex: 0, type: 'string.delim.perl' },
+				{ startIndex: 3, type: 'string.perl' },
+				{ startIndex: 8, type: 'variable.perl' },
+        { startIndex: 12, type: 'string.delim.perl' },
+			],
+		},
+	],
 ]);

src/perl/perl.ts

@@ -48,15 +48,11 @@ export const language = <ILanguage>{
 		'__DATA__',
 		'else',
 		'lock',
-		'qw',
 		'__END__',
 		'elsif',
 		'lt',
-		'qx',
 		'__FILE__',
 		'eq',
-		'm',
-		's',
 		'__LINE__',
 		'exp',
 		'ne',
@@ -64,7 +60,6 @@ export const language = <ILanguage>{
 		'__PACKAGE__',
 		'for',
 		'no',
-		'tr',
 		'and',
 		'foreach',
 		'or',
@@ -75,16 +70,12 @@ export const language = <ILanguage>{
 		'until',
 		'continue',
 		'gt',
-		'q',
 		'while',
 		'CORE',
 		'if',
-		'qq',
 		'xor',
 		'do',
 		'le',
-		'qr',
-		'y',
 
 		'__DIE__',
 		'__WARN__',
@@ -468,6 +459,7 @@ export const language = <ILanguage>{
 
 	// operators
 	symbols: /[:+\-\^*$&%@=<>!?|\/~\.]/,
+	quoteLikeOps: ['qr', 'm', 's', 'q', 'qq', 'qx', 'qw', 'tr', 'y'],
 
 	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
 
@@ -477,12 +469,13 @@ export const language = <ILanguage>{
 			{ include: '@whitespace' },
 
 			[
-				/[a-zA-Z\-_][\w\-_]+/,
+				/[a-zA-Z\-_][\w\-_]*/,
 				{
 					cases: {
 						'@keywords': 'keyword',
 						'@builtinFunctions': 'type.identifier',
 						'@builtinFileHandlers': 'variable.predefined',
+						'@quoteLikeOps': { token: '@rematch', next: 'quotedConstructs' },
 						'@default': '',
 					},
 				},
@@ -512,7 +505,7 @@ export const language = <ILanguage>{
 
 			// RegExp
 			[
-				/[goseximacplud]*[\/](?:(?:\[(?:\\]|[^\]])+\])|(?:\\\/|[^\]\/]))*[\/]\w*\s*(?=[).,;]|$)/,
+				/[\/](?:(?:\[(?:\\]|[^\]])+\])|(?:\\\/|[^\]\/]))*[\/]\w*\s*(?=[).,;]|$)/,
 				'regexp',
 			],
 
@@ -555,6 +548,79 @@ export const language = <ILanguage>{
 			[/./, 'string'],
 		],
 
+		// Quoted constructs
+		// Percent strings in Ruby are similar to quote-like operators in Perl.
+		// This is adapted from pstrings in ../ruby/ruby.ts.
+		quotedConstructs: [
+			[/(q|qw|tr|y)\s*\(/, { token: 'string.delim', switchTo: '@qstring.(.)' }],
+			[/(q|qw|tr|y)\s*\[/, { token: 'string.delim', switchTo: '@qstring.[.]' }],
+			[/(q|qw|tr|y)\s*\{/, { token: 'string.delim', switchTo: '@qstring.{.}' }],
+			[/(q|qw|tr|y)\s*</, { token: 'string.delim', switchTo: '@qstring.<.>' }],
+			[/(q|qw|tr|y)#/, { token: 'string.delim', switchTo: '@qstring.#.#' }],
+			[/(q|qw|tr|y)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }],
+			[/(q|qw|tr|y)\s+(\w)/, { token: 'string.delim', switchTo: '@qstring.$2.$2' }],
+
+			[/(qr|m|s)\s*\(/, { token: 'regexp.delim', switchTo: '@qregexp.(.)' }],
+			[/(qr|m|s)\s*\[/, { token: 'regexp.delim', switchTo: '@qregexp.[.]' }],
+			[/(qr|m|s)\s*\{/, { token: 'regexp.delim', switchTo: '@qregexp.{.}' }],
+			[/(qr|m|s)\s*</, { token: 'regexp.delim', switchTo: '@qregexp.<.>' }],
+			[/(qr|m|s)#/, { token: 'regexp.delim', switchTo: '@qregexp.#.#' }],
+			[/(qr|m|s)\s*([^A-Za-z0-9_#\s])/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }],
+			[/(qr|m|s)\s+(\w)/, { token: 'regexp.delim', switchTo: '@qregexp.$2.$2' }],
+
+			[/(qq|qx)\s*\(/, { token: 'string.delim', switchTo: '@qqstring.(.)' }],
+			[/(qq|qx)\s*\[/, { token: 'string.delim', switchTo: '@qqstring.[.]' }],
+			[/(qq|qx)\s*\{/, { token: 'string.delim', switchTo: '@qqstring.{.}' }],
+			[/(qq|qx)\s*</, { token: 'string.delim', switchTo: '@qqstring.<.>' }],
+			[/(qq|qx)#/, { token: 'string.delim', switchTo: '@qqstring.#.#' }],
+			[/(qq|qx)\s*([^A-Za-z0-9#\s])/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }],
+			[/(qq|qx)\s+(\w)/, { token: 'string.delim', switchTo: '@qqstring.$2.$2' }],
+		],
+
+		// Non-expanded quoted string
+		// qstring<open>.<close>
+		//  open = open delimiter
+		//  close = close delimiter
+		qstring: [
+			[/\\./, 'string.escape'],
+			[/./, {
+				cases: {
+					'$#==$S3': { token: 'string.delim', next: '@pop' },
+					'$#==$S2': { token: 'string.delim', next: '@push' }, // nested delimiters
+					'@default': 'string'
+				}
+			}],
+		],
+
+    // Quoted regexp
+		// qregexp.<open>.<close>
+		//  open = open delimiter
+		//  close = close delimiter
+		qregexp: [
+			{ include: '@variables' },
+			[/\\./, 'regexp.escape'],
+			[/./, {
+				cases: {
+					'$#==$S3': { token: 'regexp.delim', next: '@regexpModifiers' },
+					'$#==$S2': { token: 'regexp.delim', next: '@push' }, // nested delimiters
+					'@default': 'regexp'
+				}
+			}],
+		],
+
+    regexpModifiers: [
+			[/[msixpodualngcer]+/, { token: 'regexp.modifier', next: '@popall' }],
+		],
+
+		// Expanded quoted string
+		// qqstring.<open>.<close>
+		//  open = open delimiter
+		//  close = close delimiter
+		qqstring: [
+			{ include: '@variables' },
+			{ include: '@qstring' },
+		],
+
 		heredoc: [
 			[
 				/<<\s*['"`]?([\w\-]+)['"`]?/,