Initial release

.gitignore

@@ -0,0 +1,3 @@
+/node_modules/
+/out/
+/release/

.npmignore

@@ -0,0 +1,7 @@
+/.vscode/
+/out/
+/src/
+/test/
+/gulpfile.js
+/tsconfig.json
+/.npmignore

.vscode/settings.json

@@ -0,0 +1,9 @@
+// Place your settings in this file to overwrite default and user settings.
+{
+	"files.trimTrailingWhitespace": true,
+	"search.exclude": {
+		"**/node_modules": true,
+		"**/release": true,
+		"**/out": true
+	}
+}

LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,37 @@
-# monaco-languages
-Basic colorization support for many Monaco Editor languages.
+# Monaco Languages
+
+Colorization and configuration supports for multiple languages for the Monaco Editor:
+
+![monaco-languages](https://cloud.githubusercontent.com/assets/5047891/15938606/1fd4bac6-2e74-11e6-8839-d455da8bc8a7.gif)
+
+* bat
+* coffee script
+* cpp
+* csharp
+* fsharp
+* go
+* ini
+* jade
+* lua
+* objective-c
+* powershell
+* python
+* r
+* ruby
+* sql
+* swift
+* vb
+* xml
+
+## Installing
+
+This npm module is bundled and distributed in the [monaco-editor](https://www.npmjs.com/package/monaco-editor) npm module.
+
+## Dev:
+
+* compile with `npm run watch`
+* test with `npm run test`
+* bundle with `npm run prepublish`
+
+## License
+[MIT](https://github.com/Microsoft/monaco-typescript/blob/master/LICENSE.md)

ThirdPartyNotices.txt

@@ -0,0 +1,34 @@
+THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
+Do Not Translate or Localize
+
+This project incorporates components from the projects listed below. The original copyright notices and the licenses
+under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted
+herein, whether by implication, estoppel or otherwise.
+
+
+
+
+%% vscode-swift version 0.0.1 (https://github.com/owensd/vscode-swift)
+=========================================
+The MIT License (MIT)
+
+Copyright (c) 2015 David Owens II
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+=========================================
+END OF vscode-swift NOTICES AND INFORMATION

gulpfile.js

@@ -0,0 +1,148 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+var gulp = require('gulp');
+var tsb = require('gulp-tsb');
+var assign = require('object-assign');
+var fs = require('fs');
+var path = require('path');
+var merge = require('merge-stream');
+var rjs = require('gulp-requirejs');
+var uglify = require('gulp-uglify');
+var rimraf = require('rimraf');
+var es = require('event-stream');
+
+gulp.task('clean-release', function(cb) { rimraf('release', { maxBusyTries: 1 }, cb); });
+gulp.task('release', ['clean-release','compile'], function() {
+
+	var sha1 = getGitVersion(__dirname);
+	var semver = require('./package.json').version;
+	var headerVersion = semver + '(' + sha1 + ')';
+
+	var BUNDLED_FILE_HEADER = [
+		'/*!-----------------------------------------------------------------------------',
+		' * Copyright (c) Microsoft Corporation. All rights reserved.',
+		' * monaco-languages version: ' + headerVersion,
+		' * Released under the MIT license',
+		' * https://github.com/Microsoft/monaco-languages/blob/master/LICENSE.md',
+		' *-----------------------------------------------------------------------------*/',
+		''
+	].join('\n');
+
+	function bundleOne(moduleId, exclude) {
+		return rjs({
+			baseUrl: '/out/',
+			name: 'vs/basic-languages/' + moduleId,
+			out: moduleId + '.js',
+			exclude: exclude,
+			paths: {
+				'vs/basic-languages': __dirname + '/out'
+			}
+		})
+	}
+
+	return merge(
+			bundleOne('src/monaco.contribution'),
+			bundleOne('src/bat'),
+			bundleOne('src/coffee'),
+			bundleOne('src/cpp'),
+			bundleOne('src/csharp'),
+			bundleOne('src/dockerfile'),
+			bundleOne('src/fsharp'),
+			bundleOne('src/go'),
+			bundleOne('src/ini'),
+			bundleOne('src/jade'),
+			bundleOne('src/java'),
+			bundleOne('src/lua'),
+			bundleOne('src/objective-c'),
+			bundleOne('src/powershell'),
+			bundleOne('src/python'),
+			bundleOne('src/r'),
+			bundleOne('src/ruby'),
+			bundleOne('src/sql'),
+			bundleOne('src/swift'),
+			bundleOne('src/vb'),
+			bundleOne('src/xml')
+		)
+		.pipe(uglify({
+			preserveComments: 'some'
+		}))
+		.pipe(es.through(function(data) {
+			data.contents = new Buffer(
+				BUNDLED_FILE_HEADER
+				+ data.contents.toString()
+			);
+			this.emit('data', data);
+		}))
+		.pipe(gulp.dest('./release/'));
+});
+
+
+var compilation = tsb.create(assign({ verbose: true }, require('./tsconfig.json').compilerOptions));
+
+var tsSources = require('./tsconfig.json').filesGlob;
+
+function compileTask() {
+	return merge(
+		gulp.src(tsSources, { base: '.' }).pipe(compilation())
+	)
+	.pipe(gulp.dest('out'));
+}
+gulp.task('clean-out', function(cb) { rimraf('out', { maxBusyTries: 1 }, cb); });
+gulp.task('compile', ['clean-out'], compileTask);
+gulp.task('compile-without-clean', compileTask);
+gulp.task('watch', ['compile'], function() {
+	gulp.watch(tsSources, ['compile-without-clean']);
+});
+
+function getGitVersion(repo) {
+	var git = path.join(repo, '.git');
+	var headPath = path.join(git, 'HEAD');
+	var head;
+
+	try {
+		head = fs.readFileSync(headPath, 'utf8').trim();
+	} catch (e) {
+		return void 0;
+	}
+
+	if (/^[0-9a-f]{40}$/i.test(head)) {
+		return head;
+	}
+
+	var refMatch = /^ref: (.*)$/.exec(head);
+
+	if (!refMatch) {
+		return void 0;
+	}
+
+	var ref = refMatch[1];
+	var refPath = path.join(git, ref);
+
+	try {
+		return fs.readFileSync(refPath, 'utf8').trim();
+	} catch (e) {
+		// noop
+	}
+
+	var packedRefsPath = path.join(git, 'packed-refs');
+	var refsRaw;
+
+	try {
+		refsRaw = fs.readFileSync(packedRefsPath, 'utf8').trim();
+	} catch (e) {
+		return void 0;
+	}
+
+	var refsRegex = /^([0-9a-f]{40})\s+(.+)$/gm;
+	var refsMatch;
+	var refs = {};
+
+	while (refsMatch = refsRegex.exec(refsRaw)) {
+		refs[refsMatch[2]] = refsMatch[1];
+	}
+
+	return refs[ref];
+}

package.json

@@ -0,0 +1,34 @@
+{
+  "name": "monaco-languages",
+  "version": "0.1.0",
+  "description": "Bundle of many languages for the Monaco Editor.",
+  "scripts": {
+    "test": "node_modules/.bin/mocha",
+    "watch": "node_modules/.bin/gulp watch",
+    "prepublish": "node_modules/.bin/gulp release"
+  },
+  "author": "Microsoft Corporation",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Microsoft/monaco-languages"
+  },
+  "bugs": {
+    "url": "https://github.com/Microsoft/monaco-languages/issues"
+  },
+  "devDependencies": {
+    "event-stream": "^3.3.2",
+    "gulp": "^3.9.1",
+    "gulp-requirejs": "^0.1.3",
+    "gulp-tsb": "^1.10.4",
+    "gulp-uglify": "^1.5.3",
+    "jsdom-no-contextify": "^3.1.0",
+    "merge-stream": "^1.0.0",
+    "mocha": "^2.5.3",
+    "monaco-editor-core": "0.3.1",
+    "object-assign": "^4.1.0",
+    "rimraf": "^2.5.2",
+    "typescript": "^1.8.10",
+    "typescript-with-globs": "^0.1.4"
+  }
+}

src/bat.ts

@@ -0,0 +1,106 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: 'REM'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [
+	// 		{
+	// 			openTrigger: 'l',
+	// 			open: /setlocal$/i,
+	// 			closeComplete: 'endlocal',
+	// 			matchCase: true,
+	// 			closeTrigger: 'l',
+	// 			close: /endlocal$/i,
+	// 			tokenType: 'keyword.tag-setlocal'
+	// 		}
+	// 	],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.bat',
+
+	brackets: [
+		{ token: 'punctuation.bracket', open: '{', close: '}' },
+		{ token: 'punctuation.parenthesis', open: '(', close: ')' },
+		{ token: 'punctuation.square', open: '[', close: ']' }
+	],
+
+	keywords: /call|defined|echo|errorlevel|exist|for|goto|if|pause|set|shift|start|title|not|pushd|popd/,
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?&|+\-*\/\^;\.,]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			[/^(\s*)(rem(?:\s.*|))$/, ['','comment']],
+
+			[/(\@?)(@keywords)(?!\w)/, [{token:'support.function'}, {token:'support.function.$2'}]],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// blocks
+			[/setlocal(?!\w)/, { token: 'support.function.tag-setlocal', bracket: '@open' }],
+			[/endlocal(?!\w)/, { token: 'support.function.tag-setlocal', bracket: '@close' }],
+
+			// words
+			[/[a-zA-Z_]\w*/, ''],
+
+			// labels
+			[/:\w*/, 'metatag'],
+
+			// variables
+			[/%[^%]+%/, 'variable'],
+			[/%%[\w]+(?!\w)/, 'variable'],
+
+			// punctuations
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'punctuation'],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'constant.numeric.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'constant.numeric.hex'],
+			[/\d+/, 'constant.numeric'],
+
+			// punctuation: after number because of .\d floats
+			[/[;,.]/, 'punctuation'],
+
+			// strings:
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		string: [
+			[/[^\\"'%]+/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/%[\w ]+%/, 'variable'],
+			[/%%[\w]+(?!\w)/, 'variable'],
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+							'@default': 'string' }} ],
+			[/$/, 'string', '@popall']
+		],
+
+	}
+};

src/coffee.ts

@@ -0,0 +1,157 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#%\^\&\*\(\)\=\$\-\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		blockComment: ['###', '###'],
+		lineComment: '#'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [
+	// 		{ open: /for$/ }, { open: /while$/ },	{ open: /loop$/ }, { open: /if$/ }, { open: /unless$/ },
+	// 		{ open: /else$/ }, { open: /switch$/ }, { open: /try$/ }, { open: /catch$/ }, { open: /finally$/ },
+	// 		{ open: /class$/ }, { open: /->$/ }
+	// 	],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.coffee',
+
+	brackets: [
+		{ open:'{', close:'}', token:'delimiter.curly'},
+		{ open:'[', close:']', token:'delimiter.square'},
+		{ open:'(', close:')', token:'delimiter.parenthesis'}
+	],
+
+	regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/,
+
+	keywords: [
+		'and', 'or', 'is', 'isnt', 'not', 'on', 'yes', '@', 'no', 'off',
+		'true', 'false', 'null', 'this',
+		'new', 'delete', 'typeof', 'in', 'instanceof',
+		'return', 'throw', 'break', 'continue', 'debugger',
+		'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally',
+		'class', 'extends', 'super',
+		'undefined', 'then', 'unless', 'until', 'loop', 'of', 'by', 'when'
+		],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?&%|+\-*\/\^\.,\:]+/,
+	escapes:  /\\(?:[abfnrtv\\"'$]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// identifiers and keywords
+			[/\@[a-zA-Z_]\w*/, 'variable.predefined'],
+			[/[a-zA-Z_]\w*/, { cases: {
+				'this': 'variable.predefined',
+				'@keywords': { token: 'keyword.$0' },
+				'@default': ''
+				}
+			}],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// Comments
+			[/###/, 'comment', '@comment'],
+			[/#.*$/, 'comment'],
+
+			// regular expressions
+			['///', { token: 'regexp', next: '@hereregexp' }],
+
+			[/^(\s*)(@regEx)/, ['', 'regexp']],
+			[/(\()(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\,)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\=)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\:)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\[)(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\!)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\&)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\|)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\?)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\{)(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\;)(\s*)(@regEx)/, ['', '', 'regexp']],
+
+
+			// delimiters
+			[/}/, { cases: {
+					'$S2==interpolatedstring' : { token: 'string', next: '@pop' }
+					,	'@default'   : '@brackets' } }],
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d+[eE]([\-+]?\d+)?/, 'number.float'],
+			[/\d+\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/0[0-7]+(?!\d)/, 'number.octal'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[,.]/, 'delimiter'],
+
+			// strings:
+			[/"""/, 'string', '@herestring."""'],
+			[/'''/,  'string', '@herestring.\'\'\''],
+			[/"/,  { cases: { '@eos': 'string', '@default': {token:'string', next:'@string."'} }} ],
+			[/'/, { cases: { '@eos': 'string', '@default': {token:'string', next:'@string.\''} }} ],
+		],
+
+		string: [
+			[/[^"'\#\\]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\./, 'string.escape.invalid'],
+			[/\./, 'string.escape.invalid'],
+
+			[/#{/,  { cases: { '$S2=="': { token: 'string', next: 'root.interpolatedstring' }, '@default': 'string' }}],
+
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' }, '@default': 'string' }} ],
+			[/#/, 'string']
+		],
+
+		herestring: [
+			[/("""|''')/, { cases: { '$1==$S2': { token: 'string', next: '@pop' }, '@default': 'string' } }],
+			[/[^#\\'"]+/,'string' ],
+			[/['"]+/,'string' ],
+			[/@escapes/, 'string.escape'],
+			[/\./, 'string.escape.invalid'],
+
+			[/#{/, { token: 'string.quote', bracket: '@open', next: 'root.interpolatedstring' } ],
+			[/#/, 'string']
+		],
+
+		comment: [
+			[/[^#]+/, 'comment', ],
+			[/###/, 'comment', '@pop'],
+			[/#/, 'comment' ],
+		],
+
+		hereregexp: [
+			[/[^\\\/#]/, 'regexp'],
+			[/\\./, 'regexp'],
+			[/#.*$/, 'comment'],
+			['///[igm]*', { token: 'regexp', next: '@pop' }],
+			[/\//, 'regexp'],
+		],
+	},
+};

src/cpp.ts

@@ -0,0 +1,306 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.cpp',
+
+	brackets: [
+		{ token: 'delimiter.curly', open: '{', close: '}' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' },
+		{ token: 'delimiter.square', open: '[', close: ']' },
+		{ token: 'delimiter.angle', open: '<', close: '>' }
+	],
+
+	keywords: [
+		'abstract',
+		'amp',
+		'array',
+		'auto',
+		'bool',
+		'break',
+		'case',
+		'catch',
+		'char',
+		'class',
+		'const',
+		'constexpr',
+		'const_cast',
+		'continue',
+		'cpu',
+		'decltype',
+		'default',
+		'delegate',
+		'delete',
+		'do',
+		'double',
+		'dynamic_cast',
+		'each',
+		'else',
+		'enum',
+		'event',
+		'explicit',
+		'export',
+		'extern',
+		'false',
+		'final',
+		'finally',
+		'float',
+		'for',
+		'friend',
+		'gcnew',
+		'generic',
+		'goto',
+		'if',
+		'in',
+		'initonly',
+		'inline',
+		'int',
+		'interface',
+		'interior_ptr',
+		'internal',
+		'literal',
+		'long',
+		'mutable',
+		'namespace',
+		'new',
+		'noexcept',
+		'nullptr',
+		'__nullptr',
+		'operator',
+		'override',
+		'partial',
+		'pascal',
+		'pin_ptr',
+		'private',
+		'property',
+		'protected',
+		'public',
+		'ref',
+		'register',
+		'reinterpret_cast',
+		'restrict',
+		'return',
+		'safe_cast',
+		'sealed',
+		'short',
+		'signed',
+		'sizeof',
+		'static',
+		'static_assert',
+		'static_cast',
+		'struct',
+		'switch',
+		'template',
+		'this',
+		'thread_local',
+		'throw',
+		'tile_static',
+		'true',
+		'try',
+		'typedef',
+		'typeid',
+		'typename',
+		'union',
+		'unsigned',
+		'using',
+		'virtual',
+		'void',
+		'volatile',
+		'wchar_t',
+		'where',
+		'while',
+
+		'_asm', // reserved word with one underscores
+		'_based',
+		'_cdecl',
+		'_declspec',
+		'_fastcall',
+		'_if_exists',
+		'_if_not_exists',
+		'_inline',
+		'_multiple_inheritance',
+		'_pascal',
+		'_single_inheritance',
+		'_stdcall',
+		'_virtual_inheritance',
+		'_w64',
+
+		'__abstract',  // reserved word with two underscores
+		'__alignof',
+		'__asm',
+		'__assume',
+		'__based',
+		'__box',
+		'__builtin_alignof',
+		'__cdecl',
+		'__clrcall',
+		'__declspec',
+		'__delegate',
+		'__event',
+		'__except',
+		'__fastcall',
+		'__finally',
+		'__forceinline',
+		'__gc',
+		'__hook',
+		'__identifier',
+		'__if_exists',
+		'__if_not_exists',
+		'__inline',
+		'__int128',
+		'__int16',
+		'__int32',
+		'__int64',
+		'__int8',
+		'__interface',
+		'__leave',
+		'__m128',
+		'__m128d',
+		'__m128i',
+		'__m256',
+		'__m256d',
+		'__m256i',
+		'__m64',
+		'__multiple_inheritance',
+		'__newslot',
+		'__nogc',
+		'__noop',
+		'__nounwind',
+		'__novtordisp',
+		'__pascal',
+		'__pin',
+		'__pragma',
+		'__property',
+		'__ptr32',
+		'__ptr64',
+		'__raise',
+		'__restrict',
+		'__resume',
+		'__sealed',
+		'__single_inheritance',
+		'__stdcall',
+		'__super',
+		'__thiscall',
+		'__try',
+		'__try_cast',
+		'__typeof',
+		'__unaligned',
+		'__unhook',
+		'__uuidof',
+		'__value',
+		'__virtual_inheritance',
+		'__w64',
+		'__wchar_t'
+	],
+
+	operators: [
+		'=', '>', '<', '!', '~', '?', ':',
+		'==', '<=', '>=', '!=', '&&', '||', '++', '--',
+		'+', '-', '*', '/', '&', '|', '^', '%', '<<',
+		'>>', '>>>', '+=', '-=', '*=', '/=', '&=', '|=',
+		'^=', '%=', '<<=', '>>=', '>>>='
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /(ll|LL|u|U|l|L)?(ll|LL|u|U|l|L)?/,
+	floatsuffix: /[fFlL]?/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [[ attributes ]].
+			[/\[\[.*\]\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^\s*#\w+/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+									'@default'  : '' } } ],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/0[xX][0-9a-fA-F']*[0-9a-fA-F](@integersuffix)/, 'number.hex'],
+			[/0[0-7']*[0-7](@integersuffix)/, 'number.octal'],
+			[/0[bB][0-1']*[0-1](@integersuffix)/, 'number.binary'],
+			[/\d[\d']*\d(@integersuffix)/, 'number'],
+			[/\d(@integersuffix)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/,  'comment.doc', '@doccomment' ],
+			[/\/\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],	// this breaks block comments in the shape of /* //*/
+			[/\*\//,    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		doccomment: [
+			[/[^\/*]+/, 'comment.doc' ],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/,    'comment.doc.invalid' ],
+			[/\*\//,    'comment.doc', '@pop'  ],
+			[/[\/*]/,   'comment.doc' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        'string', '@pop' ]
+		],
+	},
+};

src/csharp.ts

@@ -0,0 +1,175 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.cs',
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '<', close: '>', token: 'delimiter.angle' }
+	],
+
+	keywords: [
+		'extern', 'alias', 'using', 'bool', 'decimal', 'sbyte', 'byte', 'short',
+		'ushort', 'int', 'uint', 'long', 'ulong', 'char', 'float', 'double',
+		'object', 'dynamic', 'string', 'assembly', 'is', 'as', 'ref',
+		'out', 'this', 'base', 'new', 'typeof', 'void', 'checked', 'unchecked',
+		'default', 'delegate', 'var', 'const', 'if', 'else', 'switch', 'case',
+		'while', 'do', 'for', 'foreach', 'in', 'break', 'continue', 'goto',
+		'return', 'throw', 'try', 'catch', 'finally', 'lock', 'yield', 'from',
+		'let', 'where', 'join', 'on', 'equals', 'into', 'orderby', 'ascending',
+		'descending', 'select', 'group', 'by', 'namespace', 'partial', 'class',
+		'field', 'event', 'method', 'param', 'property', 'public', 'protected',
+		'internal', 'private', 'abstract', 'sealed', 'static', 'struct', 'readonly',
+		'volatile', 'virtual', 'override', 'params', 'get', 'set', 'add', 'remove',
+		'operator', 'true', 'false', 'implicit', 'explicit', 'interface', 'enum',
+		'null', 'async', 'await','fixed','sizeof','stackalloc','unsafe', 'nameof',
+		'when'
+		],
+
+	namespaceFollows: [
+		'namespace', 'using',
+	],
+
+	parenFollows: [
+		'if', 'for', 'while', 'switch', 'foreach', 'using', 'catch', 'when'
+	],
+
+	operators: [
+		'=', '??', '||', '&&', '|', '^', '&', '==', '!=', '<=', '>=', '<<',
+		'+', '-', '*', '/', '%', '!', '~', '++', '--','+=',
+		'-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '>>', '=>'
+	],
+
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+
+	// escape sequences
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// identifiers and keywords
+			[/\@?[a-zA-Z_]\w*/, { cases: {
+				'@namespaceFollows': { token: 'keyword.$0', next: '@namespace' },
+				'@keywords': { token: 'keyword.$0', next: '@qualified' },
+				'@default': { token: 'identifier', next: '@qualified' }
+				}
+			}],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/}/, { cases: {
+					'$S2==interpolatedstring' : { token: 'string.quote', bracket: '@close', next: '@pop' }
+				,	'@default'   : '@brackets' } }],
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter', '@default'  : '' } } ],
+
+			// literal string
+			[/\@"/, { token: 'string.quote', bracket: '@open', next: '@litstring' } ],
+
+			// interpolated string
+			[/\$"/, { token: 'string.quote', bracket: '@open', next: '@interpolatedstring' } ],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?[fFdD]?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  { token: 'string.quote', bracket: '@open', next: '@string' } ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		qualified: [
+			[/[a-zA-Z_][\w]*/, { cases:
+				{ '@keywords': {token:'keyword.$0'},
+				'@default': 'identifier' }
+			}],
+			[/\./, 'delimiter'],
+			['','','@pop'],
+		],
+
+		namespace: [
+			{ include: '@whitespace' },
+			[/[A-Z]\w*/, 'namespace'],
+			[/[\.=]/, 'delimiter'],
+			['','','@pop'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/,    'comment', '@push' ],    // no nested comments :-(
+			['\\*/',    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+
+		litstring: [
+			[/[^"]+/,    'string'],
+			[/""/,       'string.escape'],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+
+		interpolatedstring: [
+			[/[^\\"{]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/{{/,       'string.escape'],
+			[/}}/,       'string.escape'],
+			[/{/,        { token: 'string.quote', bracket: '@open', next: 'root.interpolatedstring' } ],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+
+		whitespace: [
+			[/^[ \t\v\f]*#\w.*$/, 'namespace.cpp' ],
+			[/[ \t\v\f\r\n]+/, ''],
+			[/\/\*/,       'comment', '@comment' ],
+			[/\/\/.*$/,    'comment'],
+		],
+	},
+};

src/dockerfile.ts

@@ -0,0 +1,97 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage>{
+	defaultToken: '',
+	tokenPostfix: '.dockerfile',
+
+	instructions: /FROM|MAINTAINER|RUN|EXPOSE|ENV|ADD|VOLUME|LABEL|USER|WORKDIR|COPY|CMD|ENTRYPOINT/,
+
+	instructionAfter: /ONBUILD/,
+
+	variableAfter: /ENV/,
+
+	variable:/\${?[\w]+}?/,
+
+	tokenizer: {
+	root: [
+			{ include: '@whitespace' },
+			{ include: '@comment' },
+
+			[/(@instructionAfter)(\s+)/, ['keyword', { token: '', next: '@instructions' }]],
+			['', 'keyword', '@instructions']
+	],
+
+	instructions: [
+			[/(@variableAfter)(\s+)([\w]+)/, ['keyword', '',{token:'variable', next:'@arguments'}]],
+			[/(@instructions)/, 'keyword', '@arguments']
+	],
+
+	arguments: [
+			{ include: '@whitespace' },
+			{ include: '@strings' },
+
+			[/(@variable)/, { cases: { '@eos': {token:'variable', next:'@popall'}, '@default': 'variable' }} ],
+			[/\\/, { cases: { '@eos': '', '@default': '' }}],
+			[/./, { cases: { '@eos': {token:'', next:'@popall'}, '@default': '' } }],
+	],
+
+	// Deal with white space, including comments
+	whitespace: [
+		[/\s+/, { cases: { '@eos': {token:'', next:'@popall'}, '@default': '' }}],
+	],
+
+	comment: [
+		[/(^#.*$)/, 'comment', '@popall']
+	],
+
+	// Recognize strings, including those broken across lines with \ (but not without)
+	strings: [
+		[/'$/, 'string', '@popall'],
+		[/'/, 'string', '@stringBody'],
+		[/"$/, 'string', '@popall'],
+		[/"/, 'string', '@dblStringBody']
+	],
+	stringBody: [
+		[/[^\\\$']/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+
+		[/\\./, 'string.escape'],
+		[/'$/, 'string', '@popall'],
+		[/'/, 'string', '@pop'],
+		[/(@variable)/, 'variable' ],
+
+		[/\\$/, 'string'],
+		[/$/, 'string', '@popall']
+	],
+	dblStringBody: [
+		[/[^\\\$"]/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+
+		[/\\./, 'string.escape'],
+		[/"$/, 'string', '@popall'],
+		[/"/, 'string', '@pop'],
+		[/(@variable)/, 'variable' ],
+
+		[/\\$/, 'string'],
+		[/$/, 'string', '@popall']
+	]
+	}
+};

src/fsharp.ts

@@ -0,0 +1,133 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['(*', '*)'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] }
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.fs',
+
+	keywords: [
+		'abstract', 'and', 'atomic', 'as',
+		'assert', 'asr', 'base', 'begin',
+		'break', 'checked', 'component',
+		'const', 'constraint', 'constructor',
+		'continue', 'class', 'default',
+		'delegate','do', 'done', 'downcast',
+		'downto', 'elif', 'else', 'end',
+		'exception', 'eager', 'event', 'external',
+		'extern',	'false', 'finally',	'for',
+		'fun',	'function', 'fixed', 'functor',
+		'global', 'if', 'in', 'include', 'inherit',
+		'inline', 'interface', 'internal', 'land',
+		'lor', 'lsl','lsr', 'lxor', 'lazy', 'let',
+		'match', 'member','mod','module', 'mutable',
+		'namespace', 'method', 'mixin', 'new', 'not',
+		'null', 'of', 'open', 'or', 'object',
+		'override', 'private', 'parallel', 'process',
+		'protected', 'pure', 'public', 'rec', 'return',
+		'static', 'sealed', 'struct', 'sig', 'then',
+		'to', 'true', 'tailcall', 'trait',
+		'try', 'type', 'upcast', 'use',
+		'val', 'void', 'virtual', 'volatile',
+		'when', 'while','with', 'yield'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\^%;\.,\/]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /[uU]?[yslnLI]?/,
+	floatsuffix: /[fFmM]?/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [< attributes >].
+			[/\[<.*>\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^#(if|else|endif)/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, 'delimiter' ],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/0x[0-9a-fA-F]+LF/, 'number.float'],
+			[/0x[0-9a-fA-F]+(@integersuffix)/, 'number.hex'],
+			[/0b[0-1]+(@integersuffix)/, 'number.bin'],
+			[/\d+(@integersuffix)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"""/, 'string', '@string."""'],
+			[/"/, 'string', '@string."' ],
+
+			// literal string
+			[/\@"/, { token: 'string.quote', bracket: '@open', next: '@litstring' }],
+
+			// characters
+			[/'[^\\']'B?/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\(\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\*]+/, 'comment' ],
+			[/\*\)/,    'comment', '@pop'  ],
+			[/\*/,   'comment' ]
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/("""|"B?)/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+									'@default': 'string' }} ]
+		],
+
+		litstring: [
+			[/[^"]+/,    'string'],
+			[/""/,       'string.escape'],
+			[/"/,        { token: 'string.quote', bracket: '@close', next: '@pop' } ]
+		],
+	},
+};

src/go.ts

@@ -0,0 +1,166 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] }
+	]
+};
+
+export var language = <ILanguage> {
+
+	defaultToken: '',
+	tokenPostfix: '.go',
+
+	keywords: [
+		'break',
+		'case',
+		'chan',
+		'const',
+		'continue',
+		'default',
+		'defer',
+		'else',
+		'fallthrough',
+		'for',
+		'func',
+		'go',
+		'goto',
+		'if',
+		'import',
+		'interface',
+		'map',
+		'package',
+		'range',
+		'return',
+		'select',
+		'struct',
+		'switch',
+		'type',
+		'var',
+		'bool',
+		'true',
+		'false',
+		'uint8',
+		'uint16',
+		'uint32',
+		'uint64',
+		'int8',
+		'int16',
+		'int32',
+		'int64',
+		'float32',
+		'float64',
+		'complex64',
+		'complex128',
+		'byte',
+		'rune',
+		'uint',
+		'int',
+		'uintptr',
+		'string',
+		'nil',
+	],
+
+	operators: [
+		'+', '-', '*', '/', '%', '&', '|', '^', '<<', '>>', '&^',
+		'+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '<<=', '>>=', '&^=',
+		'&&', '||', '<-', '++', '--', '==', '<', '>', '=', '!', '!=', '<=', '>=', ':=', '...',
+		'(', ')', '', ']', '{', '}', ',', ';', '.', ':'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+																	'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [[ attributes ]].
+			[/\[\[.*\]\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^\s*#\w+/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+															'@default'  : '' } } ],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F']*[0-9a-fA-F]/, 'number.hex'],
+			[/0[0-7']*[0-7]/, 'number.octal'],
+			[/0[bB][0-1']*[0-1]/, 'number.binary'],
+			[/\d[\d']*/, 'number'],
+			[/\d/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/,  'comment.doc', '@doccomment' ],
+			[/\/\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//,    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		doccomment: [
+			[/[^\/*]+/, 'comment.doc' ],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/,    'comment.doc.invalid' ],
+			[/\*\//,    'comment.doc', '@pop'  ],
+			[/[\/*]/,   'comment.doc' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        'string', '@pop' ]
+		],
+	},
+};

src/ini.ts

@@ -0,0 +1,69 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.ini',
+
+	// we include these common regular expressions
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// sections
+			[/^\[[^\]]*\]/, 'metatag'],
+
+			// keys
+			[/(^\w+)(\s*)(\=)/, ['key', '', 'delimiter']],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// numbers
+			[/\d+/, 'number'],
+
+			// strings: recover on non-terminated strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/^\s*[#;].*$/,    		'comment'],
+		],
+
+		string: [
+			[/[^\\"']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+															'@default': 'string' }} ]
+		],
+	},
+};

src/jade.ts

@@ -0,0 +1,179 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.jade',
+
+	ignoreCase: true,
+
+	brackets: [
+			{ token:'delimiter.curly', open: '{', close: '}' },
+			{ token:'delimiter.array', open: '[', close: ']' },
+			{ token:'delimiter.parenthesis', open: '(', close: ')' }
+	],
+
+	keywords: [	'append', 'block', 'case', 'default', 'doctype', 'each', 'else', 'extends',
+		'for', 'if', 'in', 'include', 'mixin', 'typeof', 'unless', 'var', 'when'],
+
+	tags: [
+		'a', 'abbr', 'acronym', 'address', 'area', 'article', 'aside', 'audio',
+		'b', 'base', 'basefont', 'bdi', 'bdo', 'blockquote', 'body', 'br', 'button',
+		'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', 'command',
+		'datalist', 'dd', 'del', 'details', 'dfn', 'div', 'dl', 'dt',
+		'em', 'embed',
+		'fieldset', 'figcaption', 'figure', 'font', 'footer', 'form', 'frame', 'frameset',
+		'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'head', 'header', 'hgroup', 'hr', 'html',
+		'i', 'iframe', 'img', 'input', 'ins',
+		'keygen', 'kbd',
+		'label', 'li', 'link',
+		'map', 'mark', 'menu', 'meta', 'meter',
+		'nav', 'noframes', 'noscript',
+		'object', 'ol', 'optgroup', 'option', 'output',
+		'p', 'param', 'pre', 'progress',
+		'q',
+		'rp', 'rt', 'ruby',
+		's', 'samp', 'script', 'section', 'select', 'small', 'source', 'span', 'strike', 'strong', 'style', 'sub', 'summary', 'sup',
+		'table', 'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'time', 'title', 'tr', 'tracks', 'tt',
+		'u', 'ul',
+		'video',
+		'wbr'
+	],
+
+	// we include these common regular expressions
+	symbols: /[\+\-\*\%\&\|\!\=\/\.\,\:]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	tokenizer: {
+		root: [
+
+			// Tag or a keyword at start
+			[/^(\s*)([a-zA-Z_-][\w-]*)/,
+				{ cases: {
+					'$2@tags': { cases: { '@eos': ['', 'tag'], '@default': ['', { token: 'tag', next: '@tag.$1' }, ] } },
+					'$2@keywords': [ '', { token: 'keyword.$2'}, ],
+					'@default': [ '', '', ]}}
+			],
+
+			// id
+			[/^(\s*)(#[a-zA-Z_-][\w-]*)/, { cases: { '@eos': ['', 'tag.id'], '@default': ['', { token: 'tag.id', next: '@tag.$1' }] }}],
+
+			// class
+			[/^(\s*)(\.[a-zA-Z_-][\w-]*)/, { cases: { '@eos': ['', 'tag.class'], '@default': ['', { token: 'tag.class', next: '@tag.$1' }] } }],
+
+			// plain text with pipe
+			[/^(\s*)(\|.*)$/, '' ],
+
+			{ include: '@whitespace' },
+
+			// keywords
+			[/[a-zA-Z_$][\w$]*/, { cases: { '@keywords': {token:'keyword.$0'},
+											'@default': '' } }],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d+\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/\d+/, 'number'],
+
+			// strings:
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		tag: [
+			[/(\.)(\s*$)/, [ {token: 'delimiter', next:'@blockText.$S2.'}, '']],
+			[/\s+/, { token: '', next: '@simpleText' }],
+
+			// id
+			[/#[a-zA-Z_-][\w-]*/, { cases: { '@eos': { token: 'tag.id', next: '@pop' }, '@default': 'tag.id' } }],
+			// class
+			[/\.[a-zA-Z_-][\w-]*/, { cases: { '@eos': { token: 'tag.class', next: '@pop' }, '@default': 'tag.class' } }],
+			// attributes
+			[/\(/, { token: 'delimiter.parenthesis', bracket: '@open', next: '@attributeList' }],
+		],
+
+		simpleText: [
+			[/[^#]+$/, {token: '', next: '@popall'}],
+			[/[^#]+/, {token: ''}],
+
+			// interpolation
+			[/(#{)([^}]*)(})/, { cases: {
+				'@eos': ['interpolation.delimiter', 'interpolation', { token: 'interpolation.delimiter', next: '@popall' }],
+				'@default': ['interpolation.delimiter', 'interpolation', 'interpolation.delimiter'] }}],
+
+			[/#$/, { token: '', next: '@popall' }],
+			[/#/, '']
+		],
+
+		attributeList: [
+			[/\s+/, '' ],
+			[/(\w+)(\s*=\s*)("|')/, ['attribute.name', 'delimiter', { token: 'attribute.value', next:'@value.$3'}]],
+			[/\w+/, 'attribute.name'],
+
+
+			[/,/, { cases: { '@eos': { token: 'attribute.delimiter', next: '@popall' }, '@default': 'attribute.delimiter' } }],
+
+			[/\)$/, { token: 'delimiter.parenthesis', bracket: '@close', next: '@popall' }],
+			[/\)/, { token: 'delimiter.parenthesis', bracket: '@close', next: '@pop' }],
+		],
+
+		whitespace: [
+			[/^(\s*)(\/\/.*)$/, { token: 'comment', next: '@blockText.$1.comment' } ],
+			[/[ \t\r\n]+/, ''],
+			[/<!--/, { token: 'comment', bracket: '@open', next: '@comment' }],
+		],
+
+		blockText: [
+			[/^\s+.*$/, { cases: { '($S2\\s+.*$)': { token: '$S3' }, '@default': { token: '@rematch', next: '@popall' } } }],
+			[/./,  { token: '@rematch', next: '@popall' }]
+		],
+
+		comment: [
+			[/[^<\-]+/, 'comment.content' ],
+			[/-->/,  { token: 'comment', bracket: '@close', next: '@pop' } ],
+			[/<!--/, 'comment.content.invalid'],
+			[/[<\-]/, 'comment.content' ]
+		],
+
+		string: [
+			[/[^\\"'#]+/, { cases: { '@eos': { token: 'string', next: '@popall' }, '@default': 'string' } }],
+			[/@escapes/, { cases: { '@eos': { token: 'string.escape', next: '@popall' }, '@default': 'string.escape' }}],
+			[/\\./, { cases: { '@eos': { token: 'string.escape.invalid', next: '@popall' }, '@default': 'string.escape.invalid' }}],
+			// interpolation
+			[/(#{)([^}]*)(})/, ['interpolation.delimiter', 'interpolation', 'interpolation.delimiter']],
+			[/#/, 'string'],
+			[/["']/, { cases: { '$#==$S2': { token: 'string', next: '@pop' }, '@default': { token: 'string' } } }],
+		],
+
+		// Almost identical to above, except for escapes and the output token
+		value: [
+			[/[^\\"']+/, { cases: { '@eos': { token: 'attribute.value', next: '@popall' }, '@default': 'attribute.value' }}],
+			[/\\./, { cases: { '@eos': { token: 'attribute.value', next: '@popall' }, '@default': 'attribute.value' }}],
+			[/["']/, { cases: { '$#==$S2': { token: 'attribute.value', next: '@pop' }, '@default': { token: 'attribute.value' } } }],
+		],
+	},
+};

src/java.ts

@@ -0,0 +1,130 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	// the default separators except `@$`
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.java',
+
+	keywords: [
+		'abstract', 'continue', 'for', 'new', 'switch', 'assert', 'default',
+		'goto', 'package', 'synchronized', 'boolean', 'do', 'if', 'private',
+		'this', 'break', 'double', 'implements', 'protected', 'throw', 'byte',
+		'else', 'import', 'public', 'throws', 'case', 'enum', 'instanceof', 'return',
+		'transient', 'catch', 'extends', 'int', 'short', 'try', 'char', 'final',
+		'interface', 'static', 'void', 'class', 'finally', 'long', 'strictfp',
+		'volatile', 'const', 'float', 'native', 'super', 'while', 'true', 'false'
+	],
+
+	operators: [
+		'=', '>', '<', '!', '~', '?', ':',
+		'==', '<=', '>=', '!=', '&&', '||', '++', '--',
+		'+', '-', '*', '/', '&', '|', '^', '%', '<<',
+		'>>', '>>>', '+=', '-=', '*=', '/=', '&=', '|=',
+		'^=', '%=', '<<=', '>>=', '>>>='
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_$][\w$]*/, { cases: { '@keywords': {token:'keyword.$0'},
+																	'@default': 'identifier' } }],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+															'@default'  : '' } } ],
+
+			// @ annotations.
+			[/@\s*[a-zA-Z_\$][\w\$]*/, 'annotation'],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/0[xX](@hexdigits)[Ll]?/, 'number.hex'],
+			[/0(@octaldigits)[Ll]?/, 'number.octal'],
+			[/0[bB](@binarydigits)[Ll]?/, 'number.binary'],
+			[/(@digits)[fFdD]/, 'number.float'],
+			[/(@digits)[lL]?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string','string.escape','string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/,  'comment.doc', '@javadoc' ],
+			[/\/\*/,       		'comment', '@comment' ],
+			[/\/\/.*$/,    		'comment'],
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment' ],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//,    'comment', '@pop'  ],
+			[/[\/*]/,   'comment' ]
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		javadoc: [
+			[/[^\/*]+/, 'comment.doc' ],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/,    'comment.doc.invalid' ],
+			[/\*\//,    'comment.doc', '@pop'  ],
+			[/[\/*]/,   'comment.doc' ]
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"/,        'string', '@pop' ]
+		],
+	},
+};

src/lua.ts

@@ -0,0 +1,107 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '--',
+		blockComment: ['--[[', ']]'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.lua',
+
+	keywords: [
+		'and', 'break', 'do', 'else', 'elseif',
+		'end', 'false', 'for', 'function', 'goto', 'if',
+		'in', 'local', 'nil', 'not', 'or',
+		'repeat', 'return', 'then', 'true', 'until',
+		'while'
+	],
+
+	brackets: [
+		{ token: 'delimiter.bracket', open: '{', close: '}'},
+		{ token: 'delimiter.array', open: '[', close: ']'},
+		{ token: 'delimiter.parenthesis', open: '(', close: ')'}
+	],
+
+	operators: [
+		'+', '-', '*', '/', '%', '^', '#', '==', '~=', '<=', '>=', '<', '>', '=',
+		';', ':', ',', '.', '..', '...'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+			// whitespace
+			{ include: '@whitespace' },
+
+		// keys
+		[/(,)(\s*)([a-zA-Z_]\w*)(\s*)(:)(?!:)/, ['delimiter', '', 'key', '', 'delimiter']],
+		[/({)(\s*)([a-zA-Z_]\w*)(\s*)(:)(?!:)/, ['@brackets', '', 'key', '', 'delimiter']],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'delimiter',
+									'@default'  : '' } } ],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings: recover on non-terminated strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string."' ],
+			[/'/, 'string', '@string.\''],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/--\[([=]*)\[/,  'comment', '@comment.$1' ],
+			[/--.*$/,        'comment'],
+		],
+
+		comment: [
+			[/[^\]]+/, 'comment'],
+			[/\]([=]*)\]/, { cases: { '$1==$S2': { token: 'comment', next: '@pop' }, '@default' : 'comment' } } ],
+			[/./,   'comment' ]
+		],
+
+		string: [
+			[/[^\\"']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+									'@default': 'string' }} ]
+		],
+
+	},
+};

src/monaco.contribution.ts

@@ -0,0 +1,179 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+declare var require:<T>(moduleId:[string], callback:(module:T)=>void, error:(err:any)=>void)=>void;
+
+// Allow for running under nodejs/requirejs in tests
+var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+interface ILang extends monaco.languages.ILanguageExtensionPoint {
+	module: string;
+}
+
+interface ILangImpl {
+	conf: monaco.languages.IRichLanguageConfiguration;
+	language: monaco.languages.IMonarchLanguage;
+}
+
+let languageDefinitions:{[languageId:string]:ILang} = {};
+
+export function loadLanguage(languageId:string): monaco.Promise<void> {
+	let module = languageDefinitions[languageId].module;
+	return new _monaco.Promise<void>((c, e, p) => {
+		require<ILangImpl>([module], (mod) => {
+			_monaco.languages.setMonarchTokensProvider(languageId, mod.language);
+			_monaco.languages.setLanguageConfiguration(languageId, mod.conf);
+			c(void 0);
+		}, e);
+	});
+}
+
+function registerLanguage(def:ILang): void {
+	let languageId = def.id;
+
+	languageDefinitions[languageId] = def;
+	_monaco.languages.register(def);
+	_monaco.languages.onLanguage(languageId, () => {
+		loadLanguage(languageId);
+	});
+}
+
+
+registerLanguage({
+	id: 'bat',
+	extensions: [ '.bat', '.cmd'],
+	aliases: [ 'Batch', 'bat' ],
+	module: './bat'
+});
+registerLanguage({
+	id: 'coffeescript',
+	extensions: [ '.coffee' ],
+	aliases: [ 'CoffeeScript', 'coffeescript', 'coffee' ],
+	mimetypes: ['text/x-coffeescript', 'text/coffeescript'],
+	module: './coffee'
+});
+registerLanguage({
+	id: 'c',
+	extensions: [ '.c', '.h' ],
+	aliases: [ 'C', 'c' ],
+	module: './cpp'
+});
+registerLanguage({
+	id: 'cpp',
+	extensions: [ '.cpp', '.cc', '.cxx', '.hpp', '.hh', '.hxx' ],
+	aliases: [ 'C++', 'Cpp', 'cpp'],
+	module: './cpp'
+});
+registerLanguage({
+	id: 'csharp',
+	extensions: [ '.cs', '.csx' ],
+	aliases: [ 'C#', 'csharp' ],
+	module: './csharp'
+});
+registerLanguage({
+	id: 'dockerfile',
+	extensions: [ '.dockerfile' ],
+	filenames: [ 'Dockerfile' ],
+	aliases: [ 'Dockerfile' ],
+	module: './dockerfile'
+});
+registerLanguage({
+	id: 'fsharp',
+	extensions: [ '.fs', '.fsi', '.ml', '.mli', '.fsx', '.fsscript' ],
+	aliases: [ 'F#', 'FSharp', 'fsharp' ],
+	module: './fsharp'
+});
+registerLanguage({
+	id: 'go',
+	extensions: [ '.go' ],
+	aliases: [ 'Go' ],
+	module: './go'
+});
+registerLanguage({
+	id: 'ini',
+	extensions: [ '.ini', '.properties', '.gitconfig' ],
+	filenames: ['config', '.gitattributes', '.gitconfig', '.editorconfig'],
+	aliases: [ 'Ini', 'ini' ],
+	module: './ini'
+});
+registerLanguage({
+	id: 'jade',
+	extensions: [ '.jade', '.pug' ],
+	aliases: [ 'Jade', 'jade' ],
+	module: './jade'
+});
+registerLanguage({
+	id: 'java',
+	extensions: [ '.java', '.jav' ],
+	aliases: [ 'Java', 'java' ],
+	mimetypes: ['text/x-java-source', 'text/x-java'],
+	module: './java'
+});
+registerLanguage({
+	id: 'lua',
+	extensions: [ '.lua' ],
+	aliases: [ 'Lua', 'lua' ],
+	module: './lua'
+});
+registerLanguage({
+	id: 'objective-c',
+	extensions: [ '.m' ],
+	aliases: [ 'Objective-C'],
+	module: './objective-c'
+});
+registerLanguage({
+	id: 'powershell',
+	extensions: [ '.ps1', '.psm1', '.psd1' ],
+	aliases: [ 'PowerShell', 'powershell', 'ps', 'ps1' ],
+	module: './powershell'
+});
+registerLanguage({
+	id: 'python',
+	extensions: [ '.py', '.rpy', '.pyw', '.cpy', '.gyp', '.gypi' ],
+	aliases: [ 'Python', 'py' ],
+	firstLine: '^#!/.*\\bpython[0-9.-]*\\b',
+	module: './python'
+});
+registerLanguage({
+	id: 'r',
+	extensions: [ '.r', '.rhistory', '.rprofile', '.rt' ],
+	aliases: [ 'R', 'r' ],
+	module: './r'
+});
+registerLanguage({
+	id: 'ruby',
+	extensions: [ '.rb', '.rbx', '.rjs', '.gemspec', '.pp' ],
+	filenames: [ 'rakefile' ],
+	aliases: [ 'Ruby', 'rb' ],
+	module: './ruby'
+});
+registerLanguage({
+	id: 'swift',
+	aliases: ['Swift','swift'],
+	extensions: ['.swift'],
+	mimetypes: ['text/swift'],
+	module: './swift'
+});
+registerLanguage({
+	id: 'sql',
+	extensions: [ '.sql' ],
+	aliases: [ 'SQL' ],
+	module: './sql'
+});
+registerLanguage({
+	id: 'vb',
+	extensions: [ '.vb' ],
+	aliases: [ 'Visual Basic', 'vb' ],
+	module: './vb'
+});
+registerLanguage({
+	id: 'xml',
+	extensions: [ '.xml', '.dtd', '.ascx', '.csproj', '.config', '.wxi', '.wxl', '.wxs', '.xaml', '.svg', '.svgz' ],
+	firstLine : '(\\<\\?xml.*)|(\\<svg)|(\\<\\!doctype\\s+svg)',
+	aliases: [ 'XML', 'xml' ],
+	mimetypes: ['text/xml', 'application/xml', 'application/xaml+xml', 'application/xml-dtd'],
+	module: './xml'
+});

src/objective-c.ts

@@ -0,0 +1,201 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')'], ['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.objective-c',
+
+	keywords: [
+		'#import',
+		'#include',
+		'#define',
+		'#else',
+		'#endif',
+		'#if',
+		'#ifdef',
+		'#ifndef',
+		'#ident',
+		'#undef',
+
+		'@class',
+		'@defs',
+		'@dynamic',
+		'@encode',
+		'@end',
+		'@implementation',
+		'@interface',
+		'@package',
+		'@private',
+		'@protected',
+		'@property',
+		'@protocol',
+		'@public',
+		'@selector',
+		'@synthesize',
+
+		'__declspec',
+
+		'assign',
+		'auto',
+
+		'BOOL',
+		'break',
+		'bycopy',
+		'byref',
+
+		'case',
+		'char',
+		'Class',
+		'const',
+		'copy',
+		'continue',
+
+		'default',
+		'do',
+		'double',
+
+		'else',
+		'enum',
+		'extern',
+
+		'FALSE',
+		'false',
+		'float',
+		'for',
+
+		'goto',
+
+		'if',
+		'in',
+		'int',
+		'id',
+		'inout',
+		'IMP',
+
+		'long',
+
+		'nil',
+		'nonatomic',
+		'NULL',
+
+		'oneway',
+		'out',
+
+		'private',
+		'public',
+		'protected',
+
+		'readwrite',
+		'readonly',
+		'register',
+		'return',
+
+		'SEL',
+		'self',
+		'short',
+		'signed',
+		'sizeof',
+		'static',
+		'struct',
+		'super',
+		'switch',
+
+		'typedef',
+		'TRUE',
+		'true',
+
+		'union',
+		'unsigned',
+
+		'volatile',
+		'void',
+
+		'while',
+	],
+
+	decpart: /\d(_?\d)*/,
+	decimal: /0|@decpart/,
+
+	tokenizer: {
+	root: [
+			{ include: '@comments' },
+			{ include: '@whitespace' },
+			{ include: '@numbers' },
+			{ include: '@strings' },
+
+			[/[,:;]/, 'delimiter'],
+			[/[{}\[\]()<>]/, '@brackets'],
+
+			[/[a-zA-Z@#]\w*/, { cases: { '@keywords': 'keyword',
+										'@default': 'identifier' } }],
+
+			[/[<>=\\+\\-\\*\\/\\^\\|\\~,]|and\\b|or\\b|not\\b]/, 'operator'],
+	],
+
+	whitespace: [
+		[/\s+/, 'white'],
+	],
+
+	comments: [
+			['\\/\\*','comment','@comment' ],
+			['\\/\\/+.*','comment' ],
+		],
+
+	comment: [
+			['\\*\\/','comment','@pop' ],
+			['.', 'comment', ],
+		],
+
+	numbers: [
+			[/0[xX][0-9a-fA-F]*(_?[0-9a-fA-F])*/, 'number.hex'],
+			[/@decimal((\.@decpart)?([eE][\-+]?@decpart)?)[fF]*/, {
+				cases: {  	'(\\d)*': 'number',
+							'$0':'number.float' }} ]
+		],
+
+	// Recognize strings, including those broken across lines with \ (but not without)
+	strings: [
+		[/'$/, 'string.escape', '@popall'],
+		[/'/, 'string.escape', '@stringBody'],
+		[/"$/, 'string.escape', '@popall'],
+		[/"/, 'string.escape', '@dblStringBody']
+	],
+	stringBody: [
+		[/\\./, 'string'],
+		[/'/, 'string.escape', '@popall'],
+		[/.(?=.*')/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	],
+	dblStringBody: [
+		[/\\./, 'string'],
+		[/"/, 'string.escape', '@popall'],
+		[/.(?=.*")/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	]
+	}
+};

src/powershell.ts

@@ -0,0 +1,125 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	// the default separators except `$-`
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\#%\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '#',
+		blockComment: ['<#', '#>'],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [
+	// 			{ tokenType:'string', openTrigger: '"', open: /@"$/, closeComplete: '"@' },
+	// 			{ tokenType:'string', openTrigger: '\'', open: /@'$/, closeComplete: '\'@' },
+	// 			{ tokenType:'string', openTrigger: '"', open: /"$/, closeComplete: '"' },
+	// 			{ tokenType: 'string', openTrigger: '\'', open: /'$/, closeComplete: '\'' }
+	// ],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.ps1',
+
+	brackets: [
+		{ token: 'delimiter.curly', open: '{', close: '}' },
+		{ token: 'delimiter.square', open: '[', close: ']' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' }
+	],
+
+	keywords: [
+		'begin', 'break', 'catch', 'class', 'continue', 'data',
+		'define', 'do', 'dynamicparam', 'else', 'elseif', 'end',
+		'exit', 'filter', 'finally', 'for', 'foreach', 'from',
+		'function', 'if', 'in', 'param', 'process', 'return',
+		'switch', 'throw', 'trap', 'try', 'until', 'using',
+		'var', 'while', 'workflow', 'parallel', 'sequence', 'inlinescript', 'configuration'
+	],
+
+	helpKeywords: /SYNOPSIS|DESCRIPTION|PARAMETER|EXAMPLE|INPUTS|OUTPUTS|NOTES|LINK|COMPONENT|ROLE|FUNCTIONALITY|FORWARDHELPTARGETNAME|FORWARDHELPCATEGORY|REMOTEHELPRUNSPACE|EXTERNALHELP/,
+
+	// we include these common regular expressions
+	symbols: /[=><!~?&%|+\-*\/\^;\.,]+/,
+	escapes: /`(?:[abfnrtv\\"'$]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// commands and keywords
+			[/[a-zA-Z_][\w-]*/, { cases: { '@keywords': {token:'keyword.$0'},
+											'@default': '' } }],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// labels
+			[/^:\w*/, 'metatag'],
+
+			// variables
+			[/\$(\{((global|local|private|script|using):)?[\w]+\}|((global|local|private|script|using):)?[\w]+)/, 'variable'],
+
+			// Comments
+			[/<#/, 'comment', '@comment'],
+			[/#.*$/, 'comment'],
+
+			// delimiters
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings:
+			[/\@"/, 'string', '@herestring."'],
+			[/\@'/,  'string', '@herestring.\''],
+			[/"/,  { cases: { '@eos': 'string', '@default': {token:'string', next:'@string."'} }} ],
+			[/'/, { cases: { '@eos': 'string', '@default': {token:'string', next:'@string.\''} }} ],
+		],
+
+		string: [
+			[/[^"'\$`]+/, { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }}],
+			[/@escapes/, { cases: { '@eos': {token:'string.escape', next:'@popall'}, '@default': 'string.escape' }}],
+			[/`./, { cases: { '@eos': {token:'string.escape.invalid', next:'@popall'}, '@default': 'string.escape.invalid' }}],
+
+			[/\$[\w]+$/, { cases: { '$S2=="': { token: 'variable', next: '@popall' }, '@default': { token: 'string', next: '@popall' } } }],
+			[/\$[\w]+/, { cases: { '$S2=="': 'variable', '@default': 'string' }}],
+
+			[/["']/,     { cases: { '$#==$S2' : { token: 'string', next: '@pop' },
+									'@default': { cases: { '@eos': {token:'string', next:'@popall'}, '@default': 'string' }} }} ],
+		],
+
+		herestring: [
+			[/^\s*(["'])@/, { cases: { '$1==$S2': { token: 'string', next: '@pop' }, '@default': 'string' } }],
+			[/[^\$`]+/,'string' ],
+			[/@escapes/, 'string.escape'],
+			[/`./, 'string.escape.invalid'],
+			[/\$[\w]+/, { cases: { '$S2=="': 'variable', '@default': 'string' } }],
+		],
+
+		comment: [
+			[/[^#\.]+/, 'comment' ],
+			[/#>/, 'comment', '@pop'],
+			[/(\.)(@helpKeywords)(?!\w)/, { token: 'comment.keyword.$2' } ],
+			[/[\.#]/,   'comment' ]
+		],
+	},
+};

src/python.ts

@@ -0,0 +1,232 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+		blockComment: ['\'\'\'', '\'\'\''],
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+	// Cause an automatic indent to occur after lines ending in :.
+	// enhancedBrackets: [ { open: /.*:\s*$/,  closeComplete: 'else:' } ],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.python',
+
+	keywords: [
+		'and',
+		'as',
+		'assert',
+		'break',
+		'class',
+		'continue',
+		'def',
+		'del',
+		'elif',
+		'else',
+		'except',
+		'exec',
+		'finally',
+		'for',
+		'from',
+		'global',
+		'if',
+		'import',
+		'in',
+		'is',
+		'lambda',
+		'None',
+		'not',
+		'or',
+		'pass',
+		'print',
+		'raise',
+		'return',
+		'self',
+		'try',
+		'while',
+		'with',
+		'yield',
+
+		'int',
+		'float',
+		'long',
+		'complex',
+		'hex',
+
+		'abs',
+		'all',
+		'any',
+		'apply',
+		'basestring',
+		'bin',
+		'bool',
+		'buffer',
+		'bytearray',
+		'callable',
+		'chr',
+		'classmethod',
+		'cmp',
+		'coerce',
+		'compile',
+		'complex',
+		'delattr',
+		'dict',
+		'dir',
+		'divmod',
+		'enumerate',
+		'eval',
+		'execfile',
+		'file',
+		'filter',
+		'format',
+		'frozenset',
+		'getattr',
+		'globals',
+		'hasattr',
+		'hash',
+		'help',
+		'id',
+		'input',
+		'intern',
+		'isinstance',
+		'issubclass',
+		'iter',
+		'len',
+		'locals',
+		'list',
+		'map',
+		'max',
+		'memoryview',
+		'min',
+		'next',
+		'object',
+		'oct',
+		'open',
+		'ord',
+		'pow',
+		'print',
+		'property',
+		'reversed',
+		'range',
+		'raw_input',
+		'reduce',
+		'reload',
+		'repr',
+		'reversed',
+		'round',
+		'set',
+		'setattr',
+		'slice',
+		'sorted',
+		'staticmethod',
+		'str',
+		'sum',
+		'super',
+		'tuple',
+		'type',
+		'unichr',
+		'unicode',
+		'vars',
+		'xrange',
+		'zip',
+
+		'True',
+		'False',
+
+		'__dict__',
+		'__methods__',
+		'__members__',
+		'__class__',
+		'__bases__',
+		'__name__',
+		'__mro__',
+		'__subclasses__',
+		'__init__',
+		'__import__'
+	],
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.bracket' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	tokenizer: {
+	root: [
+			{ include: '@whitespace' },
+			{ include: '@numbers' },
+			{ include: '@strings' },
+
+			[/[,:;]/, 'delimiter'],
+			[/[{}\[\]()]/, '@brackets'],
+
+			[/@[a-zA-Z]\w*/, 'tag'],
+			[/[a-zA-Z]\w*/, { cases: { '@keywords': 'keyword',
+										'@default': 'identifier' } }]
+	],
+
+	// Deal with white space, including single and multi-line comments
+	whitespace: [
+		[/\s+/, 'white'],
+		[/(^#.*$)/, 'comment'],
+		[/('''.*''')|(""".*""")/, 'string'],
+		[/'''.*$/, 'string', '@endDocString'],
+		[/""".*$/, 'string', '@endDblDocString']
+	],
+	endDocString: [
+		[/\\'/, 'string'],
+		[/.*'''/, 'string', '@popall'],
+		[/.*$/, 'string']
+	],
+	endDblDocString: [
+		[/\\"/, 'string'],
+		[/.*"""/, 'string', '@popall'],
+		[/.*$/, 'string']
+	],
+
+	// Recognize hex, negatives, decimals, imaginaries, longs, and scientific notation
+	numbers: [
+		[/-?0x([abcdef]|[ABCDEF]|\d)+[lL]?/, 'number.hex'],
+		[/-?(\d*\.)?\d+([eE][+\-]?\d+)?[jJ]?[lL]?/, 'number']
+	],
+
+	// Recognize strings, including those broken across lines with \ (but not without)
+	strings: [
+		[/'$/, 'string.escape', '@popall'],
+		[/'/, 'string.escape', '@stringBody'],
+		[/"$/, 'string.escape', '@popall'],
+		[/"/, 'string.escape', '@dblStringBody']
+	],
+	stringBody: [
+		[/\\./, 'string'],
+		[/'/, 'string.escape', '@popall'],
+		[/.(?=.*')/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	],
+	dblStringBody: [
+		[/\\./, 'string'],
+		[/"/, 'string.escape', '@popall'],
+		[/.(?=.*")/, 'string'],
+		[/.*\\$/, 'string'],
+		[/.*$/, 'string', '@popall']
+	]
+	}
+};

src/r.ts

@@ -0,0 +1,230 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [['{','}'], ['[',']'], ['(',')']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.r',
+
+	roxygen: [
+		'@param',
+		'@return',
+		'@name',
+		'@rdname',
+		'@examples',
+		'@include',
+		'@docType',
+		'@S3method',
+		'@TODO',
+		'@aliases',
+		'@alias',
+		'@assignee',
+		'@author',
+		'@callGraphDepth',
+		'@callGraph',
+		'@callGraphPrimitives',
+		'@concept',
+		'@exportClass',
+		'@exportMethod',
+		'@exportPattern',
+		'@export',
+		'@formals',
+		'@format',
+		'@importClassesFrom',
+		'@importFrom',
+		'@importMethodsFrom',
+		'@import',
+		'@keywords',
+		'@method',
+		'@nord',
+		'@note',
+		'@references',
+		'@seealso',
+		'@setClass',
+		'@slot',
+		'@source',
+		'@title',
+		'@usage'
+	],
+
+	constants: [
+		'NULL',
+		'FALSE',
+		'TRUE',
+		'NA',
+		'Inf',
+		'NaN ',
+		'NA_integer_',
+		'NA_real_',
+		'NA_complex_',
+		'NA_character_ ',
+		'T',
+		'F',
+		'LETTERS',
+		'letters',
+		'month.abb',
+		'month.name',
+		'pi',
+		'R.version.string'
+	],
+
+	keywords: [
+		'break',
+		'next',
+		'return',
+		'if',
+		'else',
+		'for',
+		'in',
+		'repeat',
+		'while',
+		'array',
+		'category',
+		'character',
+		'complex',
+		'double',
+		'function',
+		'integer',
+		'list',
+		'logical',
+		'matrix',
+		'numeric',
+		'vector',
+		'data.frame',
+		'factor',
+		'library',
+		'require',
+		'attach',
+		'detach',
+		'source'
+	],
+
+	special: [
+		'\\n',
+		'\\r',
+		'\\t',
+		'\\b',
+		'\\a',
+		'\\f',
+		'\\v',
+		'\\\'',
+		'\\"',
+		'\\\\'
+	],
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.bracket' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	tokenizer: {
+		root: [
+			{ include: '@numbers' },
+			{ include: '@strings' },
+
+			[/[{}\[\]()]/, '@brackets'],
+
+			{ include: '@operators' },
+
+			[/#'/, 'comment.doc', '@roxygen'],
+			[/(^#.*$)/, 'comment'],
+
+			[/\s+/, 'white'],
+
+			[/[,:;]/, 'delimiter'],
+
+			[/@[a-zA-Z]\w*/, 'tag'],
+			[/[a-zA-Z]\w*/, {
+				cases: {
+					'@keywords': 'keyword',
+					'@constants': 'constant',
+					'@default': 'identifier'
+				}
+			}]
+		],
+
+		// Recognize Roxygen comments
+		roxygen: [
+			[/@\w+/, {
+				cases: {
+					'@roxygen': 'tag',
+					'@eos': { token: 'comment.doc', next: '@pop' },
+					'@default': 'comment.doc'
+				}
+			}],
+			[/\s+/, {
+				cases: {
+					'@eos': { token: 'comment.doc', next: '@pop' },
+					'@default': 'comment.doc'
+				}
+			}],
+			[/.*/, { token: 'comment.doc', next: '@pop' }]
+		],
+
+		// Recognize positives, negatives, decimals, imaginaries, and scientific notation
+		numbers: [
+			[/-?(\d*\.)?\d+([eE][+\-]?\d+)?/, 'number']
+		],
+
+		// Recognize operators
+		operators: [
+			[/<{1,2}-/, 'operator'],
+			[/->{1,2}/, 'operator'],
+			[/%[^%\s]+%/, 'operator'],
+			[/\*\*/, 'operator'],
+			[/%%/, 'operator'],
+			[/&&/, 'operator'],
+			[/\|\|/, 'operator'],
+			[/<</, 'operator'],
+			[/>>/, 'operator'],
+			[/[-+=&|!<>^~*/:$]/, 'operator']
+		],
+
+		// Recognize strings, including those broken across lines
+		strings: [
+			[/'/, 'string.escape', '@stringBody'],
+			[/"/, 'string.escape', '@dblStringBody']
+		],
+		stringBody: [
+			[/\\./, {
+				cases: {
+					'@special': 'string',
+					'@default': 'error-token'
+				}
+			}],
+			[/'/, 'string.escape', '@popall'],
+			[/./, 'string'],
+		],
+		dblStringBody: [
+			[/\\./, {
+				cases: {
+					'@special': 'string',
+					'@default': 'error-token'
+				}
+			}],
+			[/"/, 'string.escape', '@popall'],
+			[/./, 'string'],
+		]
+	}
+};

src/ruby.ts

@@ -0,0 +1,395 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+		blockComment: ['=begin', '=end'],
+	},
+	brackets: [['(',')'],['{','}'], ['[',']']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+	],
+	__electricCharacterSupport: {
+		// trigger outdenting on 'end'
+		embeddedElectricCharacters: ['d']
+	}
+};
+
+/*
+ * Ruby language definition
+ *
+ * Quite a complex language due to elaborate escape sequences
+ * and quoting of literate strings/regular expressions, and
+ * an 'end' keyword that does not always apply to modifiers like until and while,
+ * and a 'do' keyword that sometimes starts a block, but sometimes is part of
+ * another statement (like 'while').
+ *
+ * (1) end blocks:
+ * 'end' may end declarations like if or until, but sometimes 'if' or 'until'
+ * are modifiers where there is no 'end'. Also, 'do' sometimes starts a block
+ * that is ended by 'end', but sometimes it is part of a 'while', 'for', or 'until'
+ * To do proper brace matching we do some elaborate state manipulation.
+ * some examples:
+ *
+ *   until bla do
+ *     work until tired
+ *     list.each do
+ *       something if test
+ *     end
+ *   end
+ *
+ * or
+ *
+ * if test
+ *  something (if test then x end)
+ *  bar if bla
+ * end
+ *
+ * or, how about using class as a property..
+ *
+ * class Test
+ *   def endpoint
+ *     self.class.endpoint || routes
+ *   end
+ * end
+ *
+ * (2) quoting:
+ * there are many kinds of strings and escape sequences. But also, one can
+ * start many string-like things as '%qx' where q specifies the kind of string
+ * (like a command, escape expanded, regular expression, symbol etc.), and x is
+ * some character and only another 'x' ends the sequence. Except for brackets
+ * where the closing bracket ends the sequence.. and except for a nested bracket
+ * inside the string like entity. Also, such strings can contain interpolated
+ * ruby expressions again (and span multiple lines). Moreover, expanded
+ * regular expression can also contain comments.
+ */
+
+export var language = <ILanguage> {
+	tokenPostfix: '.ruby',
+
+	keywords: [
+		'__LINE__', '__ENCODING__', '__FILE__', 'BEGIN', 'END', 'alias', 'and', 'begin',
+		'break', 'case', 'class', 'def', 'defined?', 'do', 'else', 'elsif', 'end',
+		'ensure', 'for', 'false', 'if', 'in', 'module', 'next', 'nil', 'not', 'or', 'redo',
+		'rescue', 'retry', 'return', 'self', 'super', 'then', 'true', 'undef', 'unless',
+		'until', 'when', 'while', 'yield',
+	],
+
+	keywordops: [
+		'::', '..', '...', '?', ':', '=>'
+	],
+
+	builtins: [
+		'require', 'public', 'private', 'include', 'extend', 'attr_reader',
+		'protected', 'private_class_method', 'protected_class_method', 'new'
+	],
+
+	// these are closed by 'end' (if, while and until are handled separately)
+	declarations: [
+		'module','class','def','case','do','begin','for','if','while','until','unless'
+	],
+
+	linedecls: [
+		'def','case','do','begin','for','if','while','until','unless'
+	],
+
+	operators: [
+		'^', '&', '|', '<=>', '==', '===', '!~', '=~', '>', '>=', '<', '<=', '<<', '>>', '+',
+		'-', '*', '/', '%', '**', '~', '+@', '-@', '[]', '[]=', '`',
+		'+=', '-=', '*=', '**=', '/=', '^=', '%=', '<<=', '>>=', '&=', '&&=', '||=', '|='
+	],
+
+	brackets: [
+		{ open: '(', close: ')', token: 'delimiter.parenthesis'},
+		{ open: '{', close: '}', token: 'delimiter.curly'},
+		{ open: '[', close: ']', token: 'delimiter.square'}
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?:&|+\-*\/\^%\.]+/,
+
+	// escape sequences
+	escape:  /(?:[abefnrstv\\"'\n\r]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2}|u[0-9A-Fa-f]{4})/,
+	escapes: /\\(?:C\-(@escape|.)|c(@escape|.)|@escape)/,
+
+	decpart: /\d(_?\d)*/,
+	decimal: /0|@decpart/,
+
+	delim:     /[^a-zA-Z0-9\s\n\r]/,
+	heredelim: /(?:\w+|'[^']*'|"[^"]*"|`[^`]*`)/,
+
+	regexpctl: /[(){}\[\]\$\^|\-*+?\.]/,
+	regexpesc: /\\(?:[AzZbBdDfnrstvwWn0\\\/]|@regexpctl|c[A-Z]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4})?/,
+
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		// Main entry.
+		// root.<decl> where decl is the current opening declaration (like 'class')
+		root: [
+			// identifiers and keywords
+			// most complexity here is due to matching 'end' correctly with declarations.
+			// We distinguish a declaration that comes first on a line, versus declarations further on a line (which are most likey modifiers)
+			[/^(\s*)([a-z_]\w*[!?=]?)/, ['white',
+				{ cases: { 'for|until|while': { token: 'keyword.$2', bracket: '@open', next: '@dodecl.$2' },
+							'@declarations':   { token: 'keyword.$2', bracket: '@open', next: '@root.$2' },
+							'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' },
+							'@keywords': 'keyword',
+							'@builtins': 'predefined',
+							'@default': 'identifier' } }]],
+			[/[a-z_]\w*[!?=]?/,
+				{ cases: { 'if|unless|while|until': { token: 'keyword.$0x', bracket: '@open', next: '@modifier.$0x' },
+							'for': { token: 'keyword.$2', bracket: '@open', next: '@dodecl.$2' },
+							'@linedecls': { token: 'keyword.$0', bracket: '@open', next: '@root.$0' },
+							'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' },
+							'@keywords': 'keyword',
+							'@builtins': 'predefined',
+							'@default': 'identifier' } }],
+
+			[/[A-Z][\w]*[!?=]?/, 'constructor.identifier' ],     // constant
+			[/\$[\w]*/,    'global.constant' ],               // global
+			[/@[\w]*/,     'namespace.instance.identifier' ], // instance
+			[/@@[\w]*/,    'namespace.class.identifier' ],    // class
+
+			// here document
+			[/<<-(@heredelim).*/, { token: 'string.heredoc.delimiter', bracket: '@open', next: '@heredoc.$1' } ],
+			[/[ \t\r\n]+<<(@heredelim).*/, { token: 'string.heredoc.delimiter', bracket: '@open', next: '@heredoc.$1' } ],
+			[/^<<(@heredelim).*/, { token: 'string.heredoc.delimiter', bracket: '@open', next: '@heredoc.$1' } ],
+
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// strings
+			[/"/,  { token: 'string.d.delim', bracket: '@open', next: '@dstring.d."'} ],
+			[/'/,  { token: 'string.sq.delim', bracket: '@open', next: '@sstring.sq' } ],
+
+			// % literals. For efficiency, rematch in the 'pstring' state
+			[/%([rsqxwW]|Q?)/,  { token: '@rematch', next: 'pstring' } ],
+
+			// commands and symbols
+			[/`/,  { token: 'string.x.delim', bracket: '@open', next: '@dstring.x.`' } ],
+			[/:(\w|[$@])\w*[!?=]?/, 'string.s'],
+			[/:"/, { token: 'string.s.delim', bracket: '@open', next: '@dstring.s."' } ],
+			[/:'/, { token: 'string.s.delim', bracket: '@open', next: '@sstring.s' } ],
+
+			// regular expressions. Lookahead for a (not escaped) closing forwardslash on the same line
+			[/\/(?=(\\\/|[^\/\n])+\/)/, { token: 'regexp.delim', bracket: '@open', next: '@regexp' } ],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, { cases: { '@keywordops': 'keyword',
+									'@operators' : 'operator',
+									'@default'   : '' } } ],
+
+			[/[;,]/, 'delimiter'],
+
+			// numbers
+			[/0[xX][0-9a-fA-F](_?[0-9a-fA-F])*/, 'number.hex'],
+			[/0[_oO][0-7](_?[0-7])*/, 'number.octal'],
+			[/0[bB][01](_?[01])*/, 'number.binary'],
+			[/0[dD]@decpart/, 'number'],
+			[/@decimal((\.@decpart)?([eE][\-+]?@decpart)?)/, { cases: { '$1': 'number.float',
+																		'@default': 'number' }}],
+
+		],
+
+		// used to not treat a 'do' as a block opener if it occurs on the same
+		// line as a 'do' statement: 'while|until|for'
+		// dodecl.<decl> where decl is the declarations started, like 'while'
+		dodecl: [
+			[/^/, { token: '', switchTo: '@root.$S2' }], // get out of do-skipping mode on a new line
+			[/[a-z_]\w*[!?=]?/, { cases: { 'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' }, // end on same line
+											'do' : { token: 'keyword', switchTo: '@root.$S2' }, // do on same line: not an open bracket here
+											'@linedecls': { token: '@rematch', switchTo: '@root.$S2' }, // other declaration on same line: rematch
+											'@keywords': 'keyword',
+											'@builtins': 'predefined',
+											'@default': 'identifier' } }],
+			{ include: '@root' }
+		],
+
+		// used to prevent potential modifiers ('if|until|while|unless') to match
+		// with 'end' keywords.
+		// modifier.<decl>x where decl is the declaration starter, like 'if'
+		modifier: [
+			[/^/, '', '@pop'], // it was a modifier: get out of modifier mode on a new line
+			[/[a-z_]\w*[!?=]?/, { cases: { 'end': { token: 'keyword.$S2', bracket: '@close', next: '@pop' }, // end on same line
+											'then|else|elsif|do': { token: 'keyword', switchTo: '@root.$S2' }, // real declaration and not a modifier
+											'@linedecls': { token: '@rematch', switchTo: '@root.$S2' }, // other declaration => not a modifier
+											'@keywords': 'keyword',
+											'@builtins': 'predefined',
+											'@default': 'identifier' } }],
+			{ include: '@root' }
+		],
+
+		// single quote strings (also used for symbols)
+		// sstring.<kind>  where kind is 'sq' (single quote) or 's' (symbol)
+		sstring: [
+			[/[^\\']+/,      'string.$S2' ],
+			[/\\\\|\\'|\\$/, 'string.$S2.escape'],
+			[/\\./,          'string.$S2.invalid'],
+			[/'/,            { token: 'string.$S2.delim', bracket: '@close', next: '@pop'} ]
+		],
+
+		// double quoted "string".
+		// dstring.<kind>.<delim> where kind is 'd' (double quoted), 'x' (command), or 's' (symbol)
+		// and delim is the ending delimiter (" or `)
+		dstring: [
+			[/[^\\`"#]+/, 'string.$S2'],
+			[/#/,         'string.$S2.escape', '@interpolated' ],
+			[/\\$/,       'string.$S2.escape' ],
+			[/@escapes/,  'string.$S2.escape'],
+			[/\\./,       'string.$S2.escape.invalid'],
+			[/[`"]/,      { cases: { '$#==$S3':  { token: 'string.$S2.delim', bracket: '@close', next: '@pop'},
+										'@default': 'string.$S2' } } ]
+		],
+
+		// literal documents
+		// heredoc.<close> where close is the closing delimiter
+		heredoc: [
+			[/^(\s*)(@heredelim)$/, { cases: { '$2==$S2': ['string.heredoc', { token: 'string.heredoc.delimiter', bracket: '@close', next: '@pop' }],
+												'@default': ['string.heredoc','string.heredoc'] }}],
+			[/.*/, 'string.heredoc' ],
+		],
+
+		// interpolated sequence
+		interpolated: [
+			[/\$\w*/,      'global.constant', '@pop' ],
+			[/@\w*/,       'namespace.class.identifier', '@pop' ],
+			[/@@\w*/,      'namespace.instance.identifier', '@pop' ],
+			[/[{]/, { token: 'string.escape.curly', bracket: '@open', switchTo: '@interpolated_compound' }],
+			['', '', '@pop' ], // just a # is interpreted as a #
+		],
+
+		// any code
+		interpolated_compound: [
+			[/[}]/, { token: 'string.escape.curly', bracket: '@close', next: '@pop'} ],
+			{ include: '@root' },
+		],
+
+		// %r quoted regexp
+		// pregexp.<open>.<close> where open/close are the open/close delimiter
+		pregexp: [
+			{ include: '@whitespace' },
+			// turns out that you can quote using regex control characters, aargh!
+			// for example; %r|kgjgaj| is ok (even though | is used for alternation)
+			// so, we need to match those first
+			[/[^\(\{\[\\]/, { cases: { '$#==$S3' : { token: 'regexp.delim', bracket: '@close', next: '@pop' },
+										'$#==$S2' : { token: 'regexp.delim', bracket: '@open', next: '@push' }, // nested delimiters are allowed..
+										'~[)}\\]]' : '@brackets.regexp.escape.control',
+										'~@regexpctl': 'regexp.escape.control',
+										'@default': 'regexp' }}],
+			{ include: '@regexcontrol' },
+		],
+
+		// We match regular expression quite precisely
+		regexp: [
+			{ include:   '@regexcontrol' },
+			[/[^\\\/]/,  'regexp' ],
+			['/[ixmp]*', { token: 'regexp.delim', bracket: '@close'}, '@pop' ],
+		],
+
+		regexcontrol: [
+			[/(\{)(\d+(?:,\d*)?)(\})/, ['@brackets.regexp.escape.control', 'regexp.escape.control', '@brackets.regexp.escape.control'] ],
+			[/(\[)(\^?)/,     ['@brackets.regexp.escape.control',{ token: 'regexp.escape.control', next: '@regexrange'}]],
+			[/(\()(\?[:=!])/, ['@brackets.regexp.escape.control', 'regexp.escape.control'] ],
+			[/\(\?#/,         { token: 'regexp.escape.control', bracket: '@open', next: '@regexpcomment' }],
+			[/[()]/,        '@brackets.regexp.escape.control'],
+			[/@regexpctl/,  'regexp.escape.control'],
+			[/\\$/,         'regexp.escape' ],
+			[/@regexpesc/,  'regexp.escape' ],
+			[/\\\./,        'regexp.invalid' ],
+			[/#/,           'regexp.escape', '@interpolated' ],
+		],
+
+		regexrange: [
+			[/-/,     'regexp.escape.control'],
+			[/\^/,    'regexp.invalid'],
+			[/\\$/,   'regexp.escape' ],
+			[/@regexpesc/, 'regexp.escape'],
+			[/[^\]]/, 'regexp'],
+			[/\]/,    '@brackets.regexp.escape.control', '@pop'],
+		],
+
+		regexpcomment: [
+			[ /[^)]+/, 'comment' ],
+			[ /\)/, { token: 'regexp.escape.control', bracket: '@close', next: '@pop' } ]
+		],
+
+
+		// % quoted strings
+		// A bit repetitive since we need to often special case the kind of ending delimiter
+		pstring: [
+			[/%([qws])\(/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.(.)' } ],
+			[/%([qws])\[/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.[.]' } ],
+			[/%([qws])\{/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.{.}' } ],
+			[/%([qws])</,   { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.<.>' } ],
+			[/%([qws])(@delim)/, { token: 'string.$1.delim', bracket: '@open', switchTo: '@qstring.$1.$2.$2' } ],
+
+			[/%r\(/,  { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.(.)' } ],
+			[/%r\[/,  { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.[.]' } ],
+			[/%r\{/,  { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.{.}' } ],
+			[/%r</,   { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.<.>' } ],
+			[/%r(@delim)/, { token: 'regexp.delim', bracket: '@open', switchTo: '@pregexp.$1.$1' } ],
+
+			[/%(x|W|Q?)\(/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.(.)' } ],
+			[/%(x|W|Q?)\[/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.[.]' } ],
+			[/%(x|W|Q?)\{/,  { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.{.}' } ],
+			[/%(x|W|Q?)</,   { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.<.>' } ],
+			[/%(x|W|Q?)(@delim)/, { token: 'string.$1.delim', bracket: '@open', switchTo: '@qqstring.$1.$2.$2' } ],
+
+			[/%([rqwsxW]|Q?)./, { token: 'invalid', next: '@pop' } ], // recover
+			[/./, { token: 'invalid', next: '@pop' } ], // recover
+		],
+
+		// non-expanded quoted string.
+		// qstring.<kind>.<open>.<close>
+		//  kind = q|w|s  (single quote, array, symbol)
+		//  open = open delimiter
+		//  close = close delimiter
+		qstring: [
+			[/\\$/, 'string.$S2.escape' ],
+			[/\\./, 'string.$S2.escape' ],
+			[/./,   { cases: { '$#==$S4' : { token: 'string.$S2.delim', bracket: '@close', next: '@pop' },
+								'$#==$S3' : { token: 'string.$S2.delim', bracket: '@open', next: '@push' }, // nested delimiters are allowed..
+								'@default': 'string.$S2' }}],
+		],
+
+		// expanded quoted string.
+		// qqstring.<kind>.<open>.<close>
+		//  kind = Q|W|x  (double quote, array, command)
+		//  open = open delimiter
+		//  close = close delimiter
+		qqstring: [
+			[/#/, 'string.$S2.escape', '@interpolated' ],
+			{ include: '@qstring' }
+		],
+
+
+		// whitespace & comments
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/^\s*=begin\b/,       'comment', '@comment' ],
+			[/#.*$/,    'comment'],
+		],
+
+		comment: [
+			[/[^=]+/, 'comment' ],
+			[/^\s*=begin\b/, 'comment.invalid' ],    // nested comment
+			[/^\s*=end\b.*/, 'comment', '@pop'  ],
+			[/[=]/, 'comment' ]
+		],
+	}
+};

src/swift.ts

@@ -0,0 +1,143 @@
+/*!---------------------------------------------------------------------------------------------
+ *  Copyright (C) David Owens II, owensd.io. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [['{','}'],['[',']'],['(',')'],['<','>']],
+	autoClosingPairs: [
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.swift',
+
+	// TODO(owensd): Support the full range of unicode valid identifiers.
+	identifier: /[a-zA-Z_][\w$]*/,
+	// TODO(owensd): Support the @availability macro properly.
+	attributes: [
+		'@autoclosure', '@noescape', '@noreturn', '@NSApplicationMain', '@NSCopying', '@NSManaged',
+		'@objc', '@UIApplicationMain', '@noreturn', '@availability', '@IBAction', '@IBDesignable', '@IBInspectable', '@IBOutlet'
+	],
+	accessmodifiers: [ 'public', 'private', 'internal' ],
+	keywords: [
+		'__COLUMN__', '__FILE__', '__FUNCTION__', '__LINE__', 'as', 'as!', 'as?', 'associativity', 'break', 'case', 'catch',
+		'class', 'continue', 'convenience', 'default', 'deinit', 'didSet', 'do', 'dynamic', 'dynamicType',
+		'else', 'enum', 'extension', 'fallthrough', 'final', 'for', 'func', 'get', 'guard', 'if', 'import', 'in', 'infix',
+		'init', 'inout', 'internal', 'is', 'lazy', 'left', 'let', 'mutating', 'nil', 'none', 'nonmutating', 'operator',
+		'optional', 'override', 'postfix', 'precedence', 'prefix', 'private', 'protocol', 'Protocol', 'public',
+		'repeat', 'required', 'return', 'right', 'self', 'Self', 'set', 'static', 'struct', 'subscript', 'super', 'switch',
+		'throw', 'throws', 'try', 'try!', 'Type', 'typealias', 'unowned', 'var', 'weak', 'where', 'while', 'willSet', 'FALSE', 'TRUE'
+	],
+
+	symbols: /[=(){}\[\].,:;@#\_&\-<>`?!+*\\\/]/,
+
+	// Moved . to operatorstart so it can be a delimiter
+	operatorstart: /[\/=\-+!*%<>&|^~?\u00A1-\u00A7\u00A9\u00AB\u00AC\u00AE\u00B0-\u00B1\u00B6\u00BB\u00BF\u00D7\u00F7\u2016-\u2017\u2020-\u2027\u2030-\u203E\u2041-\u2053\u2055-\u205E\u2190-\u23FF\u2500-\u2775\u2794-\u2BFF\u2E00-\u2E7F\u3001-\u3003\u3008-\u3030]/,
+	operatorend: /[\u0300-\u036F\u1DC0-\u1DFF\u20D0-\u20FF\uFE00-\uFE0F\uFE20-\uFE2F\uE0100-\uE01EF]/,
+	operators: /(@operatorstart)((@operatorstart)|(@operatorend))*/,
+
+	// TODO(owensd): These are borrowed from C#; need to validate correctness for Swift.
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	tokenizer: {
+		root: [
+			{ include: '@comment' },
+			{ include: '@attribute' },
+			{ include: '@literal' },
+			{ include: '@keyword' },
+			{ include: '@invokedmethod' },
+			{ include: '@symbol' },
+		],
+
+		symbol: [
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/[.]/, 'delimiter'],
+			[/@operators/, 'keyword.operator'],
+			[/@symbols/, 'keyword.operator']
+		],
+
+
+		comment: [
+			[ /\/\/\/.*$/, 'comment.doc' ],
+			[ /\/\*\*/, 'comment.doc', '@commentdocbody' ],
+			[ /\/\/.*$/, 'comment' ],
+			[ /\/\*/, 'comment', '@commentbody' ]
+		],
+		commentdocbody: [
+			[ /\/\*/, 'comment', '@commentbody' ],
+			[ /\*\//, 'comment.doc', '@pop' ],
+			[ /\:[a-zA-Z]+\:/, 'comment.doc.param' ],
+			[ /./, 'comment.doc' ]
+		],
+		commentbody: [
+			[ /\/\*/, 'comment', '@commentbody' ],
+			[ /\*\//, 'comment', '@pop' ],
+			[ /./, 'comment' ]
+		],
+
+		attribute: [
+			[ /\@@identifier/, { cases: { '@attributes': 'keyword.control', '@default': '' } } ]
+		],
+
+		literal: [
+			[ /"/, { token: 'string.quote', bracket: '@open', next: '@stringlit' } ],
+			[ /0[b]([01]_?)+/, 'number.binary' ],
+			[ /0[o]([0-7]_?)+/, 'number.octal' ],
+			[ /0[x]([0-9a-fA-F]_?)+([pP][\-+](\d_?)+)?/, 'number.hex' ],
+			[ /(\d_?)*\.(\d_?)+([eE][\-+]?(\d_?)+)?/, 'number.float'],
+			[ /(\d_?)+/, 'number' ]
+		],
+
+		stringlit: [
+			[ /\\\(/, { token: 'keyword.operator', bracket: '@open', next: '@interpolatedexpression' } ],
+			[ /@escapes/, 'string' ],
+			[ /\\./, 'string.escape.invalid' ],
+			[ /"/, { token: 'string.quote', bracket: '@close', next: '@pop' } ],
+			[ /./, 'string' ]
+		],
+
+		interpolatedexpression: [
+			[ /\(/, { token: 'keyword.operator', bracket: '@open', next: '@interpolatedexpression' } ],
+			[ /\)/, { token: 'keyword.operator', bracket: '@close', next: '@pop' } ],
+			{ include: '@literal' },
+			{ include: '@keyword' },
+			{ include: '@symbol' }
+		],
+
+		keyword: [
+			[ /`/, { token: 'keyword.operator', bracket: '@open', next: '@escapedkeyword' } ],
+			[ /@identifier/, { cases: { '@keywords': 'keyword', '[A-Z][\a-zA-Z0-9$]*': 'type.identifier', '@default': 'identifier' } }]
+		],
+
+		escapedkeyword: [
+			[ /`/, { token: 'keyword.operator', bracket: '@close', next: '@pop' } ],
+			[ /./, 'identifier' ]
+		],
+
+//		symbol: [
+//			[ /@symbols/, 'keyword.operator' ],
+//			[ /@operators/, 'keyword.operator' ]
+//		],
+
+		invokedmethod: [
+			[/([.])(@identifier)/, { cases: { '$2': ['delimeter', 'type.identifier'], '@default': '' } }],
+		]
+	}
+};

src/vb.ts

@@ -0,0 +1,179 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		lineComment: '\'',
+		blockComment: ['/*', '*/'],
+	},
+	brackets: [
+		['{','}'],['[',']'],['(',')'],['<','>'],
+		['addhandler','end addhandler'],
+		['class','end class'],
+		['enum','end enum'],
+		['event','end event'],
+		['function','end function'],
+		['get','end get'],
+		['if','end if'],
+		['interface','end interface'],
+		['module','end module'],
+		['namespace','end namespace'],
+		['operator','end operator'],
+		['property','end property'],
+		['raiseevent','end raiseevent'],
+		['removehandler','end removehandler'],
+		['select','end select'],
+		['set','end set'],
+		['structure','end structure'],
+		['sub','end sub'],
+		['synclock','end synclock'],
+		['try','end try'],
+		['while','end while'],
+		['with','end with'],
+		['using','end using'],
+		['do','loop'],
+		['for','next']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: '<', close: '>', notIn: ['string', 'comment'] },
+	]
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.vb',
+	ignoreCase: true,
+
+	brackets: [
+		{ token:'delimiter.bracket', open: '{', close: '}'},
+		{ token:'delimiter.array', open: '[', close: ']'},
+		{ token:'delimiter.parenthesis', open: '(', close: ')'},
+		{ token:'delimiter.angle', open: '<', close: '>'},
+
+		// Special bracket statement pairs
+		// according to https://msdn.microsoft.com/en-us/library/tsw2a11z.aspx
+		{ token: 'keyword.tag-addhandler', open: 'addhandler', close: 'end addhandler'},
+		{ token: 'keyword.tag-class', open: 'class', close: 'end class'},
+		{ token: 'keyword.tag-enum', open: 'enum', close: 'end enum'},
+		{ token: 'keyword.tag-event', open: 'event', close: 'end event'},
+		{ token: 'keyword.tag-function', open: 'function', close: 'end function'},
+		{ token: 'keyword.tag-get', open: 'get', close: 'end get'},
+		{ token: 'keyword.tag-if', open: 'if', close: 'end if'},
+		{ token: 'keyword.tag-interface', open: 'interface', close: 'end interface'},
+		{ token: 'keyword.tag-module', open: 'module', close: 'end module'},
+		{ token: 'keyword.tag-namespace', open: 'namespace', close: 'end namespace'},
+		{ token: 'keyword.tag-operator', open: 'operator', close: 'end operator'},
+		{ token: 'keyword.tag-property', open: 'property', close: 'end property'},
+		{ token: 'keyword.tag-raiseevent', open: 'raiseevent', close: 'end raiseevent'},
+		{ token: 'keyword.tag-removehandler', open: 'removehandler', close: 'end removehandler'},
+		{ token: 'keyword.tag-select', open: 'select', close: 'end select'},
+		{ token: 'keyword.tag-set', open: 'set', close: 'end set'},
+		{ token: 'keyword.tag-structure', open: 'structure', close: 'end structure'},
+		{ token: 'keyword.tag-sub', open: 'sub', close: 'end sub'},
+		{ token: 'keyword.tag-synclock', open: 'synclock', close: 'end synclock'},
+		{ token: 'keyword.tag-try', open: 'try', close: 'end try'},
+		{ token: 'keyword.tag-while', open: 'while', close: 'end while'},
+		{ token: 'keyword.tag-with', open: 'with', close: 'end with'},
+
+		// Other pairs
+		{ token: 'keyword.tag-using', open: 'using', close: 'end using' },
+		{ token: 'keyword.tag-do', open: 'do', close: 'loop' },
+		{ token: 'keyword.tag-for', open: 'for', close: 'next' }
+	],
+
+	keywords: [
+		'AddHandler', 'AddressOf', 'Alias', 'And', 'AndAlso', 'As', 'Async', 'Boolean', 'ByRef', 'Byte', 'ByVal', 'Call',
+		'Case', 'Catch', 'CBool', 'CByte', 'CChar', 'CDate', 'CDbl', 'CDec', 'Char', 'CInt', 'Class', 'CLng',
+		'CObj', 'Const', 'Continue', 'CSByte', 'CShort', 'CSng', 'CStr', 'CType', 'CUInt', 'CULng', 'CUShort',
+		'Date', 'Decimal', 'Declare', 'Default', 'Delegate', 'Dim', 'DirectCast', 'Do', 'Double', 'Each', 'Else',
+		'ElseIf', 'End', 'EndIf', 'Enum', 'Erase', 'Error', 'Event', 'Exit', 'False', 'Finally', 'For', 'Friend',
+		'Function', 'Get', 'GetType', 'GetXMLNamespace', 'Global', 'GoSub', 'GoTo', 'Handles', 'If', 'Implements',
+		'Imports', 'In', 'Inherits', 'Integer', 'Interface', 'Is', 'IsNot', 'Let', 'Lib', 'Like', 'Long', 'Loop',
+		'Me', 'Mod', 'Module', 'MustInherit', 'MustOverride', 'MyBase', 'MyClass', 'NameOf', 'Namespace', 'Narrowing', 'New',
+		'Next', 'Not', 'Nothing', 'NotInheritable', 'NotOverridable', 'Object', 'Of', 'On', 'Operator', 'Option',
+		'Optional', 'Or', 'OrElse', 'Out', 'Overloads', 'Overridable', 'Overrides', 'ParamArray', 'Partial',
+		'Private', 'Property', 'Protected', 'Public', 'RaiseEvent', 'ReadOnly', 'ReDim', 'RemoveHandler', 'Resume',
+		'Return', 'SByte', 'Select', 'Set', 'Shadows', 'Shared', 'Short', 'Single', 'Static', 'Step', 'Stop',
+		'String', 'Structure', 'Sub', 'SyncLock', 'Then', 'Throw', 'To', 'True', 'Try', 'TryCast', 'TypeOf',
+		'UInteger', 'ULong', 'UShort', 'Using', 'Variant', 'Wend', 'When', 'While', 'Widening', 'With', 'WithEvents',
+		'WriteOnly', 'Xor'
+	],
+
+	tagwords: [
+		'If', 'Sub', 'Select', 'Try', 'Class', 'Enum',
+		'Function', 'Get', 'Interface', 'Module', 'Namespace', 'Operator', 'Set', 'Structure', 'Using', 'While', 'With',
+		'Do', 'Loop', 'For', 'Next', 'Property', 'Continue', 'AddHandler', 'RemoveHandler', 'Event', 'RaiseEvent', 'SyncLock'
+	],
+
+	// we include these common regular expressions
+	symbols:  /[=><!~?;\.,:&|+\-*\/\^%]+/,
+	escapes:  /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /U?[DI%L&S@]?/,
+	floatsuffix: /[R#F!]?/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// special ending tag-words
+			[/next(?!\w)/, { token: 'keyword.tag-for', bracket: '@close'}],
+			[/loop(?!\w)/, { token: 'keyword.tag-do', bracket: '@close' }],
+
+			// usual ending tags
+			[/end\s+(?!for|do)([a-zA-Z_]\w*)/, { token: 'keyword.tag-$1', bracket: '@close' }],
+
+			// identifiers, tagwords, and keywords
+			[/[a-zA-Z_]\w*/, { cases: { '@tagwords': {token:'keyword.tag-$0', bracket: '@open'},
+										'@keywords': {token:'keyword.$0'},
+										'@default': 'identifier' } }],
+
+			// Preprocessor directive
+			[/^\s*#\w+/, 'keyword'],
+
+			// numbers
+			[/\d*\d+e([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+(e[\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/&H[0-9a-f]+(@integersuffix)/, 'number.hex'],
+			[/&0[0-7]+(@integersuffix)/, 'number.octal'],
+			[/\d+(@integersuffix)/, 'number'],
+
+			// date literal
+			[/#.*#/, 'number'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid' ],  // non-teminated string
+			[/"/,  'string', '@string' ],
+
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/(\'|REM(?!\w)).*$/,        'comment'],
+		],
+
+		string: [
+			[/[^\\"]+/,  'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./,      'string.escape.invalid'],
+			[/"C?/,        'string', '@pop' ]
+		],
+	},
+};

src/xml.ts

@@ -0,0 +1,104 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import IRichLanguageConfiguration = monaco.languages.IRichLanguageConfiguration;
+import ILanguage = monaco.languages.IMonarchLanguage;
+
+export var conf:IRichLanguageConfiguration = {
+	comments: {
+		blockComment: ['<!--', '-->'],
+	},
+	brackets: [['{','}'],['[',']'],['(',')'],['<','>']],
+	autoClosingPairs: [
+		{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+	]
+	// enhancedBrackets: [{
+	// 	tokenType: 'tag.tag-$1.xml',
+	// 	openTrigger: '>',
+	// 	open: /<(\w[\w\d]*)([^\/>]*(?!\/)>)[^<>]*$/i,
+	// 	closeComplete: '</$1>',
+	// 	closeTrigger: '>',
+	// 	close: /<\/(\w[\w\d]*)\s*>$/i
+	// }],
+};
+
+export var language = <ILanguage> {
+	defaultToken: '',
+	tokenPostfix: '.xml',
+
+	ignoreCase: true,
+
+	// Useful regular expressions
+	qualifiedName: /(?:[\w\.\-]+:)?[\w\.\-]+/,
+
+	tokenizer: {
+		root: [
+			[/[^<&]+/, ''],
+
+			{ include: '@whitespace' },
+
+			// Standard opening tag
+			[/(<)(@qualifiedName)/, [
+				{ token: 'delimiter.start', bracket: '@open' },
+				{ token: 'tag.tag-$2', bracket: '@open', next: '@tag.$2' }]],
+
+			// Standard closing tag
+			[/(<\/)(@qualifiedName)(\s*)(>)/, [
+				{ token: 'delimiter.end', bracket: '@open' },
+				{ token: 'tag.tag-$2', bracket: '@close' },
+				'',
+				{ token: 'delimiter.end', bracket: '@close' }]],
+
+			// Meta tags - instruction
+			[/(<\?)(@qualifiedName)/, [
+				{ token: 'delimiter.start', bracket: '@open' },
+				{ token: 'metatag.instruction', next: '@tag' }]],
+
+			// Meta tags - declaration
+			[/(<\!)(@qualifiedName)/, [
+				{ token: 'delimiter.start', bracket: '@open' },
+				{ token: 'metatag.declaration', next: '@tag' }]],
+
+			// CDATA
+			[/<\!\[CDATA\[/, { token: 'delimiter.cdata', bracket: '@open', next: '@cdata' }],
+
+			[/&\w+;/, 'string.escape'],
+		],
+
+		cdata: [
+			[/[^\]]+/, ''],
+			[/\]\]>/, { token: 'delimiter.cdata', bracket: '@close', next: '@pop' }],
+			[/\]/, '']
+		],
+
+		tag: [
+			[/[ \t\r\n]+/, '' ],
+			[/(@qualifiedName)(\s*=\s*)("[^"]*"|'[^']*')/, ['attribute.name', '', 'attribute.value']],
+			[/(@qualifiedName)(\s*=\s*)("[^">?\/]*|'[^'>?\/]*)(?=[\?\/]\>)/, ['attribute.name', '', 'attribute.value']],
+			[/(@qualifiedName)(\s*=\s*)("[^">]*|'[^'>]*)/, ['attribute.name', '', 'attribute.value']],
+			[/@qualifiedName/, 'attribute.name'],
+			[/\?>/, { token: 'delimiter.start', bracket: '@close', next: '@pop' }],
+			[/(\/)(>)/, [
+				{ token: 'tag.tag-$S2', bracket: '@close' },
+				{ token: 'delimiter.start', bracket: '@close', next: '@pop' }]],
+			[/>/, { token: 'delimiter.start', bracket: '@close', next: '@pop' }],
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/<!--/, { token: 'comment', bracket: '@open', next: '@comment' }]
+		],
+
+		comment: [
+			[/[^<\-]+/, 'comment.content' ],
+			[/-->/,  { token: 'comment', bracket: '@close', next: '@pop' } ],
+			[/<!--/, 'comment.content.invalid'],
+			[/[<\-]/, 'comment.content' ]
+		],
+	},
+};

test/all.js

@@ -0,0 +1,48 @@
+var requirejs = require("requirejs");
+var jsdom = require('jsdom-no-contextify');
+
+requirejs.config({
+	baseUrl: '',
+	paths: {
+		'vs': 'node_modules/monaco-editor-core/dev/vs'
+	},
+	nodeRequire: require
+});
+
+global.document = jsdom.jsdom('<!doctype html><html><body></body></html>');
+global.document.queryCommandSupported = function() {};
+global.self = global.window = global.document.parentWindow;
+global.navigator = global.window.navigator;
+global.window.require = requirejs;
+
+function MyWorker() {}
+MyWorker.prototype.postMessage = function() {};
+global.Worker = MyWorker;
+
+requirejs([
+	'vs/editor/editor.main'
+], function() {
+	requirejs([
+		'out/test/bat.test',
+		'out/test/coffee.test',
+		'out/test/cpp.test',
+		'out/test/csharp.test',
+		'out/test/dockerfile.test',
+		'out/test/fsharp.test',
+		'out/test/go.test',
+		'out/test/jade.test',
+		'out/test/java.test',
+		'out/test/lua.test',
+		'out/test/objective-c.test',
+		'out/test/powershell.test',
+		'out/test/python.test',
+		'out/test/r.test',
+		'out/test/ruby.test',
+		'out/test/swift.test',
+		'out/test/sql.test',
+		'out/test/vb.test',
+		'out/test/xml.test',
+	], function() {
+		run(); // We can launch the tests!
+	});
+});

test/assert.d.ts

@@ -0,0 +1,44 @@
+declare module "assert" {
+    function internal (value: any, message?: string): void;
+    namespace internal {
+        export class AssertionError implements Error {
+            name: string;
+            message: string;
+            actual: any;
+            expected: any;
+            operator: string;
+            generatedMessage: boolean;
+
+            constructor(options?: {message?: string; actual?: any; expected?: any;
+                                  operator?: string; stackStartFunction?: Function});
+        }
+
+        export function fail(actual?: any, expected?: any, message?: string, operator?: string): void;
+        export function ok(value: any, message?: string): void;
+        export function equal(actual: any, expected: any, message?: string): void;
+        export function notEqual(actual: any, expected: any, message?: string): void;
+        export function deepEqual(actual: any, expected: any, message?: string): void;
+        export function notDeepEqual(acutal: any, expected: any, message?: string): void;
+        export function strictEqual(actual: any, expected: any, message?: string): void;
+        export function notStrictEqual(actual: any, expected: any, message?: string): void;
+        export function deepStrictEqual(actual: any, expected: any, message?: string): void;
+        export function notDeepStrictEqual(actual: any, expected: any, message?: string): void;
+        export var throws: {
+            (block: Function, message?: string): void;
+            (block: Function, error: Function, message?: string): void;
+            (block: Function, error: RegExp, message?: string): void;
+            (block: Function, error: (err: any) => boolean, message?: string): void;
+        };
+
+        export var doesNotThrow: {
+            (block: Function, message?: string): void;
+            (block: Function, error: Function, message?: string): void;
+            (block: Function, error: RegExp, message?: string): void;
+            (block: Function, error: (err: any) => boolean, message?: string): void;
+        };
+
+        export function ifError(value: any): void;
+    }
+
+    export = internal;
+}

test/bat.test.ts

@@ -0,0 +1,332 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('bat', [
+	// support.functions
+	[{
+	line: '@echo off title Selfhost',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.bat' },
+		{ startIndex: 1, type: 'support.function.echo.bat' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 10, type: 'support.function.title.bat' },
+		{ startIndex: 15, type: '' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: 'REM',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}],
+
+	[{
+	line: '    REM a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.bat' }
+	]}],
+
+	[{
+	line: 'REM a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}],
+
+	[{
+	line: 'REMnot a comment',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	// constant.numerics
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' }
+	]}],
+
+	[{
+	line: '0.0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.hex.bat' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '1.72e-3',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.float.bat' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' },
+		{ startIndex: 1, type: 'punctuation.bat' },
+		{ startIndex: 2, type: 'constant.numeric.bat' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' },
+		{ startIndex: 3, type: 'punctuation.bat' },
+		{ startIndex: 4, type: 'constant.numeric.bat' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'constant.numeric.bat' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'punctuation.bat' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'constant.numeric.bat' }
+	]}],
+
+	// Strings
+	[{
+	line: 'set s = "string"',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.set.bat' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 6, type: 'punctuation.bat' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'string.bat' }
+	]}],
+
+	[{
+	line: '"use strict";',
+	tokens: [
+		{ startIndex: 0, type: 'string.bat' },
+		{ startIndex: 12, type: 'punctuation.bat' }
+	]}],
+
+	// Tags
+	[{
+	line: 'setlocal endlocal',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	[{
+	line: 'setlocal ENDLOCAL',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	[{
+	line: 'SETLOCAL endlocal',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	[{
+	line: 'setlocal setlocal endlocal',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'support.function.tag-setlocal.bat' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'support.function.tag-setlocal.bat' }
+	]}],
+
+	// Monarch generated
+	[{
+	line: 'rem asdf',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'REM',
+	tokens: [
+		{ startIndex: 0, type: 'comment.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'REMOVED not a comment really',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'support.function.not.bat' },
+		{ startIndex: 11, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'echo cool',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.echo.bat' },
+		{ startIndex: 4, type: '' }
+	]}, {
+	line: '@echo off',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.bat' },
+		{ startIndex: 1, type: 'support.function.echo.bat' },
+		{ startIndex: 5, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'setlocAL',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' }
+	]}, {
+	line: '	asdf',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '	asdf',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: 'endLocaL',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.tag-setlocal.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'call',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.call.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: ':MyLabel',
+	tokens: [
+		{ startIndex: 0, type: 'metatag.bat' }
+	]}, {
+	line: 'some command',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '%sdfsdf% ',
+	tokens: [
+		{ startIndex: 0, type: 'variable.bat' },
+		{ startIndex: 8, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'this is "a string %sdf% asdf"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'string.bat' },
+		{ startIndex: 18, type: 'variable.bat' },
+		{ startIndex: 23, type: 'string.bat' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'FOR %%A IN (1 2 3) DO (',
+	tokens: [
+		{ startIndex: 0, type: 'support.function.for.bat' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.bat' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 11, type: 'punctuation.parenthesis.bat' },
+		{ startIndex: 12, type: 'constant.numeric.bat' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'constant.numeric.bat' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'constant.numeric.bat' },
+		{ startIndex: 17, type: 'punctuation.parenthesis.bat' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 22, type: 'punctuation.parenthesis.bat' }
+	]}, {
+	line: '	SET VAR1=%VAR1%%%A',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'support.function.set.bat' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 9, type: 'punctuation.bat' },
+		{ startIndex: 10, type: 'variable.bat' }
+	]}, {
+	line: '	SET VAR2=%VAR2%%%A',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'support.function.set.bat' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 9, type: 'punctuation.bat' },
+		{ startIndex: 10, type: 'variable.bat' }
+	]}, {
+	line: '	use \'string %%a asdf asdf\'',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'string.bat' },
+		{ startIndex: 13, type: 'variable.bat' },
+		{ startIndex: 16, type: 'string.bat' }
+	]}, {
+	line: '	non terminated "string %%aaa sdf',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 16, type: 'string.bat' },
+		{ startIndex: 24, type: 'variable.bat' },
+		{ startIndex: 29, type: 'string.bat' }
+	]}, {
+	line: '	this shold NOT BE red',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'support.function.not.bat' },
+		{ startIndex: 15, type: '' }
+	]}, {
+	line: ')',
+	tokens: [
+		{ startIndex: 0, type: 'punctuation.parenthesis.bat' }
+	]}]
+]);

test/cpp.test.ts

@@ -0,0 +1,666 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('cpp', [
+	// Keywords
+	[{
+	line: 'int _tmain(int argc, _TCHAR* argv[])',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 10, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 11, type: 'keyword.int.cpp' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'identifier.cpp' },
+		{ startIndex: 19, type: 'delimiter.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'identifier.cpp' },
+		{ startIndex: 27, type: 'delimiter.cpp' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'identifier.cpp' },
+		{ startIndex: 33, type: 'delimiter.square.cpp' },
+		{ startIndex: 35, type: 'delimiter.parenthesis.cpp' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.cpp' },
+		{ startIndex: 1, type: 'identifier.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cpp' }
+	]}],
+
+	[{
+	line: '/* //*/ a',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cpp' },
+		{ startIndex: 5, type: 'delimiter.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: 'int x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.cpp' },
+		{ startIndex: 9, type: 'delimiter.cpp' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.cpp' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cpp' }
+	]}],
+
+	[{
+	line: 'int x = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cpp' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.cpp' },
+		{ startIndex: 32, type: 'delimiter.cpp' }
+	]}],
+
+	[{
+	line: 'int x = /* comment */ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cpp' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'number.cpp' },
+		{ startIndex: 23, type: 'delimiter.cpp' },
+		{ startIndex: 24, type: '' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cpp' },
+		{ startIndex: 8, type: 'delimiter.cpp' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cpp' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '12l',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '34U',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '55LL',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '34ul',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '55llU',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '5\'5llU',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '100\'000\'000',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '0x100\'aafllU',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.cpp' }
+	]}],
+
+	[{
+	line: '0342\'325',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.cpp' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.cpp' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '23.5l',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72E3l',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '1.72e3l',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cpp' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 1, type: 'delimiter.cpp' },
+		{ startIndex: 2, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 3, type: 'delimiter.cpp' },
+		{ startIndex: 4, type: 'number.cpp' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cpp' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cpp' }
+	]}],
+
+	// Monarch Generated
+	[{
+	line: '#include<iostream>',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 8, type: 'delimiter.angle.cpp' },
+		{ startIndex: 9, type: 'identifier.cpp' },
+		{ startIndex: 17, type: 'delimiter.angle.cpp' }
+	]}, {
+	line: '#include "/path/to/my/file.h"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#ifdef VAR',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' }
+	]}, {
+	line: '#define SUM(A,B) (A) + (B)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 12, type: 'identifier.cpp' },
+		{ startIndex: 13, type: 'delimiter.cpp' },
+		{ startIndex: 14, type: 'identifier.cpp' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 18, type: 'identifier.cpp' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.cpp' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 24, type: 'identifier.cpp' },
+		{ startIndex: 25, type: 'delimiter.parenthesis.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'int main(int argc, char** argv)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.cpp' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cpp' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 9, type: 'keyword.int.cpp' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.cpp' },
+		{ startIndex: 17, type: 'delimiter.cpp' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'keyword.char.cpp' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 26, type: 'identifier.cpp' },
+		{ startIndex: 30, type: 'delimiter.parenthesis.cpp' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '	return 0;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.return.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.cpp' },
+		{ startIndex: 9, type: 'delimiter.cpp' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'namespace TestSpace',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cpp' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '	using Asdf.CDE;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.using.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 11, type: 'delimiter.cpp' },
+		{ startIndex: 12, type: 'identifier.cpp' },
+		{ startIndex: 15, type: 'delimiter.cpp' }
+	]}, {
+	line: '	template <typename T>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.template.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.angle.cpp' },
+		{ startIndex: 11, type: 'keyword.typename.cpp' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.cpp' },
+		{ startIndex: 21, type: 'delimiter.angle.cpp' }
+	]}, {
+	line: '	class CoolClass : protected BaseClass',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.class.cpp' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'delimiter.cpp' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'keyword.protected.cpp' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'identifier.cpp' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '		private:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.private.cpp' },
+		{ startIndex: 9, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		static T field;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.static.cpp' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.cpp' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'identifier.cpp' },
+		{ startIndex: 16, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		public:',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.public.cpp' },
+		{ startIndex: 8, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		[[deprecated]]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'annotation.cpp' }
+	]}, {
+	line: '		foo method() const override',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.cpp' },
+		{ startIndex: 12, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'keyword.const.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'keyword.override.cpp' }
+	]}, {
+	line: '		{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '			auto s = new Bar();',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.auto.cpp' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cpp' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'delimiter.cpp' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.new.cpp' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.cpp' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 21, type: 'delimiter.cpp' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '			if (s.field) {',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.if.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 8, type: 'delimiter.cpp' },
+		{ startIndex: 9, type: 'identifier.cpp' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '				for(const auto & b : s.field) {',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.for.cpp' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 8, type: 'keyword.const.cpp' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.auto.cpp' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'delimiter.cpp' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'identifier.cpp' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.cpp' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'identifier.cpp' },
+		{ startIndex: 26, type: 'delimiter.cpp' },
+		{ startIndex: 27, type: 'identifier.cpp' },
+		{ startIndex: 32, type: 'delimiter.parenthesis.cpp' },
+		{ startIndex: 33, type: '' },
+		{ startIndex: 34, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '					break;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'keyword.break.cpp' },
+		{ startIndex: 10, type: 'delimiter.cpp' }
+	]}, {
+	line: '				}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '			}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '		}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		std::string s = "hello wordld\\n";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'identifier.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 7, type: 'identifier.cpp' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'identifier.cpp' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.cpp' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'string.cpp' },
+		{ startIndex: 31, type: 'string.escape.cpp' },
+		{ startIndex: 33, type: 'string.cpp' },
+		{ startIndex: 34, type: 'delimiter.cpp' }
+	]}, {
+	line: '		',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '		int number = 123\'123\'123Ull;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.int.cpp' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.cpp' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'delimiter.cpp' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'number.cpp' },
+		{ startIndex: 29, type: 'delimiter.cpp' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cpp' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#endif',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.cpp' }
+	]}]
+]);

test/csharp.test.ts

@@ -0,0 +1,744 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('csharp', [
+
+	// Generated from sample
+	[{
+	line: 'using System;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Collections.Generic;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 24, type: 'delimiter.cs' },
+		{ startIndex: 25, type: 'namespace.cs' },
+		{ startIndex: 32, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Diagnostics;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 24, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Linq;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 17, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Text;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 17, type: 'delimiter.cs' }
+	]}, {
+	line: 'using System.Threading.Tasks;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.using.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'namespace.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' },
+		{ startIndex: 13, type: 'namespace.cs' },
+		{ startIndex: 22, type: 'delimiter.cs' },
+		{ startIndex: 23, type: 'namespace.cs' },
+		{ startIndex: 28, type: 'delimiter.cs' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'namespace VS',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'namespace.cs' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '	class Program',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.class.cs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cs' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '		static void Main(string[] args)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.static.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.void.cs' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'identifier.cs' },
+		{ startIndex: 18, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 19, type: 'keyword.string.cs' },
+		{ startIndex: 25, type: 'delimiter.square.cs' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'identifier.cs' },
+		{ startIndex: 32, type: 'delimiter.parenthesis.cs' }
+	]}, {
+	line: '		{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			ProcessStartInfo si = new ProcessStartInfo();',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.cs' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.cs' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'keyword.new.cs' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'identifier.cs' },
+		{ startIndex: 45, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 47, type: 'delimiter.cs' }
+	]}, {
+	line: '			float load= 3.2e02f;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.float.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.cs' },
+		{ startIndex: 13, type: 'delimiter.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'number.float.cs' },
+		{ startIndex: 22, type: 'delimiter.cs' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '			si.FileName = @"tools\\\\node.exe";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 5, type: 'delimiter.cs' },
+		{ startIndex: 6, type: 'identifier.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.cs' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'string.quote.cs' },
+		{ startIndex: 19, type: 'string.cs' },
+		{ startIndex: 34, type: 'string.quote.cs' },
+		{ startIndex: 35, type: 'delimiter.cs' }
+	]}, {
+	line: '			si.Arguments = "tools\\\\simpl3server.js";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 5, type: 'delimiter.cs' },
+		{ startIndex: 6, type: 'identifier.cs' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'delimiter.cs' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'string.quote.cs' },
+		{ startIndex: 19, type: 'string.cs' },
+		{ startIndex: 24, type: 'string.escape.cs' },
+		{ startIndex: 26, type: 'string.cs' },
+		{ startIndex: 41, type: 'string.quote.cs' },
+		{ startIndex: 42, type: 'delimiter.cs' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '			string someString = $"hello{outside+variable}the string again {{ escaped";',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.string.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cs' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.cs' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'string.quote.cs' },
+		{ startIndex: 25, type: 'string.cs' },
+		{ startIndex: 30, type: 'string.quote.cs' },
+		{ startIndex: 31, type: 'identifier.cs' },
+		{ startIndex: 38, type: 'delimiter.cs' },
+		{ startIndex: 39, type: 'identifier.cs' },
+		{ startIndex: 47, type: 'string.quote.cs' },
+		{ startIndex: 48, type: 'string.cs' },
+		{ startIndex: 65, type: 'string.escape.cs' },
+		{ startIndex: 67, type: 'string.cs' },
+		{ startIndex: 75, type: 'string.quote.cs' },
+		{ startIndex: 76, type: 'delimiter.cs' }
+	]}, {
+	line: '			var @string = 5;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.var.cs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.cs' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'number.cs' },
+		{ startIndex: 18, type: 'delimiter.cs' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '			if (x == 4)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.if.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 7, type: 'identifier.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.cs' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'number.cs' },
+		{ startIndex: 13, type: 'delimiter.parenthesis.cs' }
+	]}, {
+	line: '			{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '				for (int i = 4; i<10; i++)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.for.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 9, type: 'keyword.int.cs' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.cs' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'number.cs' },
+		{ startIndex: 18, type: 'delimiter.cs' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.cs' },
+		{ startIndex: 21, type: 'delimiter.angle.cs' },
+		{ startIndex: 22, type: 'number.cs' },
+		{ startIndex: 24, type: 'delimiter.cs' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'identifier.cs' },
+		{ startIndex: 27, type: 'delimiter.cs' },
+		{ startIndex: 29, type: 'delimiter.parenthesis.cs' }
+	]}, {
+	line: '				{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '					var d = i;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'keyword.var.cs' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.cs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.cs' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.cs' },
+		{ startIndex: 14, type: 'delimiter.cs' }
+	]}, {
+	line: '				}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.else.cs' }
+	]}, {
+	line: '			{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '				return;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.return.cs' },
+		{ startIndex: 10, type: 'delimiter.cs' }
+	]}, {
+	line: '			}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '			',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '			Process.Start(si);',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'identifier.cs' },
+		{ startIndex: 10, type: 'delimiter.cs' },
+		{ startIndex: 11, type: 'identifier.cs' },
+		{ startIndex: 16, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 17, type: 'identifier.cs' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 20, type: 'delimiter.cs' }
+	]}, {
+	line: '		}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '#pragma region /MapLayer/*Image* /// ',
+	tokens: [
+		{ startIndex: 0, type: 'namespace.cpp.cs' }
+	]}, {
+	line: 'namespace ShouldNotBeAComment {}',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'namespace.cs' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'delimiter.curly.cs' }
+	]}, {
+	line: '#pragma endregion Region_1',
+	tokens: [
+		{ startIndex: 0, type: 'namespace.cpp.cs' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'namespace VS { class Program { static void Main(string[] args) {} } }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'namespace.cs' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'delimiter.curly.cs' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'keyword.class.cs' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'identifier.cs' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'delimiter.curly.cs' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'keyword.static.cs' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'keyword.void.cs' },
+		{ startIndex: 42, type: '' },
+		{ startIndex: 43, type: 'identifier.cs' },
+		{ startIndex: 47, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 48, type: 'keyword.string.cs' },
+		{ startIndex: 54, type: 'delimiter.square.cs' },
+		{ startIndex: 56, type: '' },
+		{ startIndex: 57, type: 'identifier.cs' },
+		{ startIndex: 61, type: 'delimiter.parenthesis.cs' },
+		{ startIndex: 62, type: '' },
+		{ startIndex: 63, type: 'delimiter.curly.cs' },
+		{ startIndex: 65, type: '' },
+		{ startIndex: 66, type: 'delimiter.curly.cs' },
+		{ startIndex: 67, type: '' },
+		{ startIndex: 68, type: 'delimiter.curly.cs' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.cs' },
+		{ startIndex: 1, type: 'identifier.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.cs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.cs' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cs' },
+		{ startIndex: 5, type: 'delimiter.cs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: 'var x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.var.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.cs' },
+		{ startIndex: 9, type: 'delimiter.cs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.cs' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.cs' }
+	]}],
+
+	[{
+	line: 'var x = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.var.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cs' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.cs' },
+		{ startIndex: 32, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'var x = /* comment */ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.var.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.cs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.cs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.cs' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'number.cs' },
+		{ startIndex: 23, type: 'delimiter.cs' },
+		{ startIndex: 24, type: '' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cs' },
+		{ startIndex: 8, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.cs' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' }
+	]}],
+
+	[{
+	line: '0x',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: 'identifier.cs' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.cs' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '23.5d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72E3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '1.72e3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.cs' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: 'delimiter.cs' },
+		{ startIndex: 2, type: 'number.cs' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 3, type: 'delimiter.cs' },
+		{ startIndex: 4, type: 'number.cs' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.cs' }
+	]}],
+
+	// Strings
+	[{
+	line: 'x = "string";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 5, type: 'string.cs' },
+		{ startIndex: 11, type: 'string.quote.cs' },
+		{ startIndex: 12, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = "stri\\"ng";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 5, type: 'string.cs' },
+		{ startIndex: 9, type: 'string.escape.cs' },
+		{ startIndex: 11, type: 'string.cs' },
+		{ startIndex: 13, type: 'string.quote.cs' },
+		{ startIndex: 14, type: 'delimiter.cs' }
+	]}],
+
+	// Verbatim Strings
+	[{
+	line: 'x = @"verbatimstring";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' },
+		{ startIndex: 20, type: 'string.quote.cs' },
+		{ startIndex: 21, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = @"verbatim""string";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' },
+		{ startIndex: 14, type: 'string.escape.cs' },
+		{ startIndex: 16, type: 'string.cs' },
+		{ startIndex: 22, type: 'string.quote.cs' },
+		{ startIndex: 23, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = @"verbatim\\string\\";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' },
+		{ startIndex: 22, type: 'string.quote.cs' },
+		{ startIndex: 23, type: 'delimiter.cs' }
+	]}],
+
+	[{
+	line: 'x = @"verbatim',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.cs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.cs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'string.quote.cs' },
+		{ startIndex: 6, type: 'string.cs' }
+	]},{
+	line: 'string";',
+	tokens: [
+		{ startIndex: 0, type: 'string.cs' },
+		{ startIndex: 6, type: 'string.quote.cs' },
+		{ startIndex: 7, type: 'delimiter.cs' }
+	]}]
+]);

test/dockerfile.test.ts

@@ -0,0 +1,189 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('dockerfile', [
+	// All
+	[{
+	line: 'FROM mono:3.12',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 4, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ENV KRE_FEED https://www.myget.org/F/aspnetvnext/api/v2',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.dockerfile' },
+		{ startIndex: 12, type: '' }
+	]}, {
+	line: 'ENV KRE_USER_HOME /opt/kre',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.dockerfile' },
+		{ startIndex: 17, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'RUN apt-get -qq update && apt-get -qqy install unzip ',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ONBUILD RUN curl -sSL https://raw.githubusercontent.com/aspnet/Home/dev/kvminstall.sh | sh',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'keyword.dockerfile' },
+		{ startIndex: 11, type: '' }
+	]}, {
+	line: 'ONBUILD RUN bash -c "source $KRE_USER_HOME/kvm/kvm.sh \\',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'keyword.dockerfile' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 20, type: 'string.dockerfile' },
+		{ startIndex: 28, type: 'variable.dockerfile' },
+		{ startIndex: 42, type: 'string.dockerfile' }
+	]}, {
+	line: '    && kvm install latest -a default \\',
+	tokens: [
+		{ startIndex: 0, type: 'string.dockerfile' }
+	]}, {
+	line: '    && kvm alias default | xargs -i ln -s $KRE_USER_HOME/packages/{} $KRE_USER_HOME/packages/default"',
+	tokens: [
+		{ startIndex: 0, type: 'string.dockerfile' },
+		{ startIndex: 42, type: 'variable.dockerfile' },
+		{ startIndex: 56, type: 'string.dockerfile' },
+		{ startIndex: 69, type: 'variable.dockerfile' },
+		{ startIndex: 83, type: 'string.dockerfile' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Install libuv for Kestrel from source code (binary is not in wheezy and one in jessie is still too old)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.dockerfile' }
+	]}, {
+	line: 'RUN apt-get -qqy install \\',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: '    autoconf \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    automake \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    build-essential \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    libtool ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: 'RUN LIBUV_VERSION=1.0.0-rc2 \\',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: '    && curl -sSL https://github.com/joyent/libuv/archive/v${LIBUV_VERSION}.tar.gz | tar zxfv - -C /usr/local/src \\',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 58, type: 'variable.dockerfile' },
+		{ startIndex: 74, type: '' }
+	]}, {
+	line: '    && cd /usr/local/src/libuv-$LIBUV_VERSION \\',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 31, type: 'variable.dockerfile' },
+		{ startIndex: 45, type: '' }
+	]}, {
+	line: '    && sh autogen.sh && ./configure && make && make install \\',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '    && rm -rf /usr/local/src/libuv-$LIBUV_VERSION \\',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 35, type: 'variable.dockerfile' },
+		{ startIndex: 49, type: '' }
+	]}, {
+	line: '    && ldconfig',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: 'ENV PATH $PATH:$KRE_USER_HOME/packages/default/bin',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'variable.dockerfile' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'variable.dockerfile' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'variable.dockerfile' },
+		{ startIndex: 29, type: '' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '# Extra things to test',
+	tokens: [
+		{ startIndex: 0, type: 'comment.dockerfile' }
+	]}, {
+	line: 'RUN echo "string at end"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 9, type: 'string.dockerfile' }
+	]}, {
+	line: 'RUN echo must work \'some str\' and some more',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 19, type: 'string.dockerfile' },
+		{ startIndex: 29, type: '' }
+	]}, {
+	line: 'RUN echo hi this is # not a comment',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' }
+	]}, {
+	line: 'RUN echo \'String with ${VAR} and another $one here\'',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dockerfile' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 9, type: 'string.dockerfile' },
+		{ startIndex: 22, type: 'variable.dockerfile' },
+		{ startIndex: 28, type: 'string.dockerfile' },
+		{ startIndex: 41, type: 'variable.dockerfile' },
+		{ startIndex: 45, type: 'string.dockerfile' }
+	]}]
+]);

test/fsharp.test.ts

@@ -0,0 +1,392 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('fsharp', [
+	// comments - single line
+	[{
+	line: '// one line comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.fs' },
+		{ startIndex: 1, type: 'identifier.fs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.fs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: '(/*almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.parenthesis.fs' },
+		{ startIndex: 1, type: 'delimiter.fs' },
+		{ startIndex: 3, type: 'identifier.fs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.fs' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: '1 / 2; (* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.fs' },
+		{ startIndex: 5, type: 'delimiter.fs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: 'let x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.let.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.fs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.fs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.fs' },
+		{ startIndex: 9, type: 'delimiter.fs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.fs' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'namespace Application1',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.namespace.fs' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: 'type MyType',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.type.fs' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.fs' }
+	]}],
+
+	[{
+	line: 'module App =',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.module.fs' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.fs' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.fs' }
+	]}],
+
+	[{
+	line: 'let AppName = "App1"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.let.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.fs' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.fs' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'string.fs' }
+	]}],
+
+	// Comments - range comment
+	[{
+	line: '(* a simple comment *)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: 'let x = (* a simple comment *) 1',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.let.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.fs' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.fs' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.fs' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.fs' }
+	]}],
+
+	[{
+	line: 'x = (**)',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.fs' }
+	]}],
+
+	[{
+	line: 'x = (*)',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.fs' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.fs' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5M',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '23.5m',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3M',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72E3m',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3M',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '1.72e3m',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 1, type: 'delimiter.fs' },
+		{ startIndex: 2, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 3, type: 'delimiter.fs' },
+		{ startIndex: 4, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.fs' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0b00000101',
+	tokens: [
+		{ startIndex: 0, type: 'number.bin.fs' }
+	]}],
+
+	[{
+	line: '86y',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0b00000101y',
+	tokens: [
+		{ startIndex: 0, type: 'number.bin.fs' }
+	]}],
+
+	[{
+	line: '86s',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86us',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86l',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86u',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86ul',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0x00002D3Fn',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.fs' }
+	]}],
+
+	[{
+	line: '0x00002D3Fun',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.fs' }
+	]}],
+
+	[{
+	line: '86L',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '86UL',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '9999999999999999999999999999I',
+	tokens: [
+		{ startIndex: 0, type: 'number.fs' }
+	]}],
+
+	[{
+	line: '0x00002D3FLF',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.fs' }
+	]}]
+]);

test/jade.test.ts

@@ -0,0 +1,380 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('jade', [
+	// Tags [Jade]
+	[{
+	line: 'p 5',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 1, type: '' }
+	]}],
+
+	[{
+	line: 'div#container.stuff',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: 'tag.id.jade' },
+		{ startIndex: 13, type: 'tag.class.jade' }
+	]}],
+
+	[{
+	line: 'div.container#stuff',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: 'tag.class.jade' },
+		{ startIndex: 13, type: 'tag.id.jade' }
+	]}],
+
+	[{
+	line: 'div.container#stuff .container',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: 'tag.class.jade' },
+		{ startIndex: 13, type: 'tag.id.jade' },
+		{ startIndex: 19, type: '' }
+	]}],
+
+	[{
+	line: '#tag-id-1',
+	tokens: [
+		{ startIndex: 0, type: 'tag.id.jade' }
+	]}],
+
+	[{
+	line: '.tag-id-1',
+	tokens: [
+		{ startIndex: 0, type: 'tag.class.jade' }
+	]}],
+
+	// Attributes - Single Line [Jade]
+	[{
+	line: 'input(type="checkbox")',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' },
+		{ startIndex: 21, type: 'delimiter.parenthesis.jade' }
+	]}],
+
+	[{
+	line: 'input (type="checkbox")',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: '' }
+	]}],
+
+	[{
+	line: 'input(type="checkbox",name="agreement",checked)',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' },
+		{ startIndex: 21, type: 'attribute.delimiter.jade' },
+		{ startIndex: 22, type: 'attribute.name.jade' },
+		{ startIndex: 26, type: 'delimiter.jade' },
+		{ startIndex: 27, type: 'attribute.value.jade' },
+		{ startIndex: 38, type: 'attribute.delimiter.jade' },
+		{ startIndex: 39, type: 'attribute.name.jade' },
+		{ startIndex: 46, type: 'delimiter.parenthesis.jade' }
+	]}],
+
+	[{
+	line: 'input(type="checkbox"',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' }
+	]}, {
+	line: 'name="agreement"',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' },
+		{ startIndex: 4, type: 'delimiter.jade' },
+		{ startIndex: 5, type: 'attribute.value.jade' }
+	]}, {
+	line: 'checked)',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: 'body',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' }
+	]}],
+
+	// Attributes - MultiLine [Jade]
+	[{
+	line: 'input(type="checkbox"',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 6, type: 'attribute.name.jade' },
+		{ startIndex: 10, type: 'delimiter.jade' },
+		{ startIndex: 11, type: 'attribute.value.jade' }
+	]}, {
+	line: 'disabled',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' }
+	]}, {
+	line: 'checked)',
+	tokens: [
+		{ startIndex: 0, type: 'attribute.name.jade' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: 'body',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' }
+	]}],
+
+	// Interpolation [Jade]
+	[{
+	line: 'p print #{count} lines',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 8, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 10, type: 'interpolation.jade' },
+		{ startIndex: 15, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 16, type: '' }
+	]}],
+
+	[{
+	line: 'p print "#{count}" lines',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 9, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 11, type: 'interpolation.jade' },
+		{ startIndex: 16, type: 'interpolation.delimiter.jade' },
+		{ startIndex: 17, type: '' }
+	]}],
+
+	[{
+	line: '{ key: 123 }',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.jade' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 5, type: 'delimiter.jade' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'number.jade' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.curly.jade' }
+	]}],
+
+	// Comments - Single Line [Jade]
+	[{
+	line: '// html#id1.class1',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}],
+
+	[{
+	line: 'body hello // not a comment 123',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 4, type: '' }
+	]}],
+
+	// Comments - MultiLine [Jade]
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '    should be a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '    should still be a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: 'div should not be a comment',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 3, type: '' }
+	]}],
+
+	// Code [Jade]
+	[{
+	line: '- var a = 1',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.var.jade' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 8, type: 'delimiter.jade' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'number.jade' }
+	]}],
+
+	[{
+	line: 'each item in items',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.each.jade' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 10, type: 'keyword.in.jade' },
+		{ startIndex: 12, type: '' }
+	]}],
+
+	[{
+	line: '- var html = "<script></script>"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.var.jade' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 11, type: 'delimiter.jade' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'string.jade' }
+	]}],
+
+	// Generated from sample
+	[{
+	line: 'doctype 5',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.doctype.jade' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.jade' }
+	]}, {
+	line: 'html(lang="en")',
+	tokens: [
+		{ startIndex: 0, type: 'tag.jade' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 5, type: 'attribute.name.jade' },
+		{ startIndex: 9, type: 'delimiter.jade' },
+		{ startIndex: 10, type: 'attribute.value.jade' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: '    head',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'tag.jade' }
+	]}, {
+	line: '        title= pageTitle',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '        script(type=\'text/javascript\')',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 14, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 15, type: 'attribute.name.jade' },
+		{ startIndex: 19, type: 'delimiter.jade' },
+		{ startIndex: 20, type: 'attribute.value.jade' },
+		{ startIndex: 37, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: '            if (foo) {',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'keyword.if.jade' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.jade' },
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'delimiter.curly.jade' }
+	]}, {
+	line: '                bar()',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.jade' }
+	]}, {
+	line: '            }',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'delimiter.curly.jade' }
+	]}, {
+	line: '    body',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'tag.jade' }
+	]}, {
+	line: '        // Disclaimer: You will need to turn insertSpaces to true in order for the',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '         syntax highlighting to kick in properly (especially for comments)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '            Enjoy :)',
+	tokens: [
+		{ startIndex: 0, type: 'comment.jade' }
+	]}, {
+	line: '        h1 Jade - node template engine if in',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 10, type: '' }
+	]}, {
+	line: '        p.',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.jade' },
+		{ startIndex: 9, type: 'delimiter.jade' }
+	]}, {
+	line: '          text ',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '            text',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '          #container',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '         #container',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '        #container',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'tag.id.jade' }
+	]}, {
+	line: '          if youAreUsingJade',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'keyword.if.jade' },
+		{ startIndex: 12, type: '' }
+	]}, {
+	line: '            p You are amazing',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'tag.jade' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '          else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 10, type: 'keyword.else.jade' }
+	]}, {
+	line: '            p Get on it!',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 12, type: 'tag.jade' },
+		{ startIndex: 13, type: '' }
+	]}, {
+	line: '     p Text can be included in a number of different ways.',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 5, type: 'tag.jade' },
+		{ startIndex: 6, type: '' }
+	]}]
+]);

test/java.test.ts

@@ -0,0 +1,605 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('java', [
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.java' }
+	]}],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[{
+	line: '/* //*/ a',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.java' },
+		{ startIndex: 1, type: 'identifier.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.java' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.java' },
+		{ startIndex: 5, type: 'delimiter.java' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.java' }
+	]}],
+
+	[{
+	line: 'int x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.java' },
+		{ startIndex: 9, type: 'delimiter.java' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.java' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: 'int x = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.java' },
+		{ startIndex: 30, type: '' },
+		{ startIndex: 31, type: 'number.java' },
+		{ startIndex: 32, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: 'int x = /* comment */ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.java' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'number.java' },
+		{ startIndex: 23, type: 'delimiter.java' },
+		{ startIndex: 24, type: '' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.java' },
+		{ startIndex: 8, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'comment.java' }
+	]}],
+
+	// Comments - range comment, multiple lines
+	[{
+	line: '/* start of multiline comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}, {
+	line: 'a comment between without a star',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}, {
+	line: 'end of multiline comment*/',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}],
+
+	[{
+	line: 'int x = /* start a comment',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.int.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.java' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'comment.java' }
+	]}, {
+	line: ' a ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' }
+	]}, {
+	line: 'and end it */ 2;',
+	tokens: [
+		{ startIndex: 0, type: 'comment.java' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'number.java' },
+		{ startIndex: 15, type: 'delimiter.java' }
+	]}],
+
+	// Java Doc, multiple lines
+	[{
+	line: '/** start of Java Doc',
+	tokens: [
+		{ startIndex: 0, type: 'comment.doc.java' }
+	]}, {
+	line: 'a comment between without a star',
+	tokens: [
+		{ startIndex: 0, type: 'comment.doc.java' }
+	]}, {
+	line: 'end of multiline comment*/',
+	tokens: [
+		{ startIndex: 0, type: 'comment.doc.java' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'package test; class Program { static void main(String[] args) {} } }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.package.java' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.java' },
+		{ startIndex: 12, type: 'delimiter.java' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.class.java' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'identifier.java' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'delimiter.curly.java' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'keyword.static.java' },
+		{ startIndex: 36, type: '' },
+		{ startIndex: 37, type: 'keyword.void.java' },
+		{ startIndex: 41, type: '' },
+		{ startIndex: 42, type: 'identifier.java' },
+		{ startIndex: 46, type: 'delimiter.parenthesis.java' },
+		{ startIndex: 47, type: 'identifier.java' },
+		{ startIndex: 53, type: 'delimiter.square.java' },
+		{ startIndex: 55, type: '' },
+		{ startIndex: 56, type: 'identifier.java' },
+		{ startIndex: 60, type: 'delimiter.parenthesis.java' },
+		{ startIndex: 61, type: '' },
+		{ startIndex: 62, type: 'delimiter.curly.java' },
+		{ startIndex: 64, type: '' },
+		{ startIndex: 65, type: 'delimiter.curly.java' },
+		{ startIndex: 66, type: '' },
+		{ startIndex: 67, type: 'delimiter.curly.java' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '0.10',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '0x',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.java' }
+	]}],
+
+	[{
+	line: '0x5_2',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.java' }
+	]}],
+
+	[{
+	line: '023L',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' }
+	]}],
+
+	[{
+	line: '0123l',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' }
+	]}],
+
+	[{
+	line: '05_2',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' }
+	]}],
+
+	[{
+	line: '0b1010_0101',
+	tokens: [
+		{ startIndex: 0, type: 'number.binary.java' }
+	]}],
+
+	[{
+	line: '0B001',
+	tokens: [
+		{ startIndex: 0, type: 'number.binary.java' }
+	]}],
+
+	[{
+	line: '10e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '10f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5e-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5E-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23.5d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72E3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72E3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72E-3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72e3D',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72e3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '1.72e-3d',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '23L',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '23l',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '0_52',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '5_2',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '5_______2',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' }
+	]}],
+
+	[{
+	line: '3_.1415F',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: 'number.float.java' }
+	]}],
+
+	[{
+	line: '3._1415F',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'delimiter.java' },
+		{ startIndex: 2, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '999_99_9999_L',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 11, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '52_',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 2, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0_x52',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0x_52',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0x52_',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.java' },
+		{ startIndex: 4, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '052_',
+	tokens: [
+		{ startIndex: 0, type: 'number.octal.java' },
+		{ startIndex: 3, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '23.5L',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.java' },
+		{ startIndex: 4, type: 'identifier.java' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: 'delimiter.java' },
+		{ startIndex: 2, type: 'number.java' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 3, type: 'delimiter.java' },
+		{ startIndex: 4, type: 'number.java' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.java' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.java' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.java' }
+	]}],
+
+	// single line Strings
+	[{
+	line: 'String s = "I\'m a Java String";',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.java' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.java' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.java' },
+		{ startIndex: 30, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: 'String s = "concatenated" + " String" ;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.java' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.java' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.java' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.java' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'delimiter.java' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'string.java' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'delimiter.java' }
+	]}],
+
+	[{
+	line: '"quote in a string"',
+	tokens: [
+		{ startIndex: 0, type: 'string.java' }
+	]}],
+
+	[{
+	line: '"escaping \\"quotes\\" is cool"',
+	tokens: [
+		{ startIndex: 0, type: 'string.java' },
+		{ startIndex: 10, type: 'string.escape.java' },
+		{ startIndex: 12, type: 'string.java' },
+		{ startIndex: 18, type: 'string.escape.java' },
+		{ startIndex: 20, type: 'string.java' }
+	]}],
+
+	[{
+	line: '"\\"',
+	tokens: [
+		{ startIndex: 0, type: 'string.invalid.java' }
+	]}],
+
+	// Annotations
+	[{
+	line: '@',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '@Override',
+	tokens: [
+		{ startIndex: 0, type: 'annotation.java' }
+	]}],
+
+	[{
+	line: '@SuppressWarnings(value = "aString")',
+	tokens: [
+		{ startIndex: 0, type: 'annotation.java' },
+		{ startIndex: 17, type: 'delimiter.parenthesis.java' },
+		{ startIndex: 18, type: 'identifier.java' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'delimiter.java' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'string.java' },
+		{ startIndex: 35, type: 'delimiter.parenthesis.java' }
+	]}],
+
+	[{
+	line: '@ AnnotationWithKeywordAfter private',
+	tokens: [
+		{ startIndex: 0, type: 'annotation.java' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'keyword.private.java' }
+	]}]
+]);
+

test/lua.test.ts

@@ -0,0 +1,76 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('lua', [
+
+	// Keywords
+	[{
+	line: 'local x, y = 1, 10',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.local.lua' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'identifier.lua' },
+		{ startIndex: 7, type: 'delimiter.lua' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'identifier.lua' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'delimiter.lua' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'number.lua' },
+		{ startIndex: 14, type: 'delimiter.lua' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'number.lua' }
+	]}],
+
+	[{
+	line: 'foo = "Hello" .. "World"; local foo = foo',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.lua' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.lua' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'string.lua' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'delimiter.lua' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'string.lua' },
+		{ startIndex: 24, type: 'delimiter.lua' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'keyword.local.lua' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 32, type: 'identifier.lua' },
+		{ startIndex: 35, type: '' },
+		{ startIndex: 36, type: 'delimiter.lua' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'identifier.lua' }
+	]}],
+
+	// Comments
+	[{
+	line: '--[[ text ]] x',
+	tokens: [
+		{ startIndex: 0, type: 'comment.lua' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'identifier.lua' }
+	]}],
+
+	[{
+	line: '--[===[ text ]===] x',
+	tokens: [
+		{ startIndex: 0, type: 'comment.lua' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'identifier.lua' }
+	]}],
+
+	[{
+	line: '--[===[ text ]==] x',
+	tokens: [
+		{ startIndex: 0, type: 'comment.lua' }
+	]}]
+]);

test/mocha.d.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+declare function run(): void;
+
+declare function suite(name: string, fn: (err?)=>void);
+declare function test(name: string, fn: (done?: (err?)=>void)=>void);
+declare function suiteSetup(fn: (done?: (err?)=>void)=>void);
+declare function suiteTeardown(fn: (done?: (err?)=>void)=>void);
+declare function setup(fn: (done?: (err?)=>void)=>void);
+declare function teardown(fn: (done?: (err?)=>void)=>void);

test/mocha.opts

@@ -0,0 +1,3 @@
+--delay
+--ui tdd
+test/all.js

test/objective-c.test.ts

@@ -0,0 +1,291 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('objective-c', [
+	// Keywords
+	[{
+	line: '-(id) initWithParams:(id<anObject>) aHandler withDeviceStateManager:(id<anotherObject>) deviceStateManager',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 2, type: 'keyword.objective-c' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'identifier.objective-c' },
+		{ startIndex: 20, type: 'delimiter.objective-c' },
+		{ startIndex: 21, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 22, type: 'keyword.objective-c' },
+		{ startIndex: 24, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 25, type: 'identifier.objective-c' },
+		{ startIndex: 33, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 34, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 35, type: 'white.objective-c' },
+		{ startIndex: 36, type: 'identifier.objective-c' },
+		{ startIndex: 44, type: 'white.objective-c' },
+		{ startIndex: 45, type: 'identifier.objective-c' },
+		{ startIndex: 67, type: 'delimiter.objective-c' },
+		{ startIndex: 68, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 69, type: 'keyword.objective-c' },
+		{ startIndex: 71, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 72, type: 'identifier.objective-c' },
+		{ startIndex: 85, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 86, type: 'delimiter.parenthesis.objective-c' },
+		{ startIndex: 87, type: 'white.objective-c' },
+		{ startIndex: 88, type: 'identifier.objective-c' }
+	]}],
+
+	// Comments - single line
+	[{
+	line: '//',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '    // a comment',
+	tokens: [
+		{ startIndex: 0, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '// a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '//sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: '/almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'operator.objective-c' },
+		{ startIndex: 1, type: 'identifier.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'identifier.objective-c' },
+		{ startIndex: 9, type: 'white.objective-c' },
+		{ startIndex: 10, type: 'identifier.objective-c' }
+	]}],
+
+	[{
+	line: '1 / 2; /* comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'number.objective-c' },
+		{ startIndex: 5, type: 'delimiter.objective-c' },
+		{ startIndex: 6, type: 'white.objective-c' },
+		{ startIndex: 7, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: 'int x = 1; // my comment // is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'identifier.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'operator.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'number.objective-c' },
+		{ startIndex: 9, type: 'delimiter.objective-c' },
+		{ startIndex: 10, type: 'white.objective-c' },
+		{ startIndex: 11, type: 'comment.objective-c' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '/* a simple comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.objective-c' }
+	]}],
+
+	[{
+	line: 'int x = /* embedded comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'identifier.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'operator.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'comment.objective-c' },
+		{ startIndex: 30, type: 'white.objective-c' },
+		{ startIndex: 31, type: 'number.objective-c' },
+		{ startIndex: 32, type: 'delimiter.objective-c' }
+	]}],
+
+	[{
+	line: 'int x = /* comment and syntax error*/ 1; */',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'identifier.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' },
+		{ startIndex: 6, type: 'operator.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'comment.objective-c' },
+		{ startIndex: 37, type: 'white.objective-c' },
+		{ startIndex: 38, type: 'number.objective-c' },
+		{ startIndex: 39, type: 'delimiter.objective-c' },
+		{ startIndex: 40, type: 'white.objective-c' },
+		{ startIndex: 41, type: 'operator.objective-c' }
+	]}],
+
+	[{
+	line: 'x = /**/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'comment.objective-c' },
+		{ startIndex: 8, type: 'delimiter.objective-c' }
+	]}],
+
+	[{
+	line: 'x = /*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'comment.objective-c' }
+	]}],
+
+	// Non-Alpha Keywords
+	[{
+	line: '#import <GTLT.h>',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' },
+		{ startIndex: 8, type: 'delimiter.angle.objective-c' },
+		{ startIndex: 9, type: 'identifier.objective-c' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'identifier.objective-c' },
+		{ startIndex: 15, type: 'delimiter.angle.objective-c' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '0x ',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.objective-c' },
+		{ startIndex: 2, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '0x123 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 4, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5e3 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 6, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5E3 ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 6, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5F ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '23.5f ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 5, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72E3F ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72E3f ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72e3F ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '1.72e3f ',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.objective-c' },
+		{ startIndex: 7, type: 'white.objective-c' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'operator.objective-c' },
+		{ startIndex: 2, type: 'number.objective-c' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 3, type: 'operator.objective-c' },
+		{ startIndex: 4, type: 'number.objective-c' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.objective-c' },
+		{ startIndex: 1, type: 'white.objective-c' },
+		{ startIndex: 2, type: 'operator.objective-c' },
+		{ startIndex: 3, type: 'white.objective-c' },
+		{ startIndex: 4, type: 'number.objective-c' }
+	]}]
+]);

test/powershell.test.ts

@@ -0,0 +1,737 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('powershell', [
+	// Comments - single line
+	[{
+	line: '#',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '    # a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '# a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '#sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '##still a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '1 / 2 /# comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.ps1' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.ps1' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.ps1' },
+		{ startIndex: 7, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '$x = 1 # my comment # is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'comment.ps1' }
+	]}],
+
+	// Comments - range comment, single line
+	[{
+	line: '<# a simple comment #>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '$x = <# a simple comment #> 1',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '$yy = <# comment #> 14',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.ps1' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'comment.ps1' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '$x = <##>7',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' },
+		{ startIndex: 9, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '$x = <#<85',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' }
+	]}],
+
+	// Comments - range comment, multiple lines
+	[{
+	line: '<# start of multiline comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'a comment between',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'end of multiline comment#>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}],
+
+	[{
+	line: '$x = <# start a comment',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'comment.ps1' }
+	]}, {
+	line: ' a ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'and end it #> 2',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'number.ps1' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'foreach($i in $b) {if (7) continue}',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.foreach.ps1' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 8, type: 'variable.ps1' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.in.ps1' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'variable.ps1' },
+		{ startIndex: 16, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'delimiter.curly.ps1' },
+		{ startIndex: 19, type: 'keyword.if.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 23, type: 'number.ps1' },
+		{ startIndex: 24, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'keyword.continue.ps1' },
+		{ startIndex: 34, type: 'delimiter.curly.ps1' }
+	]}],
+
+	// Redirect operand
+	[{
+	line: '$i > output1.txt',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 12, type: 'delimiter.ps1' },
+		{ startIndex: 13, type: '' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '0.10',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '0X123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.ps1' }
+	]}],
+
+	[{
+	line: '0x123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.ps1' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5e-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5E-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.ps1' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 1, type: 'delimiter.ps1' },
+		{ startIndex: 2, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: 'number.ps1' }
+	]}],
+
+	[{
+	line: '10 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.ps1' }
+	]}],
+
+	// Strings
+	[{
+	line: '$s = "I am a String"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '\'I am also a ( String\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '$s = "concatenated" + " String"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'delimiter.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '"escaping `"quotes`" is cool"',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 10, type: 'string.escape.ps1' },
+		{ startIndex: 12, type: 'string.ps1' },
+		{ startIndex: 18, type: 'string.escape.ps1' },
+		{ startIndex: 20, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '\'`\'end of the string',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 1, type: 'string.escape.ps1' },
+		{ startIndex: 3, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '@"I am an expandable String"@',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '@\'I am also an expandable String\'@',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '$s = @\'I am also an expandable String\'@',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '$s = @\'I am also an expandable String\'@+7',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'delimiter.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'string.ps1' }
+	]}],
+
+	[{
+	line: '@\'I am a multiline string,',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: 'and this is the middle line,',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: 'and this is NOT the end of the string\'@foreach $i',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: '\'@',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: '${script:foo}',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' }
+	]}, {
+	line: 'foreach $i',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.foreach.ps1' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'variable.ps1' }
+	]}],
+
+	// Generated from sample
+	[{
+	line: '$SelectedObjectNames=@();',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 20, type: 'delimiter.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 24, type: 'delimiter.ps1' }
+	]}, {
+	line: '$XenCenterNodeSelected = 0;',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'number.ps1' },
+		{ startIndex: 26, type: 'delimiter.ps1' }
+	]}, {
+	line: '#the object info array contains hashmaps, each of which represent a parameter set and describe a target in the XenCenter resource list',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: 'foreach($parameterSet in $ObjInfoArray)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.foreach.ps1' },
+		{ startIndex: 7, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 8, type: 'variable.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'keyword.in.ps1' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'variable.ps1' },
+		{ startIndex: 38, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '{',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '	if ($parameterSet["class"] -eq "blank")',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.if.ps1' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 5, type: 'variable.ps1' },
+		{ startIndex: 18, type: 'delimiter.square.ps1' },
+		{ startIndex: 19, type: 'string.ps1' },
+		{ startIndex: 26, type: 'delimiter.square.ps1' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'delimiter.ps1' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 32, type: 'string.ps1' },
+		{ startIndex: 39, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		#When the XenCenter node is selected a parameter set is created for each of your connected servers with the class and objUuid keys marked as blank',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '		if ($XenCenterNodeSelected)',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'keyword.if.ps1' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 6, type: 'variable.ps1' },
+		{ startIndex: 28, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '		{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '			continue',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 3, type: 'keyword.continue.ps1' }
+	]}, {
+	line: '		}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		$XenCenterNodeSelected = 1;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'delimiter.ps1' },
+		{ startIndex: 26, type: '' },
+		{ startIndex: 27, type: 'number.ps1' },
+		{ startIndex: 28, type: 'delimiter.ps1' }
+	]}, {
+	line: '		$SelectedObjectNames += "XenCenter"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'string.ps1' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '	elseif ($parameterSet["sessionRef"] -eq "null")',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.elseif.ps1' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.ps1' },
+		{ startIndex: 9, type: 'variable.ps1' },
+		{ startIndex: 22, type: 'delimiter.square.ps1' },
+		{ startIndex: 23, type: 'string.ps1' },
+		{ startIndex: 35, type: 'delimiter.square.ps1' },
+		{ startIndex: 36, type: '' },
+		{ startIndex: 37, type: 'delimiter.ps1' },
+		{ startIndex: 38, type: '' },
+		{ startIndex: 41, type: 'string.ps1' },
+		{ startIndex: 47, type: 'delimiter.parenthesis.ps1' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		#When a disconnected server is selected there is no session information, we get null for everything except class',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '	}',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		$SelectedObjectNames += "a disconnected server"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'string.ps1' }
+	]}, {
+	line: '	else',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'keyword.else.ps1' }
+	]}, {
+	line: '	{',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '		Connect-XenServer -url $parameterSet["url"] -opaqueref $parameterSet["sessionRef"]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 20, type: 'delimiter.ps1' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 25, type: 'variable.ps1' },
+		{ startIndex: 38, type: 'delimiter.square.ps1' },
+		{ startIndex: 39, type: 'string.ps1' },
+		{ startIndex: 44, type: 'delimiter.square.ps1' },
+		{ startIndex: 45, type: '' },
+		{ startIndex: 46, type: 'delimiter.ps1' },
+		{ startIndex: 47, type: '' },
+		{ startIndex: 57, type: 'variable.ps1' },
+		{ startIndex: 70, type: 'delimiter.square.ps1' },
+		{ startIndex: 71, type: 'string.ps1' },
+		{ startIndex: 83, type: 'delimiter.square.ps1' }
+	]}, {
+	line: '		#Use $class to determine which server objects to get',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '		#-properties allows us to filter the results to just include the selected object',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'comment.ps1' }
+	]}, {
+	line: '		$exp = "Get-XenServer:{0} -properties @{{uuid=\'{1}\'}}" -f $parameterSet["class"], $parameterSet["objUuid"]',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.ps1' },
+		{ startIndex: 56, type: '' },
+		{ startIndex: 57, type: 'delimiter.ps1' },
+		{ startIndex: 58, type: '' },
+		{ startIndex: 60, type: 'variable.ps1' },
+		{ startIndex: 73, type: 'delimiter.square.ps1' },
+		{ startIndex: 74, type: 'string.ps1' },
+		{ startIndex: 81, type: 'delimiter.square.ps1' },
+		{ startIndex: 82, type: 'delimiter.ps1' },
+		{ startIndex: 83, type: '' },
+		{ startIndex: 84, type: 'variable.ps1' },
+		{ startIndex: 97, type: 'delimiter.square.ps1' },
+		{ startIndex: 98, type: 'string.ps1' },
+		{ startIndex: 107, type: 'delimiter.square.ps1' }
+	]}, {
+	line: '		$obj = Invoke-Expression $exp',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 27, type: 'variable.ps1' }
+	]}, {
+	line: '		$SelectedObjectNames += $obj.name_label;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'variable.ps1' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'delimiter.ps1' },
+		{ startIndex: 25, type: '' },
+		{ startIndex: 26, type: 'variable.ps1' },
+		{ startIndex: 30, type: 'delimiter.ps1' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 41, type: 'delimiter.ps1' }
+	]}, {
+	line: '	} ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.curly.ps1' },
+		{ startIndex: 2, type: '' }
+	]}, {
+	line: '}',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.curly.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '$test = "in string var$test"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.ps1' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'string.ps1' },
+		{ startIndex: 22, type: 'variable.ps1' },
+		{ startIndex: 27, type: 'string.ps1' }
+	]}, {
+	line: '$another = \'not a $var\'',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.ps1' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.ps1' }
+	]}, {
+	line: '$third = "a $var and not `$var string"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'delimiter.ps1' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'string.ps1' },
+		{ startIndex: 12, type: 'variable.ps1' },
+		{ startIndex: 16, type: 'string.ps1' },
+		{ startIndex: 25, type: 'string.escape.ps1' },
+		{ startIndex: 27, type: 'string.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: ':aLabel',
+	tokens: [
+		{ startIndex: 0, type: 'metatag.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '<#',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '.SYNOPSIS',
+	tokens: [
+		{ startIndex: 0, type: 'comment.keyword.synopsis.ps1' }
+	]}, {
+	line: '  some text',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '  ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '.LINK',
+	tokens: [
+		{ startIndex: 0, type: 'comment.keyword.link.ps1' }
+	]}, {
+	line: '  some more text',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '#>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '$hereString = @"',
+	tokens: [
+		{ startIndex: 0, type: 'variable.ps1' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'delimiter.ps1' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'string.ps1' }
+	]}, {
+	line: '  a string',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' }
+	]}, {
+	line: '  still "@ a string $withVar',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 20, type: 'variable.ps1' }
+	]}, {
+	line: '  still a string `$noVar',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 17, type: 'string.escape.ps1' },
+		{ startIndex: 19, type: 'string.ps1' }
+	]}, {
+	line: '',
+	tokens: [
+
+	]}, {
+	line: '"@ still a string',
+	tokens: [
+		{ startIndex: 0, type: 'string.ps1' },
+		{ startIndex: 2, type: '' }
+	]}]
+]);

test/python.test.ts

@@ -0,0 +1,105 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('python', [
+	// Keywords
+	[{
+	line: 'def func():',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.python' },
+		{ startIndex: 3, type: 'white.python' },
+		{ startIndex: 4, type: 'identifier.python' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.python' },
+		{ startIndex: 10, type: 'delimiter.python' }
+	]}],
+
+	[{
+	line: 'func(str Y3)',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.python' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.python' },
+		{ startIndex: 5, type: 'keyword.python' },
+		{ startIndex: 8, type: 'white.python' },
+		{ startIndex: 9, type: 'identifier.python' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.python' }
+	]}],
+
+	[{
+	line: '@Dec0_rator:',
+	tokens: [
+		{ startIndex: 0, type: 'tag.python' },
+		{ startIndex: 11, type: 'delimiter.python' }
+	]}],
+
+	// Comments
+	[{
+	line: ' # Comments! ## "jfkd" ',
+	tokens: [
+		{ startIndex: 0, type: 'white.python' },
+		{ startIndex: 1, type: 'comment.python' }
+	]}],
+
+	// Strings
+	[{
+	line: '\'s0\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.python' },
+		{ startIndex: 1, type: 'string.python' },
+		{ startIndex: 3, type: 'string.escape.python' }
+	]}],
+
+	[{
+	line: '"\' " "',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.python' },
+		{ startIndex: 1, type: 'string.python' },
+		{ startIndex: 3, type: 'string.escape.python' },
+		{ startIndex: 4, type: 'white.python' },
+		{ startIndex: 5, type: 'string.escape.python' }
+	]}],
+
+	[{
+	line: '\'\'\'Lots of string\'\'\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.python' }
+	]}],
+
+	[{
+	line: '"""Lots \'\'\'     \'\'\'"""',
+	tokens: [
+		{ startIndex: 0, type: 'string.python' }
+	]}],
+
+	[{
+	line: '\'\'\'Lots \'\'\'0.3e-5',
+	tokens: [
+		{ startIndex: 0, type: 'string.python' },
+		{ startIndex: 11, type: 'number.python' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0xAcBFd',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.python' }
+	]}],
+
+	[{
+	line: '0x0cH',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.python' },
+		{ startIndex: 4, type: 'identifier.python' }
+	]}],
+
+	[{
+	line: '456.7e-7j',
+	tokens: [
+		{ startIndex: 0, type: 'number.python' }
+	]}]
+]);

test/r.test.ts

@@ -0,0 +1,471 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('r', [
+	// Keywords
+	[{
+	line: 'function(a) { a }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.r' },
+		{ startIndex: 8, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 9, type: 'identifier.r' },
+		{ startIndex: 10, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 11, type: 'white.r' },
+		{ startIndex: 12, type: 'delimiter.curly.r' },
+		{ startIndex: 13, type: 'white.r' },
+		{ startIndex: 14, type: 'identifier.r' },
+		{ startIndex: 15, type: 'white.r' },
+		{ startIndex: 16, type: 'delimiter.curly.r' }
+	]}],
+
+	[{
+	line: 'while(FALSE) { break }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.r' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 6, type: 'constant.r' },
+		{ startIndex: 11, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 12, type: 'white.r' },
+		{ startIndex: 13, type: 'delimiter.curly.r' },
+		{ startIndex: 14, type: 'white.r' },
+		{ startIndex: 15, type: 'keyword.r' },
+		{ startIndex: 20, type: 'white.r' },
+		{ startIndex: 21, type: 'delimiter.curly.r' }
+	]}],
+
+	[{
+	line: 'if (a) { b } else { d }',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.r' },
+		{ startIndex: 2, type: 'white.r' },
+		{ startIndex: 3, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 4, type: 'identifier.r' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.r' },
+		{ startIndex: 6, type: 'white.r' },
+		{ startIndex: 7, type: 'delimiter.curly.r' },
+		{ startIndex: 8, type: 'white.r' },
+		{ startIndex: 9, type: 'identifier.r' },
+		{ startIndex: 10, type: 'white.r' },
+		{ startIndex: 11, type: 'delimiter.curly.r' },
+		{ startIndex: 12, type: 'white.r' },
+		{ startIndex: 13, type: 'keyword.r' },
+		{ startIndex: 17, type: 'white.r' },
+		{ startIndex: 18, type: 'delimiter.curly.r' },
+		{ startIndex: 19, type: 'white.r' },
+		{ startIndex: 20, type: 'identifier.r' },
+		{ startIndex: 21, type: 'white.r' },
+		{ startIndex: 22, type: 'delimiter.curly.r' }
+	]}],
+
+	// Identifiers
+	[{
+	line: 'a',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' }
+	]}],
+
+	// Comments
+	[{
+	line: ' # comment #',
+	tokens: [
+		{ startIndex: 0, type: 'white.r' },
+		{ startIndex: 1, type: 'comment.r' }
+	]}],
+
+	// Roxygen comments
+	[{
+	line: ' #\' @author: me ',
+	tokens: [
+		{ startIndex: 0, type: 'white.r' },
+		{ startIndex: 1, type: 'comment.doc.r' },
+		{ startIndex: 4, type: 'tag.r' },
+		{ startIndex: 11, type: 'comment.doc.r' }
+	]}],
+
+	// Strings
+	[{
+	line: '"a\\n"',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.r' },
+		{ startIndex: 1, type: 'string.r' },
+		{ startIndex: 4, type: 'string.escape.r' }
+	]}],
+
+	// '\\s' is not a special character
+	[{
+	line: '"a\\s"',
+	tokens: [
+		{ startIndex: 0, type: 'string.escape.r' },
+		{ startIndex: 1, type: 'string.r' },
+		{ startIndex: 2, type: 'error-token.r' },
+		{ startIndex: 4, type: 'string.escape.r' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-.1',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1e10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1e-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1e10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1e-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1E10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '1E-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1E10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	[{
+	line: '-1E-10',
+	tokens: [
+		{ startIndex: 0, type: 'number.r' }
+	]}],
+
+	// Operators
+	[{
+	line: 'a & b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a - b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a * b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a + b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a = b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a | b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ! b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a < b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a > b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ^ b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ~ b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a / b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a : b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a %in% b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 6, type: 'white.r' },
+		{ startIndex: 7, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a %->% b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 6, type: 'white.r' },
+		{ startIndex: 7, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a == b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a != b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a %% b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a && b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a || b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a <- b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a <<- b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 5, type: 'white.r' },
+		{ startIndex: 6, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a -> b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a ->> b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 5, type: 'white.r' },
+		{ startIndex: 6, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a $ b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 3, type: 'white.r' },
+		{ startIndex: 4, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a << b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}],
+
+	[{
+	line: 'a >> b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.r' },
+		{ startIndex: 1, type: 'white.r' },
+		{ startIndex: 2, type: 'operator.r' },
+		{ startIndex: 4, type: 'white.r' },
+		{ startIndex: 5, type: 'identifier.r' }
+	]}]
+]);

test/ruby.test.ts

@@ -0,0 +1,125 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('ruby', [
+	// Keywords
+	[{
+	line: 'class Klass def init() end',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.class.ruby' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'constructor.identifier.ruby' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.def.ruby' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.ruby' },
+		{ startIndex: 20, type: 'delimiter.parenthesis.ruby' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'keyword.def.ruby' }
+	]}],
+
+	// Single digit
+	[{
+	line: 'x == 1 ',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'operator.ruby' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'number.ruby' },
+		{ startIndex: 6, type: '' }
+	]}],
+
+	// Regex
+	[{
+	line: 'text =~ /Ruby/',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'operator.ruby' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'regexp.delim.ruby' },
+		{ startIndex: 9, type: 'regexp.ruby' },
+		{ startIndex: 13, type: 'regexp.delim.ruby' }
+	]}],
+
+	[{
+	line: 'text.sub!(/Rbuy/, "Ruby")',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 4, type: '' },
+		{ startIndex: 5, type: 'identifier.ruby' },
+		{ startIndex: 9, type: 'delimiter.parenthesis.ruby' },
+		{ startIndex: 10, type: 'regexp.delim.ruby' },
+		{ startIndex: 11, type: 'regexp.ruby' },
+		{ startIndex: 15, type: 'regexp.delim.ruby' },
+		{ startIndex: 16, type: 'delimiter.ruby' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'string.d.delim.ruby' },
+		{ startIndex: 19, type: 'string.$S2.ruby' },
+		{ startIndex: 23, type: 'string.d.delim.ruby' },
+		{ startIndex: 24, type: 'delimiter.parenthesis.ruby' }
+	]}],
+
+	// make sure that division does not match regex
+	[{
+	line: 'a / b',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'operator.ruby' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.ruby' }
+	]}],
+
+	// Heredoc
+	[{
+	line: '<<HERE',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.delimiter.ruby' }
+	]}, {
+	line: 'do some string',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.ruby' }
+	]}, {
+	line: 'HERE',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.delimiter.ruby' }
+	]}],
+
+	[{
+	line: 'x <<HERE',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: 'string.heredoc.delimiter.ruby' }
+	]}, {
+	line: 'do some string',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.ruby' }
+	]}, {
+	line: 'HERE',
+	tokens: [
+		{ startIndex: 0, type: 'string.heredoc.delimiter.ruby' }
+	]}],
+
+	[{
+	line: 'x<<HERE',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: 'operator.ruby' },
+		{ startIndex: 3, type: 'constructor.identifier.ruby' }
+	]}],
+
+	[{
+	line: 'x<<-HERE',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.ruby' },
+		{ startIndex: 1, type: 'string.heredoc.delimiter.ruby' }
+	]}]
+]);

test/sql.test.ts

@@ -0,0 +1,587 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('sql', [
+	// Comments
+	[{
+	line: '-- a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.sql' }
+	]}],
+
+	[{
+	line: '---sticky -- comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.sql' }
+	]}],
+
+	[{
+	line: '-almost a comment',
+	tokens: [
+		{ startIndex: 0, type: 'operator.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.sql' },
+		{ startIndex: 9, type: 'white.sql' },
+		{ startIndex: 10, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '/* a full line comment */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.quote.sql' },
+		{ startIndex: 2, type: 'comment.sql' },
+		{ startIndex: 23, type: 'comment.quote.sql' }
+	]}],
+
+	[{
+	line: '/* /// *** /// */',
+	tokens: [
+		{ startIndex: 0, type: 'comment.quote.sql' },
+		{ startIndex: 2, type: 'comment.sql' },
+		{ startIndex: 15, type: 'comment.quote.sql' }
+	]}],
+
+	[{
+	line: 'declare @x int = /* a simple comment */ 1;',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.sql' },
+		{ startIndex: 10, type: 'white.sql' },
+		{ startIndex: 11, type: 'keyword.sql' },
+		{ startIndex: 14, type: 'white.sql' },
+		{ startIndex: 15, type: 'operator.sql' },
+		{ startIndex: 16, type: 'white.sql' },
+		{ startIndex: 17, type: 'comment.quote.sql' },
+		{ startIndex: 19, type: 'comment.sql' },
+		{ startIndex: 37, type: 'comment.quote.sql' },
+		{ startIndex: 39, type: 'white.sql' },
+		{ startIndex: 40, type: 'number.sql' },
+		{ startIndex: 41, type: 'delimiter.sql' }
+	]}],
+
+	// Not supporting nested comments, as nested comments seem to not be standard?
+	// i.e. http://stackoverflow.com/questions/728172/are-there-multiline-comment-delimiters-in-sql-that-are-vendor-agnostic
+	[{
+	line: '@x=/* a /* nested comment  1*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 3, type: 'comment.quote.sql' },
+		{ startIndex: 5, type: 'comment.sql' },
+		{ startIndex: 28, type: 'comment.quote.sql' },
+		{ startIndex: 30, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '@x=/* another comment */ 1*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 3, type: 'comment.quote.sql' },
+		{ startIndex: 5, type: 'comment.sql' },
+		{ startIndex: 22, type: 'comment.quote.sql' },
+		{ startIndex: 24, type: 'white.sql' },
+		{ startIndex: 25, type: 'number.sql' },
+		{ startIndex: 26, type: 'operator.sql' },
+		{ startIndex: 28, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '@x=/*/;',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 3, type: 'comment.quote.sql' },
+		{ startIndex: 5, type: 'comment.sql' }
+	]}],
+
+	// Numbers
+	[{
+	line: '123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '-123',
+	tokens: [
+		{ startIndex: 0, type: 'operator.sql' },
+		{ startIndex: 1, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0xaBc123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0XaBc123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0x',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0x0',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0xAB_CD',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' },
+		{ startIndex: 4, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '$',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$-123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$-+-123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$123.5678',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$0.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$99.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$0.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '$.0',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '123',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '123.5678',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.99',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '99.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0.',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.0',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1E-2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1E+2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '0.1E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '1.E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '.1E2',
+	tokens: [
+		{ startIndex: 0, type: 'number.sql' }
+	]}],
+
+	// Identifiers
+	[{
+	line: '_abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '#abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '##abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '@abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '@@abc$01',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '$abc',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '$action',
+	tokens: [
+		{ startIndex: 0, type: 'predefined.sql' }
+	]}],
+
+	[{
+	line: '$nonexistent',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: '@@DBTS',
+	tokens: [
+		{ startIndex: 0, type: 'predefined.sql' }
+	]}],
+
+	[{
+	line: '@@nonexistent',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'declare [abc 321];',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.quote.sql' },
+		{ startIndex: 9, type: 'identifier.sql' },
+		{ startIndex: 16, type: 'identifier.quote.sql' },
+		{ startIndex: 17, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '[abc[[ 321 ]] xyz]',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 17, type: 'identifier.quote.sql' }
+	]}],
+
+	[{
+	line: '[abc',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'declare "abc 321";',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.quote.sql' },
+		{ startIndex: 9, type: 'identifier.sql' },
+		{ startIndex: 16, type: 'identifier.quote.sql' },
+		{ startIndex: 17, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '"abc"" 321 "" xyz"',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 17, type: 'identifier.quote.sql' }
+	]}],
+
+	[{
+	line: '"abc',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'int',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' }
+	]}],
+
+	[{
+	line: '[int]',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.quote.sql' },
+		{ startIndex: 1, type: 'identifier.sql' },
+		{ startIndex: 4, type: 'identifier.quote.sql' }
+	]}],
+
+	// Strings
+	[{
+	line: 'declare @x=\'a string\';',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'identifier.sql' },
+		{ startIndex: 10, type: 'operator.sql' },
+		{ startIndex: 11, type: 'string.quote.sql' },
+		{ startIndex: 12, type: 'string.sql' },
+		{ startIndex: 20, type: 'string.quote.sql' },
+		{ startIndex: 21, type: 'delimiter.sql' }
+	]}],
+
+	[{
+	line: '\'a \'\' string with quotes\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' },
+		{ startIndex: 24, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: '\'a " string with quotes\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' },
+		{ startIndex: 23, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: '\'a -- string with comment\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' },
+		{ startIndex: 25, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: 'N\'a unicode string\'',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 2, type: 'string.sql' },
+		{ startIndex: 18, type: 'string.quote.sql' }
+	]}],
+
+	[{
+	line: '\'a endless string',
+	tokens: [
+		{ startIndex: 0, type: 'string.quote.sql' },
+		{ startIndex: 1, type: 'string.sql' }
+	]}],
+
+	// Operators
+	[{
+	line: 'SET @x=@x+1',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 3, type: 'white.sql' },
+		{ startIndex: 4, type: 'identifier.sql' },
+		{ startIndex: 6, type: 'operator.sql' },
+		{ startIndex: 7, type: 'identifier.sql' },
+		{ startIndex: 9, type: 'operator.sql' },
+		{ startIndex: 10, type: 'number.sql' }
+	]}],
+
+	[{
+	line: '@x^=@x',
+	tokens: [
+		{ startIndex: 0, type: 'identifier.sql' },
+		{ startIndex: 2, type: 'operator.sql' },
+		{ startIndex: 4, type: 'identifier.sql' }
+	]}],
+
+	[{
+	line: 'WHERE x IS NOT NULL',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 5, type: 'white.sql' },
+		{ startIndex: 6, type: 'identifier.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'operator.sql' },
+		{ startIndex: 10, type: 'white.sql' },
+		{ startIndex: 11, type: 'operator.sql' },
+		{ startIndex: 14, type: 'white.sql' },
+		{ startIndex: 15, type: 'operator.sql' }
+	]}],
+
+	[{
+	line: 'SELECT * FROM dbo.MyTable WHERE MyColumn IN (1,2)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 6, type: 'white.sql' },
+		{ startIndex: 7, type: 'operator.sql' },
+		{ startIndex: 8, type: 'white.sql' },
+		{ startIndex: 9, type: 'keyword.sql' },
+		{ startIndex: 13, type: 'white.sql' },
+		{ startIndex: 14, type: 'identifier.sql' },
+		{ startIndex: 17, type: 'delimiter.sql' },
+		{ startIndex: 18, type: 'identifier.sql' },
+		{ startIndex: 25, type: 'white.sql' },
+		{ startIndex: 26, type: 'keyword.sql' },
+		{ startIndex: 31, type: 'white.sql' },
+		{ startIndex: 32, type: 'identifier.sql' },
+		{ startIndex: 40, type: 'white.sql' },
+		{ startIndex: 41, type: 'operator.sql' },
+		{ startIndex: 43, type: 'white.sql' },
+		{ startIndex: 44, type: 'delimiter.parenthesis.sql' },
+		{ startIndex: 45, type: 'number.sql' },
+		{ startIndex: 46, type: 'delimiter.sql' },
+		{ startIndex: 47, type: 'number.sql' },
+		{ startIndex: 48, type: 'delimiter.parenthesis.sql' }
+	]}],
+
+	// Scopes
+	[{
+	line: 'WHILE() BEGIN END',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 5, type: 'delimiter.parenthesis.sql' },
+		{ startIndex: 7, type: 'white.sql' },
+		{ startIndex: 8, type: 'keyword.block.sql' },
+		{ startIndex: 13, type: 'white.sql' },
+		{ startIndex: 14, type: 'keyword.block.sql' }
+	]}],
+
+	[{
+	line: 'BEGIN TRAN BEGIN TRY SELECT $ COMMIT END TRY BEGIN CATCH ROLLBACK END CATCH',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 10, type: 'white.sql' },
+		{ startIndex: 11, type: 'keyword.try.sql' },
+		{ startIndex: 20, type: 'white.sql' },
+		{ startIndex: 21, type: 'keyword.sql' },
+		{ startIndex: 27, type: 'white.sql' },
+		{ startIndex: 28, type: 'number.sql' },
+		{ startIndex: 29, type: 'white.sql' },
+		{ startIndex: 30, type: 'keyword.sql' },
+		{ startIndex: 36, type: 'white.sql' },
+		{ startIndex: 37, type: 'keyword.try.sql' },
+		{ startIndex: 44, type: 'white.sql' },
+		{ startIndex: 45, type: 'keyword.catch.sql' },
+		{ startIndex: 56, type: 'white.sql' },
+		{ startIndex: 57, type: 'keyword.sql' },
+		{ startIndex: 65, type: 'white.sql' },
+		{ startIndex: 66, type: 'keyword.catch.sql' }
+	]}],
+
+	[{
+	line: 'SELECT CASE $ WHEN 3 THEN 4 ELSE 5 END',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.sql' },
+		{ startIndex: 6, type: 'white.sql' },
+		{ startIndex: 7, type: 'keyword.block.sql' },
+		{ startIndex: 11, type: 'white.sql' },
+		{ startIndex: 12, type: 'number.sql' },
+		{ startIndex: 13, type: 'white.sql' },
+		{ startIndex: 14, type: 'keyword.choice.sql' },
+		{ startIndex: 18, type: 'white.sql' },
+		{ startIndex: 19, type: 'number.sql' },
+		{ startIndex: 20, type: 'white.sql' },
+		{ startIndex: 21, type: 'keyword.choice.sql' },
+		{ startIndex: 25, type: 'white.sql' },
+		{ startIndex: 26, type: 'number.sql' },
+		{ startIndex: 27, type: 'white.sql' },
+		{ startIndex: 28, type: 'keyword.sql' },
+		{ startIndex: 32, type: 'white.sql' },
+		{ startIndex: 33, type: 'number.sql' },
+		{ startIndex: 34, type: 'white.sql' },
+		{ startIndex: 35, type: 'keyword.block.sql' }
+	]}]
+]);

test/swift.test.ts

@@ -0,0 +1,181 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('swift', [
+
+	// Attributes
+	[{
+	line: '@noescape',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.control.swift' } /* '@noescape' */
+	]}],
+	//Keyword and Type Identifier
+	[{
+		line: 'class App: UI, UIApp, UIView {',
+		tokens: [
+		{ startIndex: 0, type: 'keyword.swift' } /* 'class' */,
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'type.identifier.swift' } /* 'App' */,
+		{ startIndex: 9, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'type.identifier.swift' } /* 'UI' */,
+		{ startIndex: 13, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'type.identifier.swift' } /* 'UIApp' */,
+		{ startIndex: 20, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'type.identifier.swift' } /* 'UIView' */,
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'delimiter.curly.swift' } /* '{' */
+	]}],
+	// Keyword, Identifier, and Type Identifier
+	[{
+		line: '    var window: UIWindow?',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.swift' } /* 'var' */,
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.swift' } /* 'window' */,
+		{ startIndex: 14, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'type.identifier.swift' } /* 'UIWindow' */,
+		{ startIndex: 24, type: 'keyword.operator.swift' } /* '?' */
+	]}],
+	//Comment
+	[{
+		line: '    // Comment',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.swift' } /* '// Comment' */
+	]}],
+	//Block Comment with Embedded Comment followed by code
+	[{
+		line: '    /* Comment //Embedded */ var y = 0b10',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.swift' }, // /* '/* Comment //Embedded */' */,
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'keyword.swift' } /* 'var' */,
+		{ startIndex: 32, type: '' },
+		{ startIndex: 33, type: 'identifier.swift' }	/* 'y' */,
+		{ startIndex: 34, type: '' },
+		{ startIndex: 35, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 36, type: '' },
+		{ startIndex: 37, type: 'number.binary.swift' } /* '0b10' */
+	]}],
+	// Method signature (broken on two lines)
+	[{
+		line: '    public func app(app: App, opts:',
+		tokens:[
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'keyword.swift' } /* 'public' */,
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'keyword.swift' } /* 'func' */,
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.swift' } /* 'app' */,
+		{ startIndex: 19, type: 'delimiter.parenthesis.swift' } /* '(' */,
+		{ startIndex: 20, type: 'identifier.swift' }/* 'app' */,
+		{ startIndex: 23, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'type.identifier.swift' } /* 'App' */,
+		{ startIndex: 28, type: 'keyword.operator.swift' } /* ',' */,
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'identifier.swift' } /* 'opts' */,
+		{ startIndex: 34, type: 'keyword.operator.swift' } /* ':' */,
+	]}],
+	// Method signature Continued
+	[{
+		line: '        [NSObject: AnyObject]?) -> Bool {',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'delimiter.square.swift' } /* '[' */,
+		{ startIndex: 9, type: 'type.identifier.swift' } /* 'NSObject' */,
+		{ startIndex: 17, type: 'keyword.operator.swift' } /* ':' */,
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'type.identifier.swift' } /* 'AnyObject' */,
+		{ startIndex: 28, type: 'delimiter.square.swift' } /* ']' */,
+		{ startIndex: 29, type: 'keyword.operator.swift' } /* '?' */,
+		{ startIndex: 30, type: 'delimiter.parenthesis.swift' } /* ')' */,
+		{ startIndex: 31, type: '' },
+		{ startIndex: 32, type: 'keyword.operator.swift' } /* '->' */,
+		{ startIndex: 34, type: '' },
+		{ startIndex: 35, type: 'type.identifier.swift' } /* 'Bool' */,
+		{ startIndex: 39, type: '' },
+		{ startIndex: 40, type: 'delimiter.curly.swift' } /* '{' */
+	]}],
+	// String with escapes
+	[{
+		line: '        var `String` = "String w/ \\"escape\\""',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'keyword.swift' } /* 'var' */,
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.operator.swift' } /* '`' */,
+		{ startIndex: 13, type: 'identifier.swift' } /* 'String' */,
+		{ startIndex: 19, type: 'keyword.operator.swift' } /* '`' */,
+		{ startIndex: 20, type: '' },
+		{ startIndex: 21, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'string.quote.swift' } /* '"' */,
+		{ startIndex: 24, type: 'string.swift' } /* 'String w/ \\"escape\\""' */,
+		{ startIndex: 44, type: 'string.quote.swift' } /* '"' */,
+	]}],
+	// String with interpolated expression
+	[{
+		line: '        let message = "\\(y) times 2.5 is \\(Double(25) * 2.5)"',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'keyword.swift' } /* 'let' */,
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.swift' } /* 'message' */,
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'string.quote.swift' } /* '"' */,
+		{ startIndex: 23, type: 'keyword.operator.swift' } /* '\(' */,
+		{ startIndex: 25, type: 'identifier.swift' },
+		{ startIndex: 26, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 27, type: 'string.swift' } /* ' times 2.5 is ' */,
+		{ startIndex: 41, type: 'keyword.operator.swift' } /* '\(' */,
+		{ startIndex: 43, type: 'type.identifier.swift' } /* 'Double' */,
+		{ startIndex: 49, type: 'keyword.operator.swift' } /* '(' */,
+		{ startIndex: 50, type: 'number.swift' } /* '25' */,
+		{ startIndex: 52, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 53, type: '' },
+		{ startIndex: 54, type: 'keyword.operator.swift' } /* '*' */,
+		{ startIndex: 55, type: '' },
+		{ startIndex: 56, type: 'number.float.swift' } /* '2.5' */,
+		{ startIndex: 59, type: 'keyword.operator.swift' } /* ')' */,
+		{ startIndex: 60, type: 'string.quote.swift' } /* '"' */
+	]}],
+	// Method invocation/property accessor.
+	[{
+		line: '        let view = self.window!.contr as! UIView',
+		tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 8, type: 'keyword.swift' } /* 'let' */,
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.swift' } /* 'view' */,
+		{ startIndex: 16, type: '' },
+		{ startIndex: 17, type: 'keyword.operator.swift' } /* '=' */,
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'keyword.swift' } /* 'self' */,
+		{ startIndex: 23, type: 'delimeter.swift' } /* '.' */,
+		{ startIndex: 24, type: 'type.identifier.swift' } /* 'window' */,
+		{ startIndex: 30, type: 'keyword.operator.swift' } /* '!' */,
+		{ startIndex: 31, type: 'delimeter.swift' } /* '.' */,
+		{ startIndex: 32, type: 'type.identifier.swift' } /* 'contr' */,
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'keyword.swift' } /* 'as' */,
+		{ startIndex: 40, type: 'keyword.operator.swift' } /* '!' */,
+		{ startIndex: 41, type: '' },
+		{ startIndex: 42, type: 'type.identifier.swift' } /* 'UIView' */
+	]}]
+]);
+

test/testRunner.ts

@@ -0,0 +1,58 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import {loadLanguage} from '../src/monaco.contribution';
+import * as assert from 'assert';
+
+// Allow for running under nodejs/requirejs in tests
+var _monaco: typeof monaco = (typeof monaco === 'undefined' ? (<any>self).monaco : monaco);
+
+export interface IRelaxedToken {
+	startIndex: number;
+	type: string;
+}
+
+export interface ITestItem {
+	line: string;
+	tokens: IRelaxedToken[];
+}
+
+export function testTokenization(languageId:string, tests:ITestItem[][]): void {
+	suite(languageId + ' tokenization', () => {
+		test('', (done) => {
+			loadLanguage(languageId).then(() => {
+				runTests(languageId, tests);
+				done();
+			}).then(null, done);
+		});
+	});
+}
+
+function runTests(languageId:string, tests:ITestItem[][]): void {
+	tests.forEach((test) => runTest(languageId, test));
+}
+
+function runTest(languageId:string, test:ITestItem[]): void {
+	let text = test.map(t => t.line).join('\n');
+	let model = _monaco.editor.createModel(text, languageId);
+
+	for (let lineNumber = 1, lineCount = model.getLineCount(); lineNumber <= lineCount; lineNumber++) {
+		let actual: IRelaxedToken[] = [];
+		let lineTokens = (<any>model).getLineTokens(lineNumber);
+		for (let j = 0; j < lineTokens.getTokenCount(); j++) {
+			actual.push({
+				startIndex: lineTokens.getTokenStartIndex(j),
+				type: lineTokens.getTokenType(j)
+			});
+		}
+
+		let expected = test[lineNumber - 1].tokens;
+		// console.log(`actual: ${JSON.stringify(actual)}`);
+		// console.log(`expected: ${JSON.stringify(expected)}`);
+		assert.deepEqual(actual, expected, 'TOKENIZING ' + text);
+	}
+
+	model.dispose();
+}

test/vb.test.ts

@@ -0,0 +1,395 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('vb', [
+
+	// Comments - single line
+	[{
+	line: '\'',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '    \' a comment',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '\' a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '\'sticky comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '1 \' 2; \' comment',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: 'Dim x = 1; \' my comment \'\' is a nice one',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.dim.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.vb' },
+		{ startIndex: 9, type: 'delimiter.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: 'REM this is a comment',
+	tokens: [
+		{ startIndex: 0, type: 'comment.vb' }
+	]}],
+
+	[{
+	line: '2 + 5 REM comment starts',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'comment.vb' }
+	]}],
+
+	// Numbers
+	[{
+	line: '0',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' }
+	]}],
+
+	[{
+	line: '0.0',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '&h123',
+	tokens: [
+		{ startIndex: 0, type: 'number.hex.vb' }
+	]}],
+
+	[{
+	line: '23.5',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5e3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5E3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5R',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '23.5r',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3#',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72E3F',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3!',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e3f',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '1.72e-3',
+	tokens: [
+		{ startIndex: 0, type: 'number.float.vb' }
+	]}],
+
+	[{
+	line: '0+0',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: 'delimiter.vb' },
+		{ startIndex: 2, type: 'number.vb' }
+	]}],
+
+	[{
+	line: '100+10',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 3, type: 'delimiter.vb' },
+		{ startIndex: 4, type: 'number.vb' }
+	]}],
+
+	[{
+	line: '0 + 0',
+	tokens: [
+		{ startIndex: 0, type: 'number.vb' },
+		{ startIndex: 1, type: '' },
+		{ startIndex: 2, type: 'delimiter.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'number.vb' }
+	]}],
+
+	// Keywords
+	[{
+	line: 'Imports Microsoft.VisualBasic',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.imports.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'identifier.vb' },
+		{ startIndex: 17, type: 'delimiter.vb' },
+		{ startIndex: 18, type: 'identifier.vb' }
+	]}],
+
+	[{
+	line: 'Private Sub Foo(ByVal sender As String)',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.private.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'keyword.tag-sub.vb' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'identifier.vb' },
+		{ startIndex: 15, type: 'delimiter.parenthesis.vb' },
+		{ startIndex: 16, type: 'keyword.byval.vb' },
+		{ startIndex: 21, type: '' },
+		{ startIndex: 22, type: 'identifier.vb' },
+		{ startIndex: 28, type: '' },
+		{ startIndex: 29, type: 'keyword.as.vb' },
+		{ startIndex: 31, type: '' },
+		{ startIndex: 32, type: 'keyword.string.vb' },
+		{ startIndex: 38, type: 'delimiter.parenthesis.vb' }
+	]}],
+
+	// Strings
+	[{
+	line: 'String s = "string"',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.string.vb' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'identifier.vb' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'delimiter.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'string.vb' }
+	]}],
+
+	[{
+	line: '"use strict";',
+	tokens: [
+		{ startIndex: 0, type: 'string.vb' },
+		{ startIndex: 12, type: 'delimiter.vb' }
+	]}],
+
+	// Tags
+	[{
+	line: 'Public Sub ToString()',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.public.vb' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'keyword.tag-sub.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'identifier.vb' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.vb' }
+	]}],
+
+	[{
+	line: 'public sub ToString()',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.public.vb' },
+		{ startIndex: 6, type: '' },
+		{ startIndex: 7, type: 'keyword.tag-sub.vb' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'identifier.vb' },
+		{ startIndex: 19, type: 'delimiter.parenthesis.vb' }
+	]}],
+
+	[{
+	line: 'While Do Continue While End While',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-while.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'keyword.tag-do.vb' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.tag-continue.vb' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'keyword.tag-while.vb' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'keyword.tag-while.vb' }
+	]}],
+
+	[{
+	line: 'While while WHILE WHile whiLe',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-while.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'keyword.tag-while.vb' },
+		{ startIndex: 11, type: '' },
+		{ startIndex: 12, type: 'keyword.tag-while.vb' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'keyword.tag-while.vb' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'keyword.tag-while.vb' }
+	]}],
+
+	[{
+	line: 'If b(i) = col Then',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-if.vb' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'identifier.vb' },
+		{ startIndex: 4, type: 'delimiter.parenthesis.vb' },
+		{ startIndex: 5, type: 'identifier.vb' },
+		{ startIndex: 6, type: 'delimiter.parenthesis.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'identifier.vb' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.then.vb' }
+	]}],
+
+	[{
+	line: 'Do stuff While True Loop',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-do.vb' },
+		{ startIndex: 2, type: '' },
+		{ startIndex: 3, type: 'identifier.vb' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'keyword.tag-while.vb' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'keyword.true.vb' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'keyword.tag-do.vb' }
+	]}],
+
+	[{
+	line: 'For i = 0 To 10 DoStuff Next',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-for.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'delimiter.vb' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'number.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'keyword.to.vb' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'number.vb' },
+		{ startIndex: 15, type: '' },
+		{ startIndex: 16, type: 'identifier.vb' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'keyword.tag-for.vb' }
+	]}],
+
+	[{
+	line: 'For stuff End For',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-for.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'keyword.end.vb' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.tag-for.vb' }
+	]}],
+
+	[{
+	line: 'For stuff end for',
+	tokens: [
+		{ startIndex: 0, type: 'keyword.tag-for.vb' },
+		{ startIndex: 3, type: '' },
+		{ startIndex: 4, type: 'identifier.vb' },
+		{ startIndex: 9, type: '' },
+		{ startIndex: 10, type: 'keyword.end.vb' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'keyword.tag-for.vb' }
+	]}]
+]);

test/xml.test.ts

@@ -0,0 +1,573 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import {testTokenization} from './testRunner';
+
+testTokenization('xml', [
+	// Complete Start Tag with Whitespace
+	[{
+	line: '<person>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 7, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<person/>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 8, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<person >',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<person />',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' },
+		{ startIndex: 7, type: '' },
+		{ startIndex: 8, type: 'tag.tag-person.xml' },
+		{ startIndex: 9, type: 'delimiter.start.xml' }
+	]}],
+
+	// Incomplete Start Tag
+	[{
+	line: '<',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '<person',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-person.xml' }
+	]}],
+
+	[{
+	line: '<input',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-input.xml' }
+	]}],
+
+	// Invalid Open Start Tag
+	[{
+	line: '< person',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: '< person>',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	[{
+	line: 'i <person;',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'tag.tag-person.xml' },
+		{ startIndex: 9, type: '' }
+	]}],
+
+	// Tag with Attribute
+	[{
+	line: '<tool name="">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 13, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name="Monaco">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name=\'Monaco\'>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Attributes
+	[{
+	line: '<tool name="Monaco" version="1.0">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'attribute.name.xml' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'attribute.value.xml' },
+		{ startIndex: 33, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Name-Only-Attribute
+	[{
+	line: '<tool name>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name version>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.name.xml' },
+		{ startIndex: 18, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Attribute And Whitespace
+	[{
+	line: '<tool name=  "monaco">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 13, type: 'attribute.value.xml' },
+		{ startIndex: 21, type: 'delimiter.start.xml' }
+	]}],
+
+	[{
+	line: '<tool name = "monaco">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 13, type: 'attribute.value.xml' },
+		{ startIndex: 21, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Invalid Attribute Name
+	[{
+	line: '<tool name!@#="bar">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 15, type: 'attribute.name.xml' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}],
+
+	// Tag with Invalid Attribute Value
+	[{
+	line: '<tool name=">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tool.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 12, type: 'delimiter.start.xml' }
+	]}],
+
+	// Complete End Tag
+	[{
+	line: '</person>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-person.xml' },
+		{ startIndex: 8, type: 'delimiter.end.xml' }
+	]}],
+
+	// Complete End Tag with Whitespace
+	[{
+	line: '</person  >',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-person.xml' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 10, type: 'delimiter.end.xml' }
+	]}],
+
+	// Incomplete End Tag
+	[{
+	line: '</person',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}],
+
+	// Comments
+	[{
+	line: '<!-- -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.xml' },
+		{ startIndex: 4, type: 'comment.content.xml' },
+		{ startIndex: 5, type: 'comment.xml' }
+	]}],
+
+	[{
+	line: '<!--a>monaco</a -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.xml' },
+		{ startIndex: 4, type: 'comment.content.xml' },
+		{ startIndex: 16, type: 'comment.xml' }
+	]}],
+
+	[{
+	line: '<!--a>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.xml' },
+		{ startIndex: 4, type: 'comment.content.xml' }
+	]},{
+	line: 'monaco ',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' }
+	]},{
+	line: 'tools</a -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' },
+		{ startIndex: 9, type: 'comment.xml' }
+	]}],
+
+	// CDATA
+	[{
+	line: '<tools><![CDATA[<person/>]]></tools>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tools.xml' },
+		{ startIndex: 6, type: 'delimiter.start.xml' },
+		{ startIndex: 7, type: 'delimiter.cdata.xml' },
+		{ startIndex: 16, type: '' },
+		{ startIndex: 25, type: 'delimiter.cdata.xml' },
+		{ startIndex: 28, type: 'delimiter.end.xml' },
+		{ startIndex: 30, type: 'tag.tag-tools.xml' },
+		{ startIndex: 35, type: 'delimiter.end.xml' }
+	]}],
+
+	[{
+	line: '<tools>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-tools.xml' },
+		{ startIndex: 6, type: 'delimiter.start.xml' }
+	]},{
+	line: '\t<![CDATA[',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.cdata.xml' }
+	]},{
+	line: '\t\t<person/>',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]},{
+	line: '\t]]>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.cdata.xml' },
+	]},{
+	line: '</tools>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-tools.xml' },
+		{ startIndex: 7, type: 'delimiter.end.xml' }
+	]}],
+
+	// Generated from sample
+	[{
+	line: '<?xml version="1.0"?>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'metatag.instruction.xml' },
+		{ startIndex: 5, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'attribute.value.xml' },
+		{ startIndex: 19, type: 'delimiter.start.xml' }
+	]}, {
+	line: '<configuration xmlns:xdt="http://schemas.microsoft.com/XML-Document-Transform">',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.start.xml' },
+		{ startIndex: 1, type: 'tag.tag-configuration.xml' },
+		{ startIndex: 14, type: '' },
+		{ startIndex: 15, type: 'attribute.name.xml' },
+		{ startIndex: 24, type: '' },
+		{ startIndex: 25, type: 'attribute.value.xml' },
+		{ startIndex: 78, type: 'delimiter.start.xml' }
+	]}, {
+	line: '  <connectionStrings>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'tag.tag-connectionstrings.xml' },
+		{ startIndex: 20, type: 'delimiter.start.xml' }
+	]}, {
+	line: '    <add name="MyDB" ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.start.xml' },
+		{ startIndex: 5, type: 'tag.tag-add.xml' },
+		{ startIndex: 8, type: '' },
+		{ startIndex: 9, type: 'attribute.name.xml' },
+		{ startIndex: 13, type: '' },
+		{ startIndex: 14, type: 'attribute.value.xml' },
+		{ startIndex: 20, type: '' }
+	]}, {
+	line: '      connectionString="value for the deployed Web.config file" ',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'attribute.value.xml' },
+		{ startIndex: 63, type: '' }
+	]}, {
+	line: '      xdt:Transform="SetAttributes" xdt:Locator="Match(name)"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 19, type: '' },
+		{ startIndex: 20, type: 'attribute.value.xml' },
+		{ startIndex: 35, type: '' },
+		{ startIndex: 36, type: 'attribute.name.xml' },
+		{ startIndex: 47, type: '' },
+		{ startIndex: 48, type: 'attribute.value.xml' },
+		{ startIndex: 61, type: 'tag.tag-add.xml' },
+		{ startIndex: 62, type: 'delimiter.start.xml' }
+	]}, {
+	line: '  </connectionStrings>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.end.xml' },
+		{ startIndex: 4, type: 'tag.tag-connectionstrings.xml' },
+		{ startIndex: 21, type: 'delimiter.end.xml' }
+	]}, {
+	line: '  <system.web>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'tag.tag-system.web.xml' },
+		{ startIndex: 13, type: 'delimiter.start.xml' }
+	]}, {
+	line: '    <customErrors defaultRedirect="GenericError.htm"',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.start.xml' },
+		{ startIndex: 5, type: 'tag.tag-customerrors.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'attribute.name.xml' },
+		{ startIndex: 33, type: '' },
+		{ startIndex: 34, type: 'attribute.value.xml' }
+	]}, {
+	line: '      mode="RemoteOnly" xdt:Transform="Replace">',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'attribute.name.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.value.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.name.xml' },
+		{ startIndex: 37, type: '' },
+		{ startIndex: 38, type: 'attribute.value.xml' },
+		{ startIndex: 47, type: 'delimiter.start.xml' }
+	]}, {
+	line: '      <error statusCode="500" redirect="InternalError.htm"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 6, type: 'delimiter.start.xml' },
+		{ startIndex: 7, type: 'tag.tag-error.xml' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'attribute.name.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.value.xml' },
+		{ startIndex: 29, type: '' },
+		{ startIndex: 30, type: 'attribute.name.xml' },
+		{ startIndex: 38, type: '' },
+		{ startIndex: 39, type: 'attribute.value.xml' },
+		{ startIndex: 58, type: 'tag.tag-error.xml' },
+		{ startIndex: 59, type: 'delimiter.start.xml' }
+	]}, {
+	line: '    </customErrors>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 4, type: 'delimiter.end.xml' },
+		{ startIndex: 6, type: 'tag.tag-customerrors.xml' },
+		{ startIndex: 18, type: 'delimiter.end.xml' }
+	]}, {
+	line: '  </system.web>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 2, type: 'delimiter.end.xml' },
+		{ startIndex: 4, type: 'tag.tag-system.web.xml' },
+		{ startIndex: 14, type: 'delimiter.end.xml' }
+	]}, {
+	line: '	',
+	tokens: [
+		{ startIndex: 0, type: '' }
+	]}, {
+	line: '	<!-- The stuff below was added for extra tokenizer testing -->',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'comment.xml' },
+		{ startIndex: 5, type: 'comment.content.xml' },
+		{ startIndex: 60, type: 'comment.xml' }
+	]}, {
+	line: '	<!-- A multi-line comment <with> </with>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'comment.xml' },
+		{ startIndex: 5, type: 'comment.content.xml' }
+	]}, {
+	line: '       <tags>',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' }
+	]}, {
+	line: '				 -->',
+	tokens: [
+		{ startIndex: 0, type: 'comment.content.xml' },
+		{ startIndex: 5, type: 'comment.xml' }
+	]}, {
+	line: '	<!DOCTYPE another meta tag>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 3, type: 'metatag.declaration.xml' },
+		{ startIndex: 10, type: '' },
+		{ startIndex: 11, type: 'attribute.name.xml' },
+		{ startIndex: 18, type: '' },
+		{ startIndex: 19, type: 'attribute.name.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.name.xml' },
+		{ startIndex: 27, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<tools><![CDATA[Some text and tags <person/>]]></tools>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-tools.xml' },
+		{ startIndex: 7, type: 'delimiter.start.xml' },
+		{ startIndex: 8, type: 'delimiter.cdata.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 45, type: 'delimiter.cdata.xml' },
+		{ startIndex: 48, type: 'delimiter.end.xml' },
+		{ startIndex: 50, type: 'tag.tag-tools.xml' },
+		{ startIndex: 55, type: 'delimiter.end.xml' }
+	]}, {
+	line: '	<aSelfClosingTag with="attribute" />',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'attribute.name.xml' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'attribute.value.xml' },
+		{ startIndex: 34, type: '' },
+		{ startIndex: 35, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 36, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<aSelfClosingTag with="attribute"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 17, type: '' },
+		{ startIndex: 18, type: 'attribute.name.xml' },
+		{ startIndex: 22, type: '' },
+		{ startIndex: 23, type: 'attribute.value.xml' },
+		{ startIndex: 34, type: 'tag.tag-aselfclosingtag.xml' },
+		{ startIndex: 35, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<namespace:aSelfClosingTag otherspace:with="attribute"/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-namespace:aselfclosingtag.xml' },
+		{ startIndex: 27, type: '' },
+		{ startIndex: 28, type: 'attribute.name.xml' },
+		{ startIndex: 43, type: '' },
+		{ startIndex: 44, type: 'attribute.value.xml' },
+		{ startIndex: 55, type: 'tag.tag-namespace:aselfclosingtag.xml' },
+		{ startIndex: 56, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<valid-name also_valid this.one=\'too is valid\'/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-valid-name.xml' },
+		{ startIndex: 12, type: '' },
+		{ startIndex: 13, type: 'attribute.name.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'attribute.name.xml' },
+		{ startIndex: 32, type: '' },
+		{ startIndex: 33, type: 'attribute.value.xml' },
+		{ startIndex: 47, type: 'tag.tag-valid-name.xml' },
+		{ startIndex: 48, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<aSimpleSelfClosingTag />',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-asimpleselfclosingtag.xml' },
+		{ startIndex: 23, type: '' },
+		{ startIndex: 24, type: 'tag.tag-asimpleselfclosingtag.xml' },
+		{ startIndex: 25, type: 'delimiter.start.xml' }
+	]}, {
+	line: '	<aSimpleSelfClosingTag/>',
+	tokens: [
+		{ startIndex: 0, type: '' },
+		{ startIndex: 1, type: 'delimiter.start.xml' },
+		{ startIndex: 2, type: 'tag.tag-asimpleselfclosingtag.xml' },
+		{ startIndex: 24, type: 'delimiter.start.xml' }
+	]}, {
+	line: '</configuration>',
+	tokens: [
+		{ startIndex: 0, type: 'delimiter.end.xml' },
+		{ startIndex: 2, type: 'tag.tag-configuration.xml' },
+		{ startIndex: 15, type: 'delimiter.end.xml' }
+	]}]
+]);

tsconfig.json

@@ -0,0 +1,58 @@
+{
+  "compilerOptions": {
+    "module": "amd",
+    "outDir": "out",
+    "target": "es5"
+  },
+  "filesGlob": [
+    "src/*.ts",
+    "test/*.ts",
+    "node_modules/monaco-editor-core/monaco.d.ts"
+  ],
+  "files": [
+    "src/bat.ts",
+    "src/coffee.ts",
+    "src/cpp.ts",
+    "src/csharp.ts",
+    "src/dockerfile.ts",
+    "src/fsharp.ts",
+    "src/go.ts",
+    "src/ini.ts",
+    "src/jade.ts",
+    "src/java.ts",
+    "src/lua.ts",
+    "src/monaco.contribution.ts",
+    "src/objective-c.ts",
+    "src/powershell.ts",
+    "src/python.ts",
+    "src/r.ts",
+    "src/ruby.ts",
+    "src/sql.ts",
+    "src/swift.ts",
+    "src/vb.ts",
+    "src/xml.ts",
+    "test/assert.d.ts",
+    "test/bat.test.ts",
+    "test/coffee.test.ts",
+    "test/cpp.test.ts",
+    "test/csharp.test.ts",
+    "test/dockerfile.test.ts",
+    "test/fsharp.test.ts",
+    "test/go.test.ts",
+    "test/jade.test.ts",
+    "test/java.test.ts",
+    "test/lua.test.ts",
+    "test/mocha.d.ts",
+    "test/objective-c.test.ts",
+    "test/powershell.test.ts",
+    "test/python.test.ts",
+    "test/r.test.ts",
+    "test/ruby.test.ts",
+    "test/sql.test.ts",
+    "test/swift.test.ts",
+    "test/testRunner.ts",
+    "test/vb.test.ts",
+    "test/xml.test.ts",
+    "node_modules/monaco-editor-core/monaco.d.ts"
+  ]
+}