Move into monaco-languages folder

monaco-languages/.editorconfig

@@ -0,0 +1,16 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+# Tab indentation
+[*]
+indent_style = tab
+indent_size = 4
+trim_trailing_whitespace = true
+
+# The indent size used in the `package.json` file cannot be changed
+# https://github.com/npm/npm/pull/3180#issuecomment-16336516
+[{*.yml,*.yaml,package.json}]
+indent_style = space
+indent_size = 2

monaco-languages/.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    name: CI
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 12
+
+      - name: npm install
+        run: npm install
+
+      - name: Compile
+        run: npm run compile
+
+      - name: Test
+        run: npm test

monaco-languages/.gitignore

@@ -0,0 +1,5 @@
+/.idea/
+/*.iml
+/node_modules/
+/out/
+/release/

monaco-languages/.npmignore

@@ -0,0 +1,15 @@
+/.vscode/
+/out/
+/release/**/*.test.js
+/release/**/test/
+/scripts/
+/src/
+/test/
+/gulpfile.js
+/tsconfig.json
+/.npmignore
+/.prettierrc
+/.prettierignore
+/.travis.yml
+/.editorconfig
+/azure-pipelines.yml

monaco-languages/.prettierignore

@@ -0,0 +1,2 @@
+/out/
+/release/

monaco-languages/.prettierrc

@@ -0,0 +1,8 @@
+{
+	"arrowParens": "always",
+	"singleQuote": true,
+	"trailingComma": "none",
+	"semi": true,
+	"useTabs": true,
+	"printWidth": 100
+}

monaco-languages/.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+	"version": "0.2.0",
+	"configurations": [
+		{
+			"name": "Unit Tests",
+			"type": "node",
+			"request": "launch",
+			"program": "./test/all.js",
+			"stopOnEntry": false,
+			"args": [
+				// "--grep",
+				// "typescript"
+			],
+			"cwd": "${workspaceRoot}",
+			"preLaunchTask": null,
+			"runtimeExecutable": null,
+			"runtimeArgs": ["--nolazy"],
+			"env": {
+				"NODE_ENV": "development"
+			},
+			"console": "internalConsole",
+			"sourceMaps": false,
+			"outDir": null
+		}
+	]
+}

monaco-languages/.vscode/settings.json

@@ -0,0 +1,14 @@
+// Place your settings in this file to overwrite default and user settings.
+{
+	"search.exclude": {
+		"**/node_modules": true,
+		"**/release": true,
+		"**/out": true
+	},
+	"files.trimTrailingWhitespace": true,
+	"files.insertFinalNewline": true,
+	"editor.tabSize": 4,
+	"editor.insertSpaces": false,
+	"editor.detectIndentation": false,
+	"typescript.tsdk": "./node_modules/typescript/lib"
+}

monaco-languages/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

monaco-languages/README.md

@@ -0,0 +1,39 @@
+# Monaco Languages [![Build Status](https://dev.azure.com/ms/monaco-languages/_apis/build/status/microsoft.monaco-languages?branchName=master)](https://dev.azure.com/ms/monaco-languages/_build/latest?definitionId=140&branchName=master)
+
+Colorization and configuration supports for multiple languages for the Monaco Editor:
+
+![monaco-languages](https://cloud.githubusercontent.com/assets/5047891/15938606/1fd4bac6-2e74-11e6-8839-d455da8bc8a7.gif)
+
+## Issues
+
+Please file issues concerning `monaco-languages` in the [`monaco-editor`-repository](https://github.com/Microsoft/monaco-editor/issues).
+
+## Installing
+
+This npm module is bundled and distributed in the [monaco-editor](https://www.npmjs.com/package/monaco-editor) npm module.
+
+## Dev: cheat sheet
+
+-   initial setup with `npm install .`
+-   compile with `npm run watch`
+-   test with `npm run test`
+-   bundle with `npm run prepublishOnly`
+
+## Dev: Adding a new language
+
+-   create `$/src/myLang/myLang.contribution.ts`
+-   create `$/src/myLang/myLang.ts`
+-   create `$/src/myLang/myLang.test.ts`
+-   edit `$/src/monaco.contribution.ts` and register your new language
+
+```js
+import './myLang/myLang.contribution';
+```
+
+## Code of Conduct
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+## License
+
+[MIT](https://github.com/Microsoft/monaco-languages/blob/master/LICENSE.md)

monaco-languages/SECURITY.md

@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.5 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](<https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)>), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+-   Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+-   Full paths of source file(s) related to the manifestation of the issue
+-   The location of the affected source code (tag/branch/commit or direct URL)
+-   Any special configuration required to reproduce the issue
+-   Step-by-step instructions to reproduce the issue
+-   Proof-of-concept or exploit code (if possible)
+-   Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

monaco-languages/ThirdPartyNotices.txt

@@ -0,0 +1,34 @@
+THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
+Do Not Translate or Localize
+
+This project incorporates components from the projects listed below. The original copyright notices and the licenses
+under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted
+herein, whether by implication, estoppel or otherwise.
+
+
+
+
+%% vscode-swift version 0.0.1 (https://github.com/owensd/vscode-swift)
+=========================================
+The MIT License (MIT)
+
+Copyright (c) 2015 David Owens II
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+=========================================
+END OF vscode-swift NOTICES AND INFORMATION

monaco-languages/package.json

@@ -0,0 +1,41 @@
+{
+	"name": "monaco-languages",
+	"version": "2.11.1",
+	"description": "Bundle of many languages for the Monaco Editor.",
+	"scripts": {
+		"compile": "mrmdir ./out && tsc -p ./src/tsconfig.json && tsc -p ./src/tsconfig.esm.json",
+		"watch": "tsc -p ./src --watch",
+		"watch-esm": "tsc -p ./src/tsconfig.esm.json --watch",
+		"test": "node ./test/all.js",
+		"prepublishOnly": "mrmdir ./release && npm run compile && node ./scripts/release.js && node ./scripts/bundle",
+		"prettier": "prettier --write ."
+	},
+	"author": "Microsoft Corporation",
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/Microsoft/monaco-languages"
+	},
+	"bugs": {
+		"url": "https://github.com/Microsoft/monaco-languages/issues"
+	},
+	"devDependencies": {
+		"@types/tape": "^4.13.2",
+		"glob": "^7.2.0",
+		"husky": "^4.3.8",
+		"jsdom": "^17.0.0",
+		"monaco-editor-core": "0.30.0",
+		"monaco-plugin-helpers": "^1.0.3",
+		"prettier": "^2.4.1",
+		"pretty-quick": "^3.1.1",
+		"requirejs": "^2.3.6",
+		"tape": "^5.3.1",
+		"terser": "^5.9.0",
+		"typescript": "4.4.3"
+	},
+	"husky": {
+		"hooks": {
+			"pre-commit": "pretty-quick --staged"
+		}
+	}
+}

monaco-languages/scripts/bundle.js

@@ -0,0 +1,72 @@
+const requirejs = require('requirejs');
+const path = require('path');
+const fs = require('fs');
+const terser = require('terser');
+const glob = require('glob');
+const helpers = require('monaco-plugin-helpers');
+
+const REPO_ROOT = path.resolve(__dirname, '..');
+
+const sha1 = helpers.getGitVersion(REPO_ROOT);
+const semver = require('../package.json').version;
+const headerVersion = semver + '(' + sha1 + ')';
+
+const BUNDLED_FILE_HEADER = [
+	'/*!-----------------------------------------------------------------------------',
+	' * Copyright (c) Microsoft Corporation. All rights reserved.',
+	' * monaco-languages version: ' + headerVersion,
+	' * Released under the MIT license',
+	' * https://github.com/Microsoft/monaco-languages/blob/master/LICENSE.md',
+	' *-----------------------------------------------------------------------------*/',
+	''
+].join('\n');
+
+bundleOne('monaco.contribution');
+glob('out/amd/*/*.contribution.js', { cwd: path.dirname(__dirname) }, function (err, files) {
+	if (err) {
+		console.log(err);
+		return;
+	}
+	files.forEach(function (file) {
+		file = file.replace(/\.contribution\.js$/, '');
+		file = file.replace(/out[/\\]amd[/\\]/, '');
+		bundleOne(file, ['vs/basic-languages/monaco.contribution']);
+	});
+});
+
+function bundleOne(moduleId, exclude) {
+	requirejs.optimize(
+		{
+			baseUrl: 'out/amd/',
+			name: 'vs/basic-languages/' + moduleId,
+			out: 'release/dev/' + moduleId + '.js',
+			exclude: exclude,
+			paths: {
+				'vs/basic-languages': REPO_ROOT + '/out/amd',
+				'vs/basic-languages/fillers/monaco-editor-core':
+					REPO_ROOT + '/out/amd/fillers/monaco-editor-core-amd'
+			},
+			optimize: 'none'
+		},
+		async function (buildResponse) {
+			const devFilePath = path.join(REPO_ROOT, 'release/dev/' + moduleId + '.js');
+			const minFilePath = path.join(REPO_ROOT, 'release/min/' + moduleId + '.js');
+			const fileContents = fs.readFileSync(devFilePath).toString();
+			console.log();
+			console.log(`Minifying ${devFilePath}...`);
+			const result = await terser.minify(fileContents, {
+				output: {
+					comments: 'some'
+				}
+			});
+			console.log(`Done minifying ${devFilePath}.`);
+			try {
+				fs.mkdirSync(path.join(REPO_ROOT, 'release/min'));
+			} catch (err) {}
+			try {
+				fs.mkdirSync(path.dirname(minFilePath));
+			} catch (err) {}
+			fs.writeFileSync(minFilePath, BUNDLED_FILE_HEADER + result.code);
+		}
+	);
+}

monaco-languages/scripts/release.js

@@ -0,0 +1,17 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+const path = require('path');
+const helpers = require('monaco-plugin-helpers');
+
+const REPO_ROOT = path.join(__dirname, '../');
+
+helpers.packageESM({
+	repoRoot: REPO_ROOT,
+	esmSource: 'out/esm',
+	esmDestination: 'release/esm',
+	entryPoints: ['monaco.contribution.js'],
+	resolveSkip: ['monaco-editor-core']
+});

monaco-languages/src/_.contribution.ts

@@ -0,0 +1,79 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { languages } from './fillers/monaco-editor-core';
+
+interface ILang extends languages.ILanguageExtensionPoint {
+	loader: () => Promise<ILangImpl>;
+}
+
+interface ILangImpl {
+	conf: languages.LanguageConfiguration;
+	language: languages.IMonarchLanguage;
+}
+
+const languageDefinitions: { [languageId: string]: ILang } = {};
+const lazyLanguageLoaders: { [languageId: string]: LazyLanguageLoader } = {};
+
+class LazyLanguageLoader {
+	public static getOrCreate(languageId: string): LazyLanguageLoader {
+		if (!lazyLanguageLoaders[languageId]) {
+			lazyLanguageLoaders[languageId] = new LazyLanguageLoader(languageId);
+		}
+		return lazyLanguageLoaders[languageId];
+	}
+
+	private readonly _languageId: string;
+	private _loadingTriggered: boolean;
+	private _lazyLoadPromise: Promise<ILangImpl>;
+	private _lazyLoadPromiseResolve!: (value: ILangImpl) => void;
+	private _lazyLoadPromiseReject!: (err: any) => void;
+
+	constructor(languageId: string) {
+		this._languageId = languageId;
+		this._loadingTriggered = false;
+		this._lazyLoadPromise = new Promise((resolve, reject) => {
+			this._lazyLoadPromiseResolve = resolve;
+			this._lazyLoadPromiseReject = reject;
+		});
+	}
+
+	public whenLoaded(): Promise<ILangImpl> {
+		return this._lazyLoadPromise;
+	}
+
+	public load(): Promise<ILangImpl> {
+		if (!this._loadingTriggered) {
+			this._loadingTriggered = true;
+			languageDefinitions[this._languageId].loader().then(
+				(mod) => this._lazyLoadPromiseResolve(mod),
+				(err) => this._lazyLoadPromiseReject(err)
+			);
+		}
+		return this._lazyLoadPromise;
+	}
+}
+
+export function loadLanguage(languageId: string): Promise<ILangImpl> {
+	return LazyLanguageLoader.getOrCreate(languageId).load();
+}
+
+export function registerLanguage(def: ILang): void {
+	const languageId = def.id;
+
+	languageDefinitions[languageId] = def;
+	languages.register(def);
+
+	const lazyLanguageLoader = LazyLanguageLoader.getOrCreate(languageId);
+	languages.setMonarchTokensProvider(
+		languageId,
+		lazyLanguageLoader.whenLoaded().then((mod) => mod.language)
+	);
+	languages.onLanguage(languageId, () => {
+		lazyLanguageLoader.load().then((mod) => {
+			languages.setLanguageConfiguration(languageId, mod.conf);
+		});
+	});
+}

monaco-languages/src/abap/abap.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'abap',
+	extensions: ['.abap'],
+	aliases: ['abap', 'ABAP'],
+	loader: () => import('./abap')
+});

monaco-languages/src/abap/abap.test.ts

@@ -0,0 +1,360 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('abap', [
+	[
+		{
+			line: '* comment',
+			tokens: [{ startIndex: 0, type: 'comment.abap' }]
+		}
+	],
+	[
+		{
+			line: ' " comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'comment.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'write hello.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'identifier.abap' },
+				{ startIndex: 11, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'IF 2 = 3.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'number.abap' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'key.abap' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'number.abap' },
+				{ startIndex: 8, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: "'hello'",
+			tokens: [{ startIndex: 0, type: 'string.abap' }]
+		}
+	],
+	[
+		{
+			line: '|hello|',
+			tokens: [{ startIndex: 0, type: 'string.abap' }]
+		}
+	],
+	[
+		{
+			line: 'write: hello, world.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 5, type: 'delimiter.abap' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.abap' },
+				{ startIndex: 12, type: 'delimiter.abap' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'identifier.abap' },
+				{ startIndex: 19, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'method_call( param ).',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.abap' },
+				{ startIndex: 11, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'identifier.abap' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 20, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: "'he'' llo'",
+			tokens: [{ startIndex: 0, type: 'string.abap' }]
+		}
+	],
+	[
+		{
+			line: '|hel\\|lo|',
+			tokens: [{ startIndex: 0, type: 'string.abap' }]
+		}
+	],
+	[
+		{
+			line: 'FIELD-SYMBOLS <foo>.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'identifier.abap' },
+				{ startIndex: 19, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'IF foo IS NOT INITIAL.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'identifier.abap' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'keyword.abap' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'keyword.abap' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'keyword.abap' },
+				{ startIndex: 21, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'WRITE `moo`.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'string.abap' },
+				{ startIndex: 11, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'FORM foo.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'identifier.abap' },
+				{ startIndex: 8, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'moo = CONV #( 1 ).',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.abap' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'key.abap' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'keyword.abap' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'key.abap' },
+				{ startIndex: 12, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'number.abap' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 17, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'WRITE foo ##pragma.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'identifier.abap' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'comment.abap' },
+				{ startIndex: 18, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'SELECT * FROM foo02 INTO @foo.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'key.abap' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.abap' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'identifier.abap' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'keyword.abap' },
+				{ startIndex: 24, type: '' },
+				{ startIndex: 25, type: 'identifier.abap' },
+				{ startIndex: 29, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'li = lines( itab ).',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.abap' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'key.abap' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'keyword.abap' },
+				{ startIndex: 10, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'identifier.abap' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 18, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: "foo = 'bar' && 'baz'.",
+			tokens: [
+				{ startIndex: 0, type: 'identifier.abap' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'key.abap' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'string.abap' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'key.abap' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'string.abap' },
+				{ startIndex: 20, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'DATA num TYPE n.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'identifier.abap' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.abap' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'type.abap' },
+				{ startIndex: 15, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'CLASS-METHODS class_constructor.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'type.abap' },
+				{ startIndex: 31, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'super->constructor( ).',
+			tokens: [
+				{ startIndex: 0, type: 'type.abap' },
+				{ startIndex: 5, type: 'tag.abap' },
+				{ startIndex: 7, type: 'type.abap' },
+				{ startIndex: 18, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 21, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'foo->my_method( ).',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.abap' },
+				{ startIndex: 3, type: 'tag.abap' },
+				{ startIndex: 5, type: 'identifier.abap' },
+				{ startIndex: 14, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 17, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'foo=>const_bar.',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.abap' },
+				{ startIndex: 3, type: 'tag.abap' },
+				{ startIndex: 5, type: 'identifier.abap' },
+				{ startIndex: 14, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'foo-bar+42(42).',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.abap' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.abap' },
+				{ startIndex: 10, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 11, type: 'number.abap' },
+				{ startIndex: 13, type: 'delimiter.parenthesis.abap' },
+				{ startIndex: 14, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: "@EndUserText.label: 'hallo'",
+			tokens: [
+				{ startIndex: 0, type: 'annotation.abap' },
+				{ startIndex: 12, type: 'delimiter.abap' },
+				{ startIndex: 13, type: 'identifier.abap' },
+				{ startIndex: 18, type: 'delimiter.abap' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'string.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'IF foo = abap_true.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'identifier.abap' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'key.abap' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'type.abap' },
+				{ startIndex: 18, type: 'delimiter.abap' }
+			]
+		}
+	],
+	[
+		{
+			line: 'LOOP AT screen.',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.abap' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'keyword.abap' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'type.abap' },
+				{ startIndex: 14, type: 'delimiter.abap' }
+			]
+		}
+	]
+]);

monaco-languages/src/apex/apex.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'apex',
+	extensions: ['.cls'],
+	aliases: ['Apex', 'apex'],
+	mimetypes: ['text/x-apex-source', 'text/x-apex'],
+	loader: () => import('./apex')
+});

monaco-languages/src/apex/apex.test.ts

@@ -0,0 +1,700 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('apex', [
+	// Comments - single line
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.apex' }
+			]
+		}
+	],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[
+		{
+			line: '/* //*/ a',
+			tokens: [
+				{ startIndex: 0, type: 'comment.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '/almost a comment',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.apex' },
+				{ startIndex: 1, type: 'identifier.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.apex' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1 / 2; /* comment',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.apex' },
+				{ startIndex: 5, type: 'delimiter.apex' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'comment.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'int x = 1; // my comment // is a nice one',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.apex' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.apex' },
+				{ startIndex: 9, type: 'delimiter.apex' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'comment.apex' }
+			]
+		}
+	],
+
+	// Comments - range comment, single line
+	[
+		{
+			line: '/* a simple comment */',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* a simple comment */ 1;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.apex' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.apex' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'number.apex' },
+				{ startIndex: 32, type: 'delimiter.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* comment */ 1; */',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.apex' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.apex' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'number.apex' },
+				{ startIndex: 23, type: 'delimiter.apex' },
+				{ startIndex: 24, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /**/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.apex' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.apex' },
+				{ startIndex: 8, type: 'delimiter.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /*/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.apex' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.apex' }
+			]
+		}
+	],
+
+	// Comments - range comment, multiple lines
+	[
+		{
+			line: '/* start of multiline comment',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		},
+		{
+			line: 'a comment between without a star',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		},
+		{
+			line: 'end of multiline comment*/',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* start a comment',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.apex' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.apex' }
+			]
+		},
+		{
+			line: ' a ',
+			tokens: [{ startIndex: 0, type: 'comment.apex' }]
+		},
+		{
+			line: 'and end it */ 2;',
+			tokens: [
+				{ startIndex: 0, type: 'comment.apex' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'number.apex' },
+				{ startIndex: 15, type: 'delimiter.apex' }
+			]
+		}
+	],
+
+	// Comments - apex doc, multiple lines
+	[
+		{
+			line: '/** start of Apex Doc',
+			tokens: [{ startIndex: 0, type: 'comment.doc.apex' }]
+		},
+		{
+			line: 'a comment between without a star',
+			tokens: [{ startIndex: 0, type: 'comment.doc.apex' }]
+		},
+		{
+			line: 'end of multiline comment*/',
+			tokens: [{ startIndex: 0, type: 'comment.doc.apex' }]
+		}
+	],
+
+	// Keywords
+	[
+		{
+			line: 'package test; class Program { static void main(String[] args) {} } }',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.package.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.apex' },
+				{ startIndex: 12, type: 'delimiter.apex' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'keyword.class.apex' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'type.identifier.apex' },
+				{ startIndex: 27, type: '' },
+				{ startIndex: 28, type: 'delimiter.curly.apex' },
+				{ startIndex: 29, type: '' },
+				{ startIndex: 30, type: 'keyword.static.apex' },
+				{ startIndex: 36, type: '' },
+				{ startIndex: 37, type: 'keyword.void.apex' },
+				{ startIndex: 41, type: '' },
+				{ startIndex: 42, type: 'identifier.apex' },
+				{ startIndex: 46, type: 'delimiter.parenthesis.apex' },
+				{ startIndex: 47, type: 'type.identifier.apex' },
+				{ startIndex: 53, type: 'delimiter.square.apex' },
+				{ startIndex: 55, type: '' },
+				{ startIndex: 56, type: 'identifier.apex' },
+				{ startIndex: 60, type: 'delimiter.parenthesis.apex' },
+				{ startIndex: 61, type: '' },
+				{ startIndex: 62, type: 'delimiter.curly.apex' },
+				{ startIndex: 64, type: '' },
+				{ startIndex: 65, type: 'delimiter.curly.apex' },
+				{ startIndex: 66, type: '' },
+				{ startIndex: 67, type: 'delimiter.curly.apex' }
+			]
+		}
+	],
+
+	// Keywords with case variations
+	[
+		{
+			line: 'Package test; CLASS Program { Static void main(String[] args) {} } }',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.Package.apex' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.apex' },
+				{ startIndex: 12, type: 'delimiter.apex' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'keyword.CLASS.apex' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'type.identifier.apex' },
+				{ startIndex: 27, type: '' },
+				{ startIndex: 28, type: 'delimiter.curly.apex' },
+				{ startIndex: 29, type: '' },
+				{ startIndex: 30, type: 'keyword.Static.apex' },
+				{ startIndex: 36, type: '' },
+				{ startIndex: 37, type: 'keyword.void.apex' },
+				{ startIndex: 41, type: '' },
+				{ startIndex: 42, type: 'identifier.apex' },
+				{ startIndex: 46, type: 'delimiter.parenthesis.apex' },
+				{ startIndex: 47, type: 'type.identifier.apex' },
+				{ startIndex: 53, type: 'delimiter.square.apex' },
+				{ startIndex: 55, type: '' },
+				{ startIndex: 56, type: 'identifier.apex' },
+				{ startIndex: 60, type: 'delimiter.parenthesis.apex' },
+				{ startIndex: 61, type: '' },
+				{ startIndex: 62, type: 'delimiter.curly.apex' },
+				{ startIndex: 64, type: '' },
+				{ startIndex: 65, type: 'delimiter.curly.apex' },
+				{ startIndex: 66, type: '' },
+				{ startIndex: 67, type: 'delimiter.curly.apex' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '0.10',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '0x',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '10e3',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '10f',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e-3',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E-3',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5F',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5f',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5D',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5d',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3D',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3d',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E-3d',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3D',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3d',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e-3d',
+			tokens: [{ startIndex: 0, type: 'number.float.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23L',
+			tokens: [{ startIndex: 0, type: 'number.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '23l',
+			tokens: [{ startIndex: 0, type: 'number.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '0_52',
+			tokens: [{ startIndex: 0, type: 'number.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '5_2',
+			tokens: [{ startIndex: 0, type: 'number.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '5_______2',
+			tokens: [{ startIndex: 0, type: 'number.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '3_.1415F',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: 'identifier.apex' },
+				{ startIndex: 2, type: 'delimiter.apex' },
+				{ startIndex: 3, type: 'number.float.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '3._1415F',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: 'delimiter.apex' },
+				{ startIndex: 2, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '999_99_9999_L',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 11, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '52_',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 2, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0_x52',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x_52',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: 'identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '23.5L',
+			tokens: [
+				{ startIndex: 0, type: 'number.float.apex' },
+				{ startIndex: 4, type: 'type.identifier.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: 'delimiter.apex' },
+				{ startIndex: 2, type: 'number.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 3, type: 'delimiter.apex' },
+				{ startIndex: 4, type: 'number.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.apex' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.apex' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.apex' }
+			]
+		}
+	],
+
+	// single line Strings
+	[
+		{
+			line: 'String s = "I\'m an Apex String";',
+			tokens: [
+				{ startIndex: 0, type: 'type.identifier.apex' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.apex' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.apex' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.apex' },
+				{ startIndex: 31, type: 'delimiter.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'String s = "concatenated" + " String" ;',
+			tokens: [
+				{ startIndex: 0, type: 'type.identifier.apex' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.apex' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.apex' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.apex' },
+				{ startIndex: 25, type: '' },
+				{ startIndex: 26, type: 'delimiter.apex' },
+				{ startIndex: 27, type: '' },
+				{ startIndex: 28, type: 'string.apex' },
+				{ startIndex: 37, type: '' },
+				{ startIndex: 38, type: 'delimiter.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"quote in a string"',
+			tokens: [{ startIndex: 0, type: 'string.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '"escaping \\"quotes\\" is cool"',
+			tokens: [
+				{ startIndex: 0, type: 'string.apex' },
+				{ startIndex: 10, type: 'string.escape.apex' },
+				{ startIndex: 12, type: 'string.apex' },
+				{ startIndex: 18, type: 'string.escape.apex' },
+				{ startIndex: 20, type: 'string.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"\\"',
+			tokens: [{ startIndex: 0, type: 'string.invalid.apex' }]
+		}
+	],
+
+	// Annotations
+	[
+		{
+			line: '@',
+			tokens: [{ startIndex: 0, type: '' }]
+		}
+	],
+
+	[
+		{
+			line: '@Override',
+			tokens: [{ startIndex: 0, type: 'annotation.apex' }]
+		}
+	],
+
+	[
+		{
+			line: '@SuppressWarnings(value = "aString")',
+			tokens: [
+				{ startIndex: 0, type: 'annotation.apex' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.apex' },
+				{ startIndex: 18, type: 'identifier.apex' },
+				{ startIndex: 23, type: '' },
+				{ startIndex: 24, type: 'delimiter.apex' },
+				{ startIndex: 25, type: '' },
+				{ startIndex: 26, type: 'string.apex' },
+				{ startIndex: 35, type: 'delimiter.parenthesis.apex' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '@ AnnotationWithKeywordAfter private',
+			tokens: [
+				{ startIndex: 0, type: 'annotation.apex' },
+				{ startIndex: 28, type: '' },
+				{ startIndex: 29, type: 'keyword.private.apex' }
+			]
+		}
+	]
+]);

monaco-languages/src/apex/apex.ts

@@ -0,0 +1,356 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	// the default separators except `@$`
+	wordPattern:
+		/(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '<', close: '>' }
+	],
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*//\\s*(?:(?:#?region\\b)|(?:<editor-fold\\b))'),
+			end: new RegExp('^\\s*//\\s*(?:(?:#?endregion\\b)|(?:</editor-fold>))')
+		}
+	}
+};
+
+const keywords = [
+	'abstract',
+	'activate',
+	'and',
+	'any',
+	'array',
+	'as',
+	'asc',
+	'assert',
+	'autonomous',
+	'begin',
+	'bigdecimal',
+	'blob',
+	'boolean',
+	'break',
+	'bulk',
+	'by',
+	'case',
+	'cast',
+	'catch',
+	'char',
+	'class',
+	'collect',
+	'commit',
+	'const',
+	'continue',
+	'convertcurrency',
+	'decimal',
+	'default',
+	'delete',
+	'desc',
+	'do',
+	'double',
+	'else',
+	'end',
+	'enum',
+	'exception',
+	'exit',
+	'export',
+	'extends',
+	'false',
+	'final',
+	'finally',
+	'float',
+	'for',
+	'from',
+	'future',
+	'get',
+	'global',
+	'goto',
+	'group',
+	'having',
+	'hint',
+	'if',
+	'implements',
+	'import',
+	'in',
+	'inner',
+	'insert',
+	'instanceof',
+	'int',
+	'interface',
+	'into',
+	'join',
+	'last_90_days',
+	'last_month',
+	'last_n_days',
+	'last_week',
+	'like',
+	'limit',
+	'list',
+	'long',
+	'loop',
+	'map',
+	'merge',
+	'native',
+	'new',
+	'next_90_days',
+	'next_month',
+	'next_n_days',
+	'next_week',
+	'not',
+	'null',
+	'nulls',
+	'number',
+	'object',
+	'of',
+	'on',
+	'or',
+	'outer',
+	'override',
+	'package',
+	'parallel',
+	'pragma',
+	'private',
+	'protected',
+	'public',
+	'retrieve',
+	'return',
+	'returning',
+	'rollback',
+	'savepoint',
+	'search',
+	'select',
+	'set',
+	'short',
+	'sort',
+	'stat',
+	'static',
+	'strictfp',
+	'super',
+	'switch',
+	'synchronized',
+	'system',
+	'testmethod',
+	'then',
+	'this',
+	'this_month',
+	'this_week',
+	'throw',
+	'throws',
+	'today',
+	'tolabel',
+	'tomorrow',
+	'transaction',
+	'transient',
+	'trigger',
+	'true',
+	'try',
+	'type',
+	'undelete',
+	'update',
+	'upsert',
+	'using',
+	'virtual',
+	'void',
+	'volatile',
+	'webservice',
+	'when',
+	'where',
+	'while',
+	'yesterday'
+];
+
+// create case variations of the keywords - apex is case insensitive, but we can't make the highlighter case insensitive
+// because we use a heuristic to assume that identifiers starting with an upper case letter are types.
+const uppercaseFirstLetter = (lowercase: string) =>
+	lowercase.charAt(0).toUpperCase() + lowercase.substr(1);
+
+let keywordsWithCaseVariations: string[] = [];
+keywords.forEach((lowercase) => {
+	keywordsWithCaseVariations.push(lowercase);
+	keywordsWithCaseVariations.push(lowercase.toUpperCase());
+	keywordsWithCaseVariations.push(uppercaseFirstLetter(lowercase));
+});
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.apex',
+
+	keywords: keywordsWithCaseVariations,
+
+	operators: [
+		'=',
+		'>',
+		'<',
+		'!',
+		'~',
+		'?',
+		':',
+		'==',
+		'<=',
+		'>=',
+		'!=',
+		'&&',
+		'||',
+		'++',
+		'--',
+		'+',
+		'-',
+		'*',
+		'/',
+		'&',
+		'|',
+		'^',
+		'%',
+		'<<',
+		'>>',
+		'>>>',
+		'+=',
+		'-=',
+		'*=',
+		'/=',
+		'&=',
+		'|=',
+		'^=',
+		'%=',
+		'<<=',
+		'>>=',
+		'>>>='
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[
+				/[a-z_$][\w$]*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			// assume that identifiers starting with an uppercase letter are types
+			[
+				/[A-Z][\w\$]*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'type.identifier'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// @ annotations.
+			[/@\s*[a-zA-Z_\$][\w\$]*/, 'annotation'],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)[fFdD]/, 'number.float'],
+			[/(@digits)[lL]?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, 'string', '@string."'],
+			[/'/, 'string', "@string.'"],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@apexdoc'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		//Identical copy of comment above, except for the addition of .doc
+		apexdoc: [
+			[/[^\/*]+/, 'comment.doc'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string: [
+			[/[^\\"']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[
+				/["']/,
+				{
+					cases: {
+						'$#==$S2': { token: 'string', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			]
+		]
+	}
+};

monaco-languages/src/azcli/azcli.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'azcli',
+	extensions: ['.azcli'],
+	aliases: ['Azure CLI', 'azcli'],
+	loader: () => import('./azcli')
+});

monaco-languages/src/azcli/azcli.test.ts

@@ -0,0 +1,172 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization as actualTestTokenization, ITestItem } from '../test/testRunner';
+
+function testTokenization(_language: string | string[], tests: ITestItem[][]): void {
+	tests = tests.map((t) => {
+		return t.map((t) => {
+			return {
+				line: t.line.replace(/\n/g, ' '),
+				tokens: t.tokens
+			};
+		});
+	});
+	actualTestTokenization(_language, tests);
+}
+
+testTokenization('azcli', [
+	// Comment single line
+	[
+		{
+			line: '#',
+			tokens: [{ startIndex: 0, type: 'comment.azcli' }]
+		}
+	],
+	[
+		{
+			line: '# az find -q secret',
+			tokens: [{ startIndex: 0, type: 'comment.azcli' }]
+		}
+	],
+	[
+		{
+			line: '    # az find -q secret',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.azcli' },
+				{ startIndex: 4, type: 'comment.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: '#az find -q secret',
+			tokens: [{ startIndex: 0, type: 'comment.azcli' }]
+		}
+	],
+
+	// Other cases
+	[
+		{
+			line: 'az find -q secret',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.azcli' },
+				{ startIndex: 7, type: 'key.identifier.azcli' },
+				{ startIndex: 11, type: 'string.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: '',
+			tokens: []
+		}
+	],
+	[
+		{
+			line: ' ',
+			tokens: [{ startIndex: 0, type: 'keyword.azcli' }]
+		}
+	],
+	[
+		{
+			line: '--assignee',
+			tokens: [{ startIndex: 0, type: 'key.identifier.azcli' }]
+		}
+	],
+	[
+		{
+			line: '    --service-principal',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.azcli' },
+				{ startIndex: 3, type: 'key.identifier.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: 'az ad sp create-for-rb  --name ServicePrincipalName --password PASSWORD',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.azcli' },
+				{ startIndex: 23, type: 'key.identifier.azcli' },
+				{ startIndex: 31, type: 'string.azcli' },
+				{ startIndex: 52, type: 'key.identifier.azcli' },
+				{ startIndex: 63, type: 'string.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: '--name!~`"$%^&*(|/.,-=+',
+			tokens: [{ startIndex: 0, type: 'key.identifier.azcli' }]
+		}
+	],
+	[
+		{
+			line: '--name#some comment',
+			tokens: [
+				{ startIndex: 0, type: 'key.identifier.azcli' },
+				{ startIndex: 6, type: 'comment.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: '--query osPro ``````',
+			tokens: [
+				{ startIndex: 0, type: 'key.identifier.azcli' },
+				{ startIndex: 8, type: 'string.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: 'az ad sp create-for-rbac',
+			tokens: [{ startIndex: 0, type: 'keyword.azcli' }]
+		}
+	],
+	[
+		{
+			line: '123456789',
+			tokens: [{ startIndex: 0, type: 'keyword.azcli' }]
+		}
+	],
+	[
+		{
+			line: '- abc',
+			tokens: [
+				{ startIndex: 0, type: 'key.identifier.azcli' },
+				{ startIndex: 2, type: 'string.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: '- @!$()',
+			tokens: [
+				{ startIndex: 0, type: 'key.identifier.azcli' },
+				{ startIndex: 2, type: 'string.azcli' }
+			]
+		}
+	],
+	[
+		{
+			line: '""',
+			tokens: [{ startIndex: 0, type: 'keyword.azcli' }]
+		}
+	],
+	[
+		{
+			line: '// some text',
+			tokens: [{ startIndex: 0, type: 'keyword.azcli' }]
+		}
+	],
+	[
+		{
+			line: `'APP_ID'`,
+			tokens: [{ startIndex: 0, type: 'keyword.azcli' }]
+		}
+	]
+]);

monaco-languages/src/azcli/azcli.ts

@@ -0,0 +1,77 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: 'keyword',
+	ignoreCase: true,
+	tokenPostfix: '.azcli',
+
+	str: /[^#\s]/,
+
+	tokenizer: {
+		root: [
+			{ include: '@comment' },
+			[
+				/\s-+@str*\s*/,
+				{
+					cases: {
+						'@eos': { token: 'key.identifier', next: '@popall' },
+						'@default': { token: 'key.identifier', next: '@type' }
+					}
+				}
+			],
+			[
+				/^-+@str*\s*/,
+				{
+					cases: {
+						'@eos': { token: 'key.identifier', next: '@popall' },
+						'@default': { token: 'key.identifier', next: '@type' }
+					}
+				}
+			]
+		],
+
+		type: [
+			{ include: '@comment' },
+			[
+				/-+@str*\s*/,
+				{
+					cases: {
+						'@eos': { token: 'key.identifier', next: '@popall' },
+						'@default': 'key.identifier'
+					}
+				}
+			],
+			[
+				/@str+\s*/,
+				{
+					cases: {
+						'@eos': { token: 'string', next: '@popall' },
+						'@default': 'string'
+					}
+				}
+			]
+		],
+
+		comment: [
+			[
+				/#.*$/,
+				{
+					cases: {
+						'@eos': { token: 'comment', next: '@popall' }
+					}
+				}
+			]
+		]
+	}
+};

monaco-languages/src/bat/bat.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'bat',
+	extensions: ['.bat', '.cmd'],
+	aliases: ['Batch', 'bat'],
+	loader: () => import('./bat')
+});

monaco-languages/src/bat/bat.test.ts

@@ -0,0 +1,397 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('bat', [
+	// Keywords
+	[
+		{
+			line: '@echo off title Selfhost',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.bat' },
+				{ startIndex: 1, type: 'keyword.echo.bat' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 10, type: 'keyword.title.bat' },
+				{ startIndex: 15, type: '' }
+			]
+		}
+	],
+
+	// Comments - single line
+	[
+		{
+			line: 'REM',
+			tokens: [{ startIndex: 0, type: 'comment.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '    REM a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.bat' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'REM a comment',
+			tokens: [{ startIndex: 0, type: 'comment.bat' }]
+		}
+	],
+
+	[
+		{
+			line: 'REMnot a comment',
+			tokens: [{ startIndex: 0, type: '' }]
+		}
+	],
+
+	// number
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '0.0',
+			tokens: [{ startIndex: 0, type: 'number.float.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e-3',
+			tokens: [{ startIndex: 0, type: 'number.float.bat' }]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.bat' },
+				{ startIndex: 1, type: 'delimiter.bat' },
+				{ startIndex: 2, type: 'number.bat' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.bat' },
+				{ startIndex: 3, type: 'delimiter.bat' },
+				{ startIndex: 4, type: 'number.bat' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.bat' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.bat' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.bat' }
+			]
+		}
+	],
+
+	// Strings
+	[
+		{
+			line: 'set s = "string"',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.set.bat' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 6, type: 'delimiter.bat' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'string.bat' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"use strict";',
+			tokens: [
+				{ startIndex: 0, type: 'string.bat' },
+				{ startIndex: 12, type: 'delimiter.bat' }
+			]
+		}
+	],
+
+	// Tags
+	[
+		{
+			line: 'setlocal endlocal',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.tag-setlocal.bat' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'setlocal ENDLOCAL',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.tag-setlocal.bat' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'SETLOCAL endlocal',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.tag-setlocal.bat' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'setlocal setlocal endlocal',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.tag-setlocal.bat' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.tag-setlocal.bat' },
+				{ startIndex: 17, type: '' },
+				{ startIndex: 18, type: 'keyword.tag-setlocal.bat' }
+			]
+		}
+	],
+
+	// Monarch generated
+	[
+		{
+			line: 'rem asdf',
+			tokens: [{ startIndex: 0, type: 'comment.bat' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'REM',
+			tokens: [{ startIndex: 0, type: 'comment.bat' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'REMOVED not a comment really',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 8, type: 'keyword.not.bat' },
+				{ startIndex: 11, type: '' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'echo cool',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.echo.bat' },
+				{ startIndex: 4, type: '' }
+			]
+		},
+		{
+			line: '@echo off',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.bat' },
+				{ startIndex: 1, type: 'keyword.echo.bat' },
+				{ startIndex: 5, type: '' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'setlocAL',
+			tokens: [{ startIndex: 0, type: 'keyword.tag-setlocal.bat' }]
+		},
+		{
+			line: '	asdf',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '	asdf',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: 'endLocaL',
+			tokens: [{ startIndex: 0, type: 'keyword.tag-setlocal.bat' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'call',
+			tokens: [{ startIndex: 0, type: 'keyword.call.bat' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: ':MyLabel',
+			tokens: [{ startIndex: 0, type: 'metatag.bat' }]
+		},
+		{
+			line: 'some command',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '%sdfsdf% ',
+			tokens: [
+				{ startIndex: 0, type: 'variable.bat' },
+				{ startIndex: 8, type: '' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'this is "a string %sdf% asdf"',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 8, type: 'string.bat' },
+				{ startIndex: 18, type: 'variable.bat' },
+				{ startIndex: 23, type: 'string.bat' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'FOR %%A IN (1 2 3) DO (',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.for.bat' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'variable.bat' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 11, type: 'delimiter.parenthesis.bat' },
+				{ startIndex: 12, type: 'number.bat' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'number.bat' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'number.bat' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.bat' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 22, type: 'delimiter.parenthesis.bat' }
+			]
+		},
+		{
+			line: '	SET VAR1=%VAR1%%%A',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'keyword.set.bat' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 9, type: 'delimiter.bat' },
+				{ startIndex: 10, type: 'variable.bat' }
+			]
+		},
+		{
+			line: '	SET VAR2=%VAR2%%%A',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'keyword.set.bat' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 9, type: 'delimiter.bat' },
+				{ startIndex: 10, type: 'variable.bat' }
+			]
+		},
+		{
+			line: "	use 'string %%a asdf asdf'",
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 5, type: 'string.bat' },
+				{ startIndex: 13, type: 'variable.bat' },
+				{ startIndex: 16, type: 'string.bat' }
+			]
+		},
+		{
+			line: '	non terminated "string %%aaa sdf',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 16, type: 'string.bat' },
+				{ startIndex: 24, type: 'variable.bat' },
+				{ startIndex: 29, type: 'string.bat' }
+			]
+		},
+		{
+			line: '	this shold NOT BE red',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 12, type: 'keyword.not.bat' },
+				{ startIndex: 15, type: '' }
+			]
+		},
+		{
+			line: ')',
+			tokens: [{ startIndex: 0, type: 'delimiter.parenthesis.bat' }]
+		}
+	]
+]);

monaco-languages/src/bat/bat.ts

@@ -0,0 +1,121 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: 'REM'
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' }
+	],
+	surroundingPairs: [
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' }
+	],
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*(::\\s*|REM\\s+)#region'),
+			end: new RegExp('^\\s*(::\\s*|REM\\s+)#endregion')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.bat',
+
+	brackets: [
+		{ token: 'delimiter.bracket', open: '{', close: '}' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' },
+		{ token: 'delimiter.square', open: '[', close: ']' }
+	],
+
+	keywords:
+		/call|defined|echo|errorlevel|exist|for|goto|if|pause|set|shift|start|title|not|pushd|popd/,
+
+	// we include these common regular expressions
+	symbols: /[=><!~?&|+\-*\/\^;\.,]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			[/^(\s*)(rem(?:\s.*|))$/, ['', 'comment']],
+
+			[/(\@?)(@keywords)(?!\w)/, [{ token: 'keyword' }, { token: 'keyword.$2' }]],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// blocks
+			[/setlocal(?!\w)/, 'keyword.tag-setlocal'],
+			[/endlocal(?!\w)/, 'keyword.tag-setlocal'],
+
+			// words
+			[/[a-zA-Z_]\w*/, ''],
+
+			// labels
+			[/:\w*/, 'metatag'],
+
+			// variables
+			[/%[^%]+%/, 'variable'],
+			[/%%[\w]+(?!\w)/, 'variable'],
+
+			// punctuations
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]*[0-9a-fA-F]/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// punctuation: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings:
+			[/"/, 'string', '@string."'],
+			[/'/, 'string', "@string.'"]
+		],
+
+		string: [
+			[
+				/[^\\"'%]+/,
+				{
+					cases: {
+						'@eos': { token: 'string', next: '@popall' },
+						'@default': 'string'
+					}
+				}
+			],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/%[\w ]+%/, 'variable'],
+			[/%%[\w]+(?!\w)/, 'variable'],
+			[
+				/["']/,
+				{
+					cases: {
+						'$#==$S2': { token: 'string', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			],
+			[/$/, 'string', '@popall']
+		]
+	}
+};

monaco-languages/src/bicep/bicep.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'bicep',
+	extensions: ['.bicep'],
+	aliases: ['Bicep'],
+	loader: () => import('./bicep')
+});

monaco-languages/src/bicep/bicep.ts

@@ -0,0 +1,131 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+const bounded = (text: string) => `\\b${text}\\b`;
+
+const identifierStart = '[_a-zA-Z]';
+const identifierContinue = '[_a-zA-Z0-9]';
+const identifier = bounded(`${identifierStart}${identifierContinue}*`);
+
+const keywords = [
+	'targetScope',
+	'resource',
+	'module',
+	'param',
+	'var',
+	'output',
+	'for',
+	'in',
+	'if',
+	'existing'
+];
+
+const namedLiterals = ['true', 'false', 'null'];
+
+const nonCommentWs = `[ \\t\\r\\n]`;
+
+const numericLiteral = `[0-9]+`;
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: "'", close: "'" },
+		{ open: "'''", close: "'''" }
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: "'", close: "'", notIn: ['string', 'comment'] },
+		{ open: "'''", close: "'''", notIn: ['string', 'comment'] }
+	],
+	autoCloseBefore: ":.,=}])' \n\t",
+	indentationRules: {
+		increaseIndentPattern: new RegExp(
+			'^((?!\\/\\/).)*(\\{[^}"\'`]*|\\([^)"\'`]*|\\[[^\\]"\'`]*)$'
+		),
+		decreaseIndentPattern: new RegExp('^((?!.*?\\/\\*).*\\*/)?\\s*[\\}\\]].*$')
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.bicep',
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	symbols: /[=><!~?:&|+\-*/^%]+/,
+
+	keywords,
+	namedLiterals,
+
+	escapes: `\\\\(u{[0-9A-Fa-f]+}|n|r|t|\\\\|'|\\\${)`,
+
+	tokenizer: {
+		root: [{ include: '@expression' }, { include: '@whitespace' }],
+
+		stringVerbatim: [
+			{ regex: `(|'|'')[^']`, action: { token: 'string' } },
+			{ regex: `'''`, action: { token: 'string.quote', next: '@pop' } }
+		],
+
+		stringLiteral: [
+			{ regex: `\\\${`, action: { token: 'delimiter.bracket', next: '@bracketCounting' } },
+			{ regex: `[^\\\\'$]+`, action: { token: 'string' } },
+			{ regex: '@escapes', action: { token: 'string.escape' } },
+			{ regex: `\\\\.`, action: { token: 'string.escape.invalid' } },
+			{ regex: `'`, action: { token: 'string', next: '@pop' } }
+		],
+
+		bracketCounting: [
+			{ regex: `{`, action: { token: 'delimiter.bracket', next: '@bracketCounting' } },
+			{ regex: `}`, action: { token: 'delimiter.bracket', next: '@pop' } },
+			{ include: 'expression' }
+		],
+
+		comment: [
+			{ regex: `[^\\*]+`, action: { token: 'comment' } },
+			{ regex: `\\*\\/`, action: { token: 'comment', next: '@pop' } },
+			{ regex: `[\\/*]`, action: { token: 'comment' } }
+		],
+
+		whitespace: [
+			{ regex: nonCommentWs },
+			{ regex: `\\/\\*`, action: { token: 'comment', next: '@comment' } },
+			{ regex: `\\/\\/.*$`, action: { token: 'comment' } }
+		],
+
+		expression: [
+			{ regex: `'''`, action: { token: 'string.quote', next: '@stringVerbatim' } },
+			{ regex: `'`, action: { token: 'string.quote', next: '@stringLiteral' } },
+			{ regex: numericLiteral, action: { token: 'number' } },
+			{
+				regex: identifier,
+				action: {
+					cases: {
+						'@keywords': { token: 'keyword' },
+						'@namedLiterals': { token: 'keyword' },
+						'@default': { token: 'identifier' }
+					}
+				}
+			}
+		]
+	}
+};

monaco-languages/src/cameligo/cameligo.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'cameligo',
+	extensions: ['.mligo'],
+	aliases: ['Cameligo'],
+	loader: () => import('./cameligo')
+});

monaco-languages/src/cameligo/cameligo.test.ts

@@ -0,0 +1,144 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('cameligo', [
+	// Comments - single line
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: 'white.cameligo' },
+				{ startIndex: 4, type: 'comment.cameligo' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		}
+	],
+
+	// Comments - multi line (single line)
+	[
+		{
+			line: '(**)',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		}
+	],
+
+	[
+		{
+			line: '    (* a comment *)',
+			tokens: [
+				{ startIndex: 0, type: 'white.cameligo' },
+				{ startIndex: 4, type: 'comment.cameligo' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '(* a comment *)',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		}
+	],
+
+	[
+		{
+			line: '(*sticky comment*)',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		}
+	],
+
+	// Comments - multi line (multi line)
+	[
+		{
+			line: '(* start of multiline comment ',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		},
+		{
+			line: 'a comment between curly',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		},
+		{
+			line: 'end of multiline comment*)',
+			tokens: [{ startIndex: 0, type: 'comment.cameligo' }]
+		}
+	],
+
+	// Keywords
+	[
+		{
+			line: 'let check if Current.amount',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.let.cameligo' },
+				{ startIndex: 3, type: 'white.cameligo' },
+				{ startIndex: 4, type: 'identifier.cameligo' },
+				{ startIndex: 9, type: 'white.cameligo' },
+				{ startIndex: 10, type: 'keyword.if.cameligo' },
+				{ startIndex: 12, type: 'white.cameligo' },
+				{ startIndex: 13, type: 'keyword.current.cameligo' },
+				{ startIndex: 20, type: 'delimiter.cameligo' },
+				{ startIndex: 21, type: 'identifier.cameligo' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.cameligo' }]
+		}
+	],
+	[
+		{
+			line: '0;',
+			tokens: [
+				{ startIndex: 0, type: 'number.cameligo' },
+				{ startIndex: 1, type: 'delimiter.cameligo' }
+			]
+		}
+	],
+	[
+		{
+			line: '2.4',
+			tokens: [{ startIndex: 0, type: 'number.float.cameligo' }]
+		}
+	],
+	[
+		{
+			line: '2.4;',
+			tokens: [
+				{ startIndex: 0, type: 'number.float.cameligo' },
+				{ startIndex: 3, type: 'delimiter.cameligo' }
+			]
+		}
+	],
+	[
+		{
+			line: '$123FF',
+			tokens: [{ startIndex: 0, type: 'number.hex.cameligo' }]
+		}
+	]
+]);

monaco-languages/src/cameligo/cameligo.ts

@@ -0,0 +1,193 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['(*', '*)']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')'],
+		['<', '>']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' },
+		{ open: "'", close: "'" },
+		{ open: '"', close: '"' },
+		{ open: '(*', close: '*)' }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' },
+		{ open: "'", close: "'" },
+		{ open: '"', close: '"' },
+		{ open: '(*', close: '*)' }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.cameligo',
+	ignoreCase: true,
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '<', close: '>', token: 'delimiter.angle' }
+	],
+
+	keywords: [
+		'abs',
+		'assert',
+		'block',
+		'Bytes',
+		'case',
+		'Crypto',
+		'Current',
+		'else',
+		'failwith',
+		'false',
+		'for',
+		'fun',
+		'if',
+		'in',
+		'let',
+		'let%entry',
+		'let%init',
+		'List',
+		'list',
+		'Map',
+		'map',
+		'match',
+		'match%nat',
+		'mod',
+		'not',
+		'operation',
+		'Operation',
+		'of',
+		'record',
+		'Set',
+		'set',
+		'sender',
+		'skip',
+		'source',
+		'String',
+		'then',
+		'to',
+		'true',
+		'type',
+		'with'
+	],
+
+	typeKeywords: ['int', 'unit', 'string', 'tz', 'nat', 'bool'],
+
+	operators: [
+		'=',
+		'>',
+		'<',
+		'<=',
+		'>=',
+		'<>',
+		':',
+		':=',
+		'and',
+		'mod',
+		'or',
+		'+',
+		'-',
+		'*',
+		'/',
+		'@',
+		'&',
+		'^',
+		'%',
+		'->',
+		'<-',
+		'&&',
+		'||'
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><:@\^&|+\-*\/\^%]+/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[
+				/[a-zA-Z_][\w]*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/\$[0-9a-fA-F]{1,16}/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/'([^'\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/'/, 'string', '@string'],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/'/, 'string.invalid'],
+			[/\#\d+/, 'string']
+		],
+		/* */
+
+		comment: [
+			[/[^\(\*]+/, 'comment'],
+			//[/\(\*/,    'comment', '@push' ],    // nested comment  not allowed :-(
+			[/\*\)/, 'comment', '@pop'],
+			[/\(\*/, 'comment']
+		],
+
+		string: [
+			[/[^\\']+/, 'string'],
+			[/\\./, 'string.escape.invalid'],
+			[/'/, { token: 'string.quote', bracket: '@close', next: '@pop' }]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, 'white'],
+			[/\(\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		]
+	}
+};

monaco-languages/src/clojure/clojure.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'clojure',
+	extensions: ['.clj', '.cljs', '.cljc', '.edn'],
+	aliases: ['clojure', 'Clojure'],
+	loader: () => import('./clojure')
+});

monaco-languages/src/clojure/clojure.test.ts

@@ -0,0 +1,943 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ITestItem, testTokenization } from '../test/testRunner';
+
+const specialForms = [
+	'.',
+	'catch',
+	'def',
+	'do',
+	'if',
+	'monitor-enter',
+	'monitor-exit',
+	'new',
+	'quote',
+	'recur',
+	'set!',
+	'throw',
+	'try',
+	'var'
+];
+
+const coreSymbols = [
+	'*',
+	"*'",
+	'*1',
+	'*2',
+	'*3',
+	'*agent*',
+	'*allow-unresolved-vars*',
+	'*assert*',
+	'*clojure-version*',
+	'*command-line-args*',
+	'*compile-files*',
+	'*compile-path*',
+	'*compiler-options*',
+	'*data-readers*',
+	'*default-data-reader-fn*',
+	'*e',
+	'*err*',
+	'*file*',
+	'*flush-on-newline*',
+	'*fn-loader*',
+	'*in*',
+	'*math-context*',
+	'*ns*',
+	'*out*',
+	'*print-dup*',
+	'*print-length*',
+	'*print-level*',
+	'*print-meta*',
+	'*print-namespace-maps*',
+	'*print-readably*',
+	'*read-eval*',
+	'*reader-resolver*',
+	'*source-path*',
+	'*suppress-read*',
+	'*unchecked-math*',
+	'*use-context-classloader*',
+	'*verbose-defrecords*',
+	'*warn-on-reflection*',
+	'+',
+	"+'",
+	'-',
+	"-'",
+	'->',
+	'->>',
+	'->ArrayChunk',
+	'->Eduction',
+	'->Vec',
+	'->VecNode',
+	'->VecSeq',
+	'-cache-protocol-fn',
+	'-reset-methods',
+	'..',
+	'/',
+	'<',
+	'<=',
+	'=',
+	'==',
+	'>',
+	'>=',
+	'EMPTY-NODE',
+	'Inst',
+	'StackTraceElement->vec',
+	'Throwable->map',
+	'accessor',
+	'aclone',
+	'add-classpath',
+	'add-watch',
+	'agent',
+	'agent-error',
+	'agent-errors',
+	'aget',
+	'alength',
+	'alias',
+	'all-ns',
+	'alter',
+	'alter-meta!',
+	'alter-var-root',
+	'amap',
+	'ancestors',
+	'and',
+	'any?',
+	'apply',
+	'areduce',
+	'array-map',
+	'as->',
+	'aset',
+	'aset-boolean',
+	'aset-byte',
+	'aset-char',
+	'aset-double',
+	'aset-float',
+	'aset-int',
+	'aset-long',
+	'aset-short',
+	'assert',
+	'assoc',
+	'assoc!',
+	'assoc-in',
+	'associative?',
+	'atom',
+	'await',
+	'await-for',
+	'await1',
+	'bases',
+	'bean',
+	'bigdec',
+	'bigint',
+	'biginteger',
+	'binding',
+	'bit-and',
+	'bit-and-not',
+	'bit-clear',
+	'bit-flip',
+	'bit-not',
+	'bit-or',
+	'bit-set',
+	'bit-shift-left',
+	'bit-shift-right',
+	'bit-test',
+	'bit-xor',
+	'boolean',
+	'boolean-array',
+	'boolean?',
+	'booleans',
+	'bound-fn',
+	'bound-fn*',
+	'bound?',
+	'bounded-count',
+	'butlast',
+	'byte',
+	'byte-array',
+	'bytes',
+	'bytes?',
+	'case',
+	'cast',
+	'cat',
+	'char',
+	'char-array',
+	'char-escape-string',
+	'char-name-string',
+	'char?',
+	'chars',
+	'chunk',
+	'chunk-append',
+	'chunk-buffer',
+	'chunk-cons',
+	'chunk-first',
+	'chunk-next',
+	'chunk-rest',
+	'chunked-seq?',
+	'class',
+	'class?',
+	'clear-agent-errors',
+	'clojure-version',
+	'coll?',
+	'comment',
+	'commute',
+	'comp',
+	'comparator',
+	'compare',
+	'compare-and-set!',
+	'compile',
+	'complement',
+	'completing',
+	'concat',
+	'cond',
+	'cond->',
+	'cond->>',
+	'condp',
+	'conj',
+	'conj!',
+	'cons',
+	'constantly',
+	'construct-proxy',
+	'contains?',
+	'count',
+	'counted?',
+	'create-ns',
+	'create-struct',
+	'cycle',
+	'dec',
+	"dec'",
+	'decimal?',
+	'declare',
+	'dedupe',
+	'default-data-readers',
+	'definline',
+	'definterface',
+	'defmacro',
+	'defmethod',
+	'defmulti',
+	'defn',
+	'defn-',
+	'defonce',
+	'defprotocol',
+	'defrecord',
+	'defstruct',
+	'deftype',
+	'delay',
+	'delay?',
+	'deliver',
+	'denominator',
+	'deref',
+	'derive',
+	'descendants',
+	'destructure',
+	'disj',
+	'disj!',
+	'dissoc',
+	'dissoc!',
+	'distinct',
+	'distinct?',
+	'doall',
+	'dorun',
+	'doseq',
+	'dosync',
+	'dotimes',
+	'doto',
+	'double',
+	'double-array',
+	'double?',
+	'doubles',
+	'drop',
+	'drop-last',
+	'drop-while',
+	'eduction',
+	'empty',
+	'empty?',
+	'ensure',
+	'ensure-reduced',
+	'enumeration-seq',
+	'error-handler',
+	'error-mode',
+	'eval',
+	'even?',
+	'every-pred',
+	'every?',
+	'ex-data',
+	'ex-info',
+	'extend',
+	'extend-protocol',
+	'extend-type',
+	'extenders',
+	'extends?',
+	'false?',
+	'ffirst',
+	'file-seq',
+	'filter',
+	'filterv',
+	'find',
+	'find-keyword',
+	'find-ns',
+	'find-protocol-impl',
+	'find-protocol-method',
+	'find-var',
+	'first',
+	'flatten',
+	'float',
+	'float-array',
+	'float?',
+	'floats',
+	'flush',
+	'fn',
+	'fn?',
+	'fnext',
+	'fnil',
+	'for',
+	'force',
+	'format',
+	'frequencies',
+	'future',
+	'future-call',
+	'future-cancel',
+	'future-cancelled?',
+	'future-done?',
+	'future?',
+	'gen-class',
+	'gen-interface',
+	'gensym',
+	'get',
+	'get-in',
+	'get-method',
+	'get-proxy-class',
+	'get-thread-bindings',
+	'get-validator',
+	'group-by',
+	'halt-when',
+	'hash',
+	'hash-combine',
+	'hash-map',
+	'hash-ordered-coll',
+	'hash-set',
+	'hash-unordered-coll',
+	'ident?',
+	'identical?',
+	'identity',
+	'if-let',
+	'if-not',
+	'if-some',
+	'ifn?',
+	'import',
+	'in-ns',
+	'inc',
+	"inc'",
+	'indexed?',
+	'init-proxy',
+	'inst-ms',
+	'inst-ms*',
+	'inst?',
+	'instance?',
+	'int',
+	'int-array',
+	'int?',
+	'integer?',
+	'interleave',
+	'intern',
+	'interpose',
+	'into',
+	'into-array',
+	'ints',
+	'io!',
+	'isa?',
+	'iterate',
+	'iterator-seq',
+	'juxt',
+	'keep',
+	'keep-indexed',
+	'key',
+	'keys',
+	'keyword',
+	'keyword?',
+	'last',
+	'lazy-cat',
+	'lazy-seq',
+	'let',
+	'letfn',
+	'line-seq',
+	'list',
+	'list*',
+	'list?',
+	'load',
+	'load-file',
+	'load-reader',
+	'load-string',
+	'loaded-libs',
+	'locking',
+	'long',
+	'long-array',
+	'longs',
+	'loop',
+	'macroexpand',
+	'macroexpand-1',
+	'make-array',
+	'make-hierarchy',
+	'map',
+	'map-entry?',
+	'map-indexed',
+	'map?',
+	'mapcat',
+	'mapv',
+	'max',
+	'max-key',
+	'memfn',
+	'memoize',
+	'merge',
+	'merge-with',
+	'meta',
+	'method-sig',
+	'methods',
+	'min',
+	'min-key',
+	'mix-collection-hash',
+	'mod',
+	'munge',
+	'name',
+	'namespace',
+	'namespace-munge',
+	'nat-int?',
+	'neg-int?',
+	'neg?',
+	'newline',
+	'next',
+	'nfirst',
+	'nil?',
+	'nnext',
+	'not',
+	'not-any?',
+	'not-empty',
+	'not-every?',
+	'not=',
+	'ns',
+	'ns-aliases',
+	'ns-imports',
+	'ns-interns',
+	'ns-map',
+	'ns-name',
+	'ns-publics',
+	'ns-refers',
+	'ns-resolve',
+	'ns-unalias',
+	'ns-unmap',
+	'nth',
+	'nthnext',
+	'nthrest',
+	'num',
+	'number?',
+	'numerator',
+	'object-array',
+	'odd?',
+	'or',
+	'parents',
+	'partial',
+	'partition',
+	'partition-all',
+	'partition-by',
+	'pcalls',
+	'peek',
+	'persistent!',
+	'pmap',
+	'pop',
+	'pop!',
+	'pop-thread-bindings',
+	'pos-int?',
+	'pos?',
+	'pr',
+	'pr-str',
+	'prefer-method',
+	'prefers',
+	'primitives-classnames',
+	'print',
+	'print-ctor',
+	'print-dup',
+	'print-method',
+	'print-simple',
+	'print-str',
+	'printf',
+	'println',
+	'println-str',
+	'prn',
+	'prn-str',
+	'promise',
+	'proxy',
+	'proxy-call-with-super',
+	'proxy-mappings',
+	'proxy-name',
+	'proxy-super',
+	'push-thread-bindings',
+	'pvalues',
+	'qualified-ident?',
+	'qualified-keyword?',
+	'qualified-symbol?',
+	'quot',
+	'rand',
+	'rand-int',
+	'rand-nth',
+	'random-sample',
+	'range',
+	'ratio?',
+	'rational?',
+	'rationalize',
+	're-find',
+	're-groups',
+	're-matcher',
+	're-matches',
+	're-pattern',
+	're-seq',
+	'read',
+	'read-line',
+	'read-string',
+	'reader-conditional',
+	'reader-conditional?',
+	'realized?',
+	'record?',
+	'reduce',
+	'reduce-kv',
+	'reduced',
+	'reduced?',
+	'reductions',
+	'ref',
+	'ref-history-count',
+	'ref-max-history',
+	'ref-min-history',
+	'ref-set',
+	'refer',
+	'refer-clojure',
+	'reify',
+	'release-pending-sends',
+	'rem',
+	'remove',
+	'remove-all-methods',
+	'remove-method',
+	'remove-ns',
+	'remove-watch',
+	'repeat',
+	'repeatedly',
+	'replace',
+	'replicate',
+	'require',
+	'reset!',
+	'reset-meta!',
+	'reset-vals!',
+	'resolve',
+	'rest',
+	'restart-agent',
+	'resultset-seq',
+	'reverse',
+	'reversible?',
+	'rseq',
+	'rsubseq',
+	'run!',
+	'satisfies?',
+	'second',
+	'select-keys',
+	'send',
+	'send-off',
+	'send-via',
+	'seq',
+	'seq?',
+	'seqable?',
+	'seque',
+	'sequence',
+	'sequential?',
+	'set',
+	'set-agent-send-executor!',
+	'set-agent-send-off-executor!',
+	'set-error-handler!',
+	'set-error-mode!',
+	'set-validator!',
+	'set?',
+	'short',
+	'short-array',
+	'shorts',
+	'shuffle',
+	'shutdown-agents',
+	'simple-ident?',
+	'simple-keyword?',
+	'simple-symbol?',
+	'slurp',
+	'some',
+	'some->',
+	'some->>',
+	'some-fn',
+	'some?',
+	'sort',
+	'sort-by',
+	'sorted-map',
+	'sorted-map-by',
+	'sorted-set',
+	'sorted-set-by',
+	'sorted?',
+	'special-symbol?',
+	'spit',
+	'split-at',
+	'split-with',
+	'str',
+	'string?',
+	'struct',
+	'struct-map',
+	'subs',
+	'subseq',
+	'subvec',
+	'supers',
+	'swap!',
+	'swap-vals!',
+	'symbol',
+	'symbol?',
+	'sync',
+	'tagged-literal',
+	'tagged-literal?',
+	'take',
+	'take-last',
+	'take-nth',
+	'take-while',
+	'test',
+	'the-ns',
+	'thread-bound?',
+	'time',
+	'to-array',
+	'to-array-2d',
+	'trampoline',
+	'transduce',
+	'transient',
+	'tree-seq',
+	'true?',
+	'type',
+	'unchecked-add',
+	'unchecked-add-int',
+	'unchecked-byte',
+	'unchecked-char',
+	'unchecked-dec',
+	'unchecked-dec-int',
+	'unchecked-divide-int',
+	'unchecked-double',
+	'unchecked-float',
+	'unchecked-inc',
+	'unchecked-inc-int',
+	'unchecked-int',
+	'unchecked-long',
+	'unchecked-multiply',
+	'unchecked-multiply-int',
+	'unchecked-negate',
+	'unchecked-negate-int',
+	'unchecked-remainder-int',
+	'unchecked-short',
+	'unchecked-subtract',
+	'unchecked-subtract-int',
+	'underive',
+	'unquote',
+	'unquote-splicing',
+	'unreduced',
+	'unsigned-bit-shift-right',
+	'update',
+	'update-in',
+	'update-proxy',
+	'uri?',
+	'use',
+	'uuid?',
+	'val',
+	'vals',
+	'var-get',
+	'var-set',
+	'var?',
+	'vary-meta',
+	'vec',
+	'vector',
+	'vector-of',
+	'vector?',
+	'volatile!',
+	'volatile?',
+	'vreset!',
+	'vswap!',
+	'when',
+	'when-first',
+	'when-let',
+	'when-not',
+	'when-some',
+	'while',
+	'with-bindings',
+	'with-bindings*',
+	'with-in-str',
+	'with-loading-context',
+	'with-local-vars',
+	'with-meta',
+	'with-open',
+	'with-out-str',
+	'with-precision',
+	'with-redefs',
+	'with-redefs-fn',
+	'xml-seq',
+	'zero?',
+	'zipmap'
+];
+
+function createTestCases(specialForms: string[], type: string): ITestItem[] {
+	const testCases = [];
+
+	for (const specialForm of specialForms) {
+		testCases.push({
+			line: `${specialForm}`,
+			tokens: [{ startIndex: 0, type: `${type}.clj` }]
+		});
+	}
+
+	return testCases;
+}
+
+testTokenization('clojure', [
+	// special forms
+	createTestCases(specialForms, 'keyword'),
+
+	// core symbols
+	createTestCases(coreSymbols, 'keyword'),
+
+	// atoms
+	createTestCases(['false', 'nil', 'true'], 'constant'),
+
+	// keywords
+	createTestCases([':foo', '::bar', ':foo/bar', ':foo.bar/baz'], 'constant'),
+
+	// numbers
+	createTestCases(
+		[
+			'42',
+			'+42',
+			'-421',
+			'42N',
+			'+42N',
+			'-42N',
+			'0.42',
+			'+0.42',
+			'-0.42',
+			'42M',
+			'+42M',
+			'-42M',
+			'42.42M',
+			'+42.42M',
+			'-42.42M',
+			'1/42',
+			'+1/42',
+			'-1/42',
+			'0x42af',
+			'+0x42af',
+			'-0x42af',
+			'0x42AF',
+			'+0x42AF',
+			'-0x42AF',
+			'1e2',
+			'1e+2',
+			'1e-2',
+			'+1e2',
+			'+1e+2',
+			'+1e-2',
+			'-1e2',
+			'-1e+2',
+			'-1e-2',
+			'-1.0e2',
+			'-0.1e+2',
+			'-1.01e-2',
+			'1E2',
+			'1E+2',
+			'1E-2',
+			'+1E2',
+			'+1E+2',
+			'+1E-2',
+			'-1E2',
+			'-1E+2',
+			'-1E-2',
+			'-1.0E2',
+			'-0.1E+2',
+			'-1.01E-2',
+			'2r101010',
+			'+2r101010',
+			'-2r101010',
+			'2r101010',
+			'+2r101010',
+			'-2r101010',
+			'8r52',
+			'+8r52',
+			'-8r52',
+			'36rhello',
+			'+36rhello',
+			'-36rhello',
+			'36rz',
+			'+36rz',
+			'-36rz',
+			'36rZ',
+			'+36rZ',
+			'-36rZ'
+		],
+		'number'
+	),
+
+	// characters
+	createTestCases(
+		[
+			'\\1',
+			'\\a',
+			'\\#',
+			'\\\\',
+			'\\"',
+			'\\(',
+			'\\A',
+			'\\backspace',
+			'\\formfeed',
+			'\\newline',
+			'\\space',
+			'\\return',
+			'\\tab',
+			'\\o123',
+			'\\u1000',
+			'\\uAaAa',
+			'\\u9F9F'
+		],
+		'string'
+	),
+
+	// strings
+	[
+		{
+			line: '"I\'m a little teapot."',
+			tokens: [{ startIndex: 0, type: 'string.clj' }]
+		},
+		{
+			line: '"I\'m a \\"little\\" teapot."',
+			tokens: [
+				{ startIndex: 0, type: 'string.clj' },
+				{ startIndex: 7, type: 'string.escape.clj' },
+				{ startIndex: 9, type: 'string.clj' },
+				{ startIndex: 15, type: 'string.escape.clj' },
+				{ startIndex: 17, type: 'string.clj' }
+			]
+		}
+	],
+
+	// multi-line strings
+	[
+		{
+			line: '"I\'m',
+			tokens: [{ startIndex: 0, type: 'string.clj' }]
+		},
+		{
+			line: '\\"a little\\"',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.clj' },
+				{ startIndex: 2, type: 'string.clj' },
+				{ startIndex: 10, type: 'string.escape.clj' }
+			]
+		},
+		{
+			line: 'teapot."',
+			tokens: [{ startIndex: 0, type: 'string.clj' }]
+		}
+	],
+
+	// strings with other escapes in them (\" \' \\ \b \f \n \r \t)
+	[
+		{
+			line: '"the escape \\" \\\' \\\\ \\b \\f \\n \\r \\t characters"',
+			tokens: [
+				{ startIndex: 0, type: 'string.clj' },
+				{ startIndex: 12, type: 'string.escape.clj' },
+				{ startIndex: 14, type: 'string.clj' },
+				{ startIndex: 15, type: 'string.escape.clj' },
+				{ startIndex: 17, type: 'string.clj' },
+				{ startIndex: 18, type: 'string.escape.clj' },
+				{ startIndex: 20, type: 'string.clj' },
+				{ startIndex: 21, type: 'string.escape.clj' },
+				{ startIndex: 23, type: 'string.clj' },
+				{ startIndex: 24, type: 'string.escape.clj' },
+				{ startIndex: 26, type: 'string.clj' },
+				{ startIndex: 27, type: 'string.escape.clj' },
+				{ startIndex: 29, type: 'string.clj' },
+				{ startIndex: 30, type: 'string.escape.clj' },
+				{ startIndex: 32, type: 'string.clj' },
+				{ startIndex: 33, type: 'string.escape.clj' },
+				{ startIndex: 35, type: 'string.clj' }
+			]
+		}
+	],
+
+	// comments
+	createTestCases(['; this is an in-line comment.', ';; this is a line comment.'], 'comment'),
+
+	// `comment`
+	[
+		{
+			line: '(comment)',
+			tokens: [{ startIndex: 0, type: 'comment.clj' }]
+		},
+		{
+			line: 'foo :bar 42',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.clj' },
+				{ startIndex: 3, type: 'white.clj' },
+				{ startIndex: 4, type: 'constant.clj' },
+				{ startIndex: 8, type: 'white.clj' },
+				{ startIndex: 9, type: 'number.clj' }
+			]
+		},
+		{
+			line: '(comment (foo [bar :baz 1 "qux"]))',
+			tokens: [{ startIndex: 0, type: 'comment.clj' }]
+		},
+		{
+			line: '(comments foo bar)',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.parenthesis.clj' },
+				{ startIndex: 1, type: 'identifier.clj' },
+				{ startIndex: 9, type: 'white.clj' },
+				{ startIndex: 10, type: 'identifier.clj' },
+				{ startIndex: 13, type: 'white.clj' },
+				{ startIndex: 14, type: 'identifier.clj' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.clj' }
+			]
+		},
+		{
+			line: '(comment6 foo bar)',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.parenthesis.clj' },
+				{ startIndex: 1, type: 'identifier.clj' },
+				{ startIndex: 9, type: 'white.clj' },
+				{ startIndex: 10, type: 'identifier.clj' },
+				{ startIndex: 13, type: 'white.clj' },
+				{ startIndex: 14, type: 'identifier.clj' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.clj' }
+			]
+		},
+		{
+			line: '(comment foo',
+			tokens: [{ startIndex: 0, type: 'comment.clj' }]
+		},
+		{
+			line: 'foo',
+			tokens: [{ startIndex: 0, type: 'comment.clj' }]
+		}
+	],
+
+	// reader macro characters
+	createTestCases(['#', '@', '^', '`', '~', "'"], 'meta'),
+
+	// treat comma as whitespace
+	[
+		{
+			line: ', foo, :bar, "one", 2, ',
+			tokens: [
+				{ startIndex: 0, type: 'white.clj' },
+				{ startIndex: 2, type: 'identifier.clj' },
+				{ startIndex: 5, type: 'white.clj' },
+				{ startIndex: 7, type: 'constant.clj' },
+				{ startIndex: 11, type: 'white.clj' },
+				{ startIndex: 13, type: 'string.clj' },
+				{ startIndex: 18, type: 'white.clj' },
+				{ startIndex: 20, type: 'number.clj' },
+				{ startIndex: 21, type: 'white.clj' }
+			]
+		}
+	]
+]);

monaco-languages/src/clojure/clojure.ts

@@ -0,0 +1,792 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: ';;'
+	},
+
+	brackets: [
+		['[', ']'],
+		['(', ')'],
+		['{', '}']
+	],
+
+	autoClosingPairs: [
+		{ open: '[', close: ']' },
+		{ open: '"', close: '"' },
+		{ open: '(', close: ')' },
+		{ open: '{', close: '}' }
+	],
+
+	surroundingPairs: [
+		{ open: '[', close: ']' },
+		{ open: '"', close: '"' },
+		{ open: '(', close: ')' },
+		{ open: '{', close: '}' }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.clj',
+
+	brackets: [
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '{', close: '}', token: 'delimiter.curly' }
+	],
+
+	constants: ['true', 'false', 'nil'],
+
+	// delimiters: /[\\\[\]\s"#'(),;@^`{}~]|$/,
+
+	numbers:
+		/^(?:[+\-]?\d+(?:(?:N|(?:[eE][+\-]?\d+))|(?:\.?\d*(?:M|(?:[eE][+\-]?\d+))?)|\/\d+|[xX][0-9a-fA-F]+|r[0-9a-zA-Z]+)?(?=[\\\[\]\s"#'(),;@^`{}~]|$))/,
+
+	characters:
+		/^(?:\\(?:backspace|formfeed|newline|return|space|tab|o[0-7]{3}|u[0-9A-Fa-f]{4}|x[0-9A-Fa-f]{4}|.)?(?=[\\\[\]\s"(),;@^`{}~]|$))/,
+
+	escapes: /^\\(?:["'\\bfnrt]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// simple-namespace := /^[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*/
+	// simple-symbol    := /^(?:\/|[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)/
+	// qualified-symbol := (<simple-namespace>(<.><simple-namespace>)*</>)?<simple-symbol>
+	qualifiedSymbols:
+		/^(?:(?:[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*(?:\.[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)*\/)?(?:\/|[^\\\/\[\]\d\s"#'(),;@^`{}~][^\\\[\]\s"(),;@^`{}~]*)*(?=[\\\[\]\s"(),;@^`{}~]|$))/,
+
+	specialForms: [
+		'.',
+		'catch',
+		'def',
+		'do',
+		'if',
+		'monitor-enter',
+		'monitor-exit',
+		'new',
+		'quote',
+		'recur',
+		'set!',
+		'throw',
+		'try',
+		'var'
+	],
+
+	coreSymbols: [
+		'*',
+		"*'",
+		'*1',
+		'*2',
+		'*3',
+		'*agent*',
+		'*allow-unresolved-vars*',
+		'*assert*',
+		'*clojure-version*',
+		'*command-line-args*',
+		'*compile-files*',
+		'*compile-path*',
+		'*compiler-options*',
+		'*data-readers*',
+		'*default-data-reader-fn*',
+		'*e',
+		'*err*',
+		'*file*',
+		'*flush-on-newline*',
+		'*fn-loader*',
+		'*in*',
+		'*math-context*',
+		'*ns*',
+		'*out*',
+		'*print-dup*',
+		'*print-length*',
+		'*print-level*',
+		'*print-meta*',
+		'*print-namespace-maps*',
+		'*print-readably*',
+		'*read-eval*',
+		'*reader-resolver*',
+		'*source-path*',
+		'*suppress-read*',
+		'*unchecked-math*',
+		'*use-context-classloader*',
+		'*verbose-defrecords*',
+		'*warn-on-reflection*',
+		'+',
+		"+'",
+		'-',
+		"-'",
+		'->',
+		'->>',
+		'->ArrayChunk',
+		'->Eduction',
+		'->Vec',
+		'->VecNode',
+		'->VecSeq',
+		'-cache-protocol-fn',
+		'-reset-methods',
+		'..',
+		'/',
+		'<',
+		'<=',
+		'=',
+		'==',
+		'>',
+		'>=',
+		'EMPTY-NODE',
+		'Inst',
+		'StackTraceElement->vec',
+		'Throwable->map',
+		'accessor',
+		'aclone',
+		'add-classpath',
+		'add-watch',
+		'agent',
+		'agent-error',
+		'agent-errors',
+		'aget',
+		'alength',
+		'alias',
+		'all-ns',
+		'alter',
+		'alter-meta!',
+		'alter-var-root',
+		'amap',
+		'ancestors',
+		'and',
+		'any?',
+		'apply',
+		'areduce',
+		'array-map',
+		'as->',
+		'aset',
+		'aset-boolean',
+		'aset-byte',
+		'aset-char',
+		'aset-double',
+		'aset-float',
+		'aset-int',
+		'aset-long',
+		'aset-short',
+		'assert',
+		'assoc',
+		'assoc!',
+		'assoc-in',
+		'associative?',
+		'atom',
+		'await',
+		'await-for',
+		'await1',
+		'bases',
+		'bean',
+		'bigdec',
+		'bigint',
+		'biginteger',
+		'binding',
+		'bit-and',
+		'bit-and-not',
+		'bit-clear',
+		'bit-flip',
+		'bit-not',
+		'bit-or',
+		'bit-set',
+		'bit-shift-left',
+		'bit-shift-right',
+		'bit-test',
+		'bit-xor',
+		'boolean',
+		'boolean-array',
+		'boolean?',
+		'booleans',
+		'bound-fn',
+		'bound-fn*',
+		'bound?',
+		'bounded-count',
+		'butlast',
+		'byte',
+		'byte-array',
+		'bytes',
+		'bytes?',
+		'case',
+		'cast',
+		'cat',
+		'char',
+		'char-array',
+		'char-escape-string',
+		'char-name-string',
+		'char?',
+		'chars',
+		'chunk',
+		'chunk-append',
+		'chunk-buffer',
+		'chunk-cons',
+		'chunk-first',
+		'chunk-next',
+		'chunk-rest',
+		'chunked-seq?',
+		'class',
+		'class?',
+		'clear-agent-errors',
+		'clojure-version',
+		'coll?',
+		'comment',
+		'commute',
+		'comp',
+		'comparator',
+		'compare',
+		'compare-and-set!',
+		'compile',
+		'complement',
+		'completing',
+		'concat',
+		'cond',
+		'cond->',
+		'cond->>',
+		'condp',
+		'conj',
+		'conj!',
+		'cons',
+		'constantly',
+		'construct-proxy',
+		'contains?',
+		'count',
+		'counted?',
+		'create-ns',
+		'create-struct',
+		'cycle',
+		'dec',
+		"dec'",
+		'decimal?',
+		'declare',
+		'dedupe',
+		'default-data-readers',
+		'definline',
+		'definterface',
+		'defmacro',
+		'defmethod',
+		'defmulti',
+		'defn',
+		'defn-',
+		'defonce',
+		'defprotocol',
+		'defrecord',
+		'defstruct',
+		'deftype',
+		'delay',
+		'delay?',
+		'deliver',
+		'denominator',
+		'deref',
+		'derive',
+		'descendants',
+		'destructure',
+		'disj',
+		'disj!',
+		'dissoc',
+		'dissoc!',
+		'distinct',
+		'distinct?',
+		'doall',
+		'dorun',
+		'doseq',
+		'dosync',
+		'dotimes',
+		'doto',
+		'double',
+		'double-array',
+		'double?',
+		'doubles',
+		'drop',
+		'drop-last',
+		'drop-while',
+		'eduction',
+		'empty',
+		'empty?',
+		'ensure',
+		'ensure-reduced',
+		'enumeration-seq',
+		'error-handler',
+		'error-mode',
+		'eval',
+		'even?',
+		'every-pred',
+		'every?',
+		'ex-data',
+		'ex-info',
+		'extend',
+		'extend-protocol',
+		'extend-type',
+		'extenders',
+		'extends?',
+		'false?',
+		'ffirst',
+		'file-seq',
+		'filter',
+		'filterv',
+		'find',
+		'find-keyword',
+		'find-ns',
+		'find-protocol-impl',
+		'find-protocol-method',
+		'find-var',
+		'first',
+		'flatten',
+		'float',
+		'float-array',
+		'float?',
+		'floats',
+		'flush',
+		'fn',
+		'fn?',
+		'fnext',
+		'fnil',
+		'for',
+		'force',
+		'format',
+		'frequencies',
+		'future',
+		'future-call',
+		'future-cancel',
+		'future-cancelled?',
+		'future-done?',
+		'future?',
+		'gen-class',
+		'gen-interface',
+		'gensym',
+		'get',
+		'get-in',
+		'get-method',
+		'get-proxy-class',
+		'get-thread-bindings',
+		'get-validator',
+		'group-by',
+		'halt-when',
+		'hash',
+		'hash-combine',
+		'hash-map',
+		'hash-ordered-coll',
+		'hash-set',
+		'hash-unordered-coll',
+		'ident?',
+		'identical?',
+		'identity',
+		'if-let',
+		'if-not',
+		'if-some',
+		'ifn?',
+		'import',
+		'in-ns',
+		'inc',
+		"inc'",
+		'indexed?',
+		'init-proxy',
+		'inst-ms',
+		'inst-ms*',
+		'inst?',
+		'instance?',
+		'int',
+		'int-array',
+		'int?',
+		'integer?',
+		'interleave',
+		'intern',
+		'interpose',
+		'into',
+		'into-array',
+		'ints',
+		'io!',
+		'isa?',
+		'iterate',
+		'iterator-seq',
+		'juxt',
+		'keep',
+		'keep-indexed',
+		'key',
+		'keys',
+		'keyword',
+		'keyword?',
+		'last',
+		'lazy-cat',
+		'lazy-seq',
+		'let',
+		'letfn',
+		'line-seq',
+		'list',
+		'list*',
+		'list?',
+		'load',
+		'load-file',
+		'load-reader',
+		'load-string',
+		'loaded-libs',
+		'locking',
+		'long',
+		'long-array',
+		'longs',
+		'loop',
+		'macroexpand',
+		'macroexpand-1',
+		'make-array',
+		'make-hierarchy',
+		'map',
+		'map-entry?',
+		'map-indexed',
+		'map?',
+		'mapcat',
+		'mapv',
+		'max',
+		'max-key',
+		'memfn',
+		'memoize',
+		'merge',
+		'merge-with',
+		'meta',
+		'method-sig',
+		'methods',
+		'min',
+		'min-key',
+		'mix-collection-hash',
+		'mod',
+		'munge',
+		'name',
+		'namespace',
+		'namespace-munge',
+		'nat-int?',
+		'neg-int?',
+		'neg?',
+		'newline',
+		'next',
+		'nfirst',
+		'nil?',
+		'nnext',
+		'not',
+		'not-any?',
+		'not-empty',
+		'not-every?',
+		'not=',
+		'ns',
+		'ns-aliases',
+		'ns-imports',
+		'ns-interns',
+		'ns-map',
+		'ns-name',
+		'ns-publics',
+		'ns-refers',
+		'ns-resolve',
+		'ns-unalias',
+		'ns-unmap',
+		'nth',
+		'nthnext',
+		'nthrest',
+		'num',
+		'number?',
+		'numerator',
+		'object-array',
+		'odd?',
+		'or',
+		'parents',
+		'partial',
+		'partition',
+		'partition-all',
+		'partition-by',
+		'pcalls',
+		'peek',
+		'persistent!',
+		'pmap',
+		'pop',
+		'pop!',
+		'pop-thread-bindings',
+		'pos-int?',
+		'pos?',
+		'pr',
+		'pr-str',
+		'prefer-method',
+		'prefers',
+		'primitives-classnames',
+		'print',
+		'print-ctor',
+		'print-dup',
+		'print-method',
+		'print-simple',
+		'print-str',
+		'printf',
+		'println',
+		'println-str',
+		'prn',
+		'prn-str',
+		'promise',
+		'proxy',
+		'proxy-call-with-super',
+		'proxy-mappings',
+		'proxy-name',
+		'proxy-super',
+		'push-thread-bindings',
+		'pvalues',
+		'qualified-ident?',
+		'qualified-keyword?',
+		'qualified-symbol?',
+		'quot',
+		'rand',
+		'rand-int',
+		'rand-nth',
+		'random-sample',
+		'range',
+		'ratio?',
+		'rational?',
+		'rationalize',
+		're-find',
+		're-groups',
+		're-matcher',
+		're-matches',
+		're-pattern',
+		're-seq',
+		'read',
+		'read-line',
+		'read-string',
+		'reader-conditional',
+		'reader-conditional?',
+		'realized?',
+		'record?',
+		'reduce',
+		'reduce-kv',
+		'reduced',
+		'reduced?',
+		'reductions',
+		'ref',
+		'ref-history-count',
+		'ref-max-history',
+		'ref-min-history',
+		'ref-set',
+		'refer',
+		'refer-clojure',
+		'reify',
+		'release-pending-sends',
+		'rem',
+		'remove',
+		'remove-all-methods',
+		'remove-method',
+		'remove-ns',
+		'remove-watch',
+		'repeat',
+		'repeatedly',
+		'replace',
+		'replicate',
+		'require',
+		'reset!',
+		'reset-meta!',
+		'reset-vals!',
+		'resolve',
+		'rest',
+		'restart-agent',
+		'resultset-seq',
+		'reverse',
+		'reversible?',
+		'rseq',
+		'rsubseq',
+		'run!',
+		'satisfies?',
+		'second',
+		'select-keys',
+		'send',
+		'send-off',
+		'send-via',
+		'seq',
+		'seq?',
+		'seqable?',
+		'seque',
+		'sequence',
+		'sequential?',
+		'set',
+		'set-agent-send-executor!',
+		'set-agent-send-off-executor!',
+		'set-error-handler!',
+		'set-error-mode!',
+		'set-validator!',
+		'set?',
+		'short',
+		'short-array',
+		'shorts',
+		'shuffle',
+		'shutdown-agents',
+		'simple-ident?',
+		'simple-keyword?',
+		'simple-symbol?',
+		'slurp',
+		'some',
+		'some->',
+		'some->>',
+		'some-fn',
+		'some?',
+		'sort',
+		'sort-by',
+		'sorted-map',
+		'sorted-map-by',
+		'sorted-set',
+		'sorted-set-by',
+		'sorted?',
+		'special-symbol?',
+		'spit',
+		'split-at',
+		'split-with',
+		'str',
+		'string?',
+		'struct',
+		'struct-map',
+		'subs',
+		'subseq',
+		'subvec',
+		'supers',
+		'swap!',
+		'swap-vals!',
+		'symbol',
+		'symbol?',
+		'sync',
+		'tagged-literal',
+		'tagged-literal?',
+		'take',
+		'take-last',
+		'take-nth',
+		'take-while',
+		'test',
+		'the-ns',
+		'thread-bound?',
+		'time',
+		'to-array',
+		'to-array-2d',
+		'trampoline',
+		'transduce',
+		'transient',
+		'tree-seq',
+		'true?',
+		'type',
+		'unchecked-add',
+		'unchecked-add-int',
+		'unchecked-byte',
+		'unchecked-char',
+		'unchecked-dec',
+		'unchecked-dec-int',
+		'unchecked-divide-int',
+		'unchecked-double',
+		'unchecked-float',
+		'unchecked-inc',
+		'unchecked-inc-int',
+		'unchecked-int',
+		'unchecked-long',
+		'unchecked-multiply',
+		'unchecked-multiply-int',
+		'unchecked-negate',
+		'unchecked-negate-int',
+		'unchecked-remainder-int',
+		'unchecked-short',
+		'unchecked-subtract',
+		'unchecked-subtract-int',
+		'underive',
+		'unquote',
+		'unquote-splicing',
+		'unreduced',
+		'unsigned-bit-shift-right',
+		'update',
+		'update-in',
+		'update-proxy',
+		'uri?',
+		'use',
+		'uuid?',
+		'val',
+		'vals',
+		'var-get',
+		'var-set',
+		'var?',
+		'vary-meta',
+		'vec',
+		'vector',
+		'vector-of',
+		'vector?',
+		'volatile!',
+		'volatile?',
+		'vreset!',
+		'vswap!',
+		'when',
+		'when-first',
+		'when-let',
+		'when-not',
+		'when-some',
+		'while',
+		'with-bindings',
+		'with-bindings*',
+		'with-in-str',
+		'with-loading-context',
+		'with-local-vars',
+		'with-meta',
+		'with-open',
+		'with-out-str',
+		'with-precision',
+		'with-redefs',
+		'with-redefs-fn',
+		'xml-seq',
+		'zero?',
+		'zipmap'
+	],
+
+	tokenizer: {
+		root: [
+			// whitespaces and comments
+			{ include: '@whitespace' },
+
+			// numbers
+			[/@numbers/, 'number'],
+
+			// characters
+			[/@characters/, 'string'],
+
+			// strings
+			{ include: '@string' },
+
+			// brackets
+			[/[()\[\]{}]/, '@brackets'],
+
+			// regular expressions
+			[/\/#"(?:\.|(?:")|[^"\n])*"\/g/, 'regexp'],
+
+			// reader macro characters
+			[/[#'@^`~]/, 'meta'],
+
+			// symbols
+			[
+				/@qualifiedSymbols/,
+				{
+					cases: {
+						'^:.+$': 'constant', // Clojure keywords (e.g., `:foo/bar`)
+						'@specialForms': 'keyword',
+						'@coreSymbols': 'keyword',
+						'@constants': 'constant',
+						'@default': 'identifier'
+					}
+				}
+			]
+		],
+
+		whitespace: [
+			[/[\s,]+/, 'white'],
+			[/;.*$/, 'comment'],
+			[/\(comment\b/, 'comment', '@comment']
+		],
+
+		comment: [
+			[/\(/, 'comment', '@push'],
+			[/\)/, 'comment', '@pop'],
+			[/[^()]/, 'comment']
+		],
+
+		string: [[/"/, 'string', '@multiLineString']],
+
+		multiLineString: [
+			[/"/, 'string', '@popall'],
+			[/@escapes/, 'string.escape'],
+			[/./, 'string']
+		]
+	}
+};

monaco-languages/src/coffee/coffee.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'coffeescript',
+	extensions: ['.coffee'],
+	aliases: ['CoffeeScript', 'coffeescript', 'coffee'],
+	mimetypes: ['text/x-coffeescript', 'text/coffeescript'],
+	loader: () => import('./coffee')
+});

monaco-languages/src/coffee/coffee.ts

@@ -0,0 +1,258 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	wordPattern:
+		/(-?\d*\.\d\w*)|([^\`\~\!\@\#%\^\&\*\(\)\=\$\-\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		blockComment: ['###', '###'],
+		lineComment: '#'
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*#region\\b'),
+			end: new RegExp('^\\s*#endregion\\b')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	ignoreCase: true,
+	tokenPostfix: '.coffee',
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' }
+	],
+
+	regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/,
+
+	keywords: [
+		'and',
+		'or',
+		'is',
+		'isnt',
+		'not',
+		'on',
+		'yes',
+		'@',
+		'no',
+		'off',
+		'true',
+		'false',
+		'null',
+		'this',
+		'new',
+		'delete',
+		'typeof',
+		'in',
+		'instanceof',
+		'return',
+		'throw',
+		'break',
+		'continue',
+		'debugger',
+		'if',
+		'else',
+		'switch',
+		'for',
+		'while',
+		'do',
+		'try',
+		'catch',
+		'finally',
+		'class',
+		'extends',
+		'super',
+		'undefined',
+		'then',
+		'unless',
+		'until',
+		'loop',
+		'of',
+		'by',
+		'when'
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?&%|+\-*\/\^\.,\:]+/,
+	escapes: /\\(?:[abfnrtv\\"'$]|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[/\@[a-zA-Z_]\w*/, 'variable.predefined'],
+			[
+				/[a-zA-Z_]\w*/,
+				{
+					cases: {
+						this: 'variable.predefined',
+						'@keywords': { token: 'keyword.$0' },
+						'@default': ''
+					}
+				}
+			],
+
+			// whitespace
+			[/[ \t\r\n]+/, ''],
+
+			// Comments
+			[/###/, 'comment', '@comment'],
+			[/#.*$/, 'comment'],
+
+			// regular expressions
+			['///', { token: 'regexp', next: '@hereregexp' }],
+
+			[/^(\s*)(@regEx)/, ['', 'regexp']],
+			[/(\()(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\,)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\=)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\:)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\[)(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\!)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\&)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\|)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\?)(\s*)(@regEx)/, ['delimiter', '', 'regexp']],
+			[/(\{)(\s*)(@regEx)/, ['@brackets', '', 'regexp']],
+			[/(\;)(\s*)(@regEx)/, ['', '', 'regexp']],
+
+			// delimiters
+			[
+				/}/,
+				{
+					cases: {
+						'$S2==interpolatedstring': {
+							token: 'string',
+							next: '@pop'
+						},
+						'@default': '@brackets'
+					}
+				}
+			],
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d+[eE]([\-+]?\d+)?/, 'number.float'],
+			[/\d+\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/0[0-7]+(?!\d)/, 'number.octal'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[,.]/, 'delimiter'],
+
+			// strings:
+			[/"""/, 'string', '@herestring."""'],
+			[/'''/, 'string', "@herestring.'''"],
+			[
+				/"/,
+				{
+					cases: {
+						'@eos': 'string',
+						'@default': { token: 'string', next: '@string."' }
+					}
+				}
+			],
+			[
+				/'/,
+				{
+					cases: {
+						'@eos': 'string',
+						'@default': { token: 'string', next: "@string.'" }
+					}
+				}
+			]
+		],
+
+		string: [
+			[/[^"'\#\\]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\./, 'string.escape.invalid'],
+			[/\./, 'string.escape.invalid'],
+
+			[
+				/#{/,
+				{
+					cases: {
+						'$S2=="': {
+							token: 'string',
+							next: 'root.interpolatedstring'
+						},
+						'@default': 'string'
+					}
+				}
+			],
+
+			[
+				/["']/,
+				{
+					cases: {
+						'$#==$S2': { token: 'string', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			],
+			[/#/, 'string']
+		],
+
+		herestring: [
+			[
+				/("""|''')/,
+				{
+					cases: {
+						'$1==$S2': { token: 'string', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			],
+			[/[^#\\'"]+/, 'string'],
+			[/['"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\./, 'string.escape.invalid'],
+
+			[/#{/, { token: 'string.quote', next: 'root.interpolatedstring' }],
+			[/#/, 'string']
+		],
+
+		comment: [
+			[/[^#]+/, 'comment'],
+			[/###/, 'comment', '@pop'],
+			[/#/, 'comment']
+		],
+
+		hereregexp: [
+			[/[^\\\/#]+/, 'regexp'],
+			[/\\./, 'regexp'],
+			[/#.*$/, 'comment'],
+			['///[igm]*', { token: 'regexp', next: '@pop' }],
+			[/\//, 'regexp']
+		]
+	}
+};

monaco-languages/src/cpp/cpp.contribution.ts

@@ -0,0 +1,19 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'c',
+	extensions: ['.c', '.h'],
+	aliases: ['C', 'c'],
+	loader: () => import('./cpp')
+});
+registerLanguage({
+	id: 'cpp',
+	extensions: ['.cpp', '.cc', '.cxx', '.hpp', '.hh', '.hxx'],
+	aliases: ['C++', 'Cpp', 'cpp'],
+	loader: () => import('./cpp')
+});

monaco-languages/src/cpp/cpp.test.ts

@@ -0,0 +1,979 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('cpp', [
+	// Keywords
+	[
+		{
+			line: 'int _tmain(int argc, _TCHAR* argv[])',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cpp' },
+				{ startIndex: 10, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 11, type: 'keyword.int.cpp' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'identifier.cpp' },
+				{ startIndex: 19, type: 'delimiter.cpp' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'identifier.cpp' },
+				{ startIndex: 27, type: 'delimiter.cpp' },
+				{ startIndex: 28, type: '' },
+				{ startIndex: 29, type: 'identifier.cpp' },
+				{ startIndex: 33, type: 'delimiter.square.cpp' },
+				{ startIndex: 35, type: 'delimiter.parenthesis.cpp' }
+			]
+		}
+	],
+
+	// Comments - single line
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '/almost a comment',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.cpp' },
+				{ startIndex: 1, type: 'identifier.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.cpp' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '/* //*/ a',
+			tokens: [
+				{ startIndex: 0, type: 'comment.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1 / 2; /* comment',
+			tokens: [
+				{ startIndex: 0, type: 'number.cpp' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.cpp' },
+				{ startIndex: 5, type: 'delimiter.cpp' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'comment.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'int x = 1; // my comment // is a nice one',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.cpp' },
+				{ startIndex: 9, type: 'delimiter.cpp' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'comment.cpp' }
+			]
+		}
+	],
+
+	// Comments - range comment, single line
+	[
+		{
+			line: '/* a simple comment */',
+			tokens: [{ startIndex: 0, type: 'comment.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* a simple comment */ 1;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.cpp' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'number.cpp' },
+				{ startIndex: 32, type: 'delimiter.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* comment */ 1; */',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.cpp' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'number.cpp' },
+				{ startIndex: 23, type: 'delimiter.cpp' },
+				{ startIndex: 24, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /**/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cpp' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.cpp' },
+				{ startIndex: 8, type: 'delimiter.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /*/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cpp' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.cpp' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '12l',
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '34U',
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '55LL',
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '34ul',
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '55llU',
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: "5'5llU",
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: "100'000'000",
+			tokens: [{ startIndex: 0, type: 'number.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: "0x100'aafllU",
+			tokens: [{ startIndex: 0, type: 'number.hex.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: "0342'325",
+			tokens: [{ startIndex: 0, type: 'number.octal.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5F',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5f',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3F',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3f',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3F',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3f',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5L',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5l',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3L',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3l',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3L',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3l',
+			tokens: [{ startIndex: 0, type: 'number.float.cpp' }]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.cpp' },
+				{ startIndex: 1, type: 'delimiter.cpp' },
+				{ startIndex: 2, type: 'number.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.cpp' },
+				{ startIndex: 3, type: 'delimiter.cpp' },
+				{ startIndex: 4, type: 'number.cpp' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.cpp' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.cpp' }
+			]
+		}
+	],
+
+	// Monarch Generated
+	[
+		{
+			line: '#include<iostream>',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.include.cpp' },
+				{ startIndex: 8, type: 'keyword.directive.include.begin.cpp' },
+				{ startIndex: 9, type: 'string.include.identifier.cpp' },
+				{ startIndex: 17, type: 'keyword.directive.include.end.cpp' }
+			]
+		},
+		{
+			line: '#include "/path/to/my/file.h"',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.include.cpp' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.directive.include.begin.cpp' },
+				{ startIndex: 10, type: 'string.include.identifier.cpp' },
+				{ startIndex: 28, type: 'keyword.directive.include.end.cpp' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '#ifdef VAR',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.cpp' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.cpp' }
+			]
+		},
+		{
+			line: '#define SUM(A,B) (A) + (B)',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.cpp' },
+				{ startIndex: 11, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 12, type: 'identifier.cpp' },
+				{ startIndex: 13, type: 'delimiter.cpp' },
+				{ startIndex: 14, type: 'identifier.cpp' },
+				{ startIndex: 15, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 18, type: 'identifier.cpp' },
+				{ startIndex: 19, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'delimiter.cpp' },
+				{ startIndex: 22, type: '' },
+				{ startIndex: 23, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 24, type: 'identifier.cpp' },
+				{ startIndex: 25, type: 'delimiter.parenthesis.cpp' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'int main(int argc, char** argv)',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cpp' },
+				{ startIndex: 8, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 9, type: 'keyword.int.cpp' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'identifier.cpp' },
+				{ startIndex: 17, type: 'delimiter.cpp' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'keyword.char.cpp' },
+				{ startIndex: 23, type: '' },
+				{ startIndex: 26, type: 'identifier.cpp' },
+				{ startIndex: 30, type: 'delimiter.parenthesis.cpp' }
+			]
+		},
+		{
+			line: '{',
+			tokens: [{ startIndex: 0, type: 'delimiter.curly.cpp' }]
+		},
+		{
+			line: '	return 0;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'keyword.return.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.cpp' },
+				{ startIndex: 9, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '}',
+			tokens: [{ startIndex: 0, type: 'delimiter.curly.cpp' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'namespace TestSpace',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.namespace.cpp' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.cpp' }
+			]
+		},
+		{
+			line: '{',
+			tokens: [{ startIndex: 0, type: 'delimiter.curly.cpp' }]
+		},
+		{
+			line: '	using Asdf.CDE;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'keyword.using.cpp' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.cpp' },
+				{ startIndex: 11, type: 'delimiter.cpp' },
+				{ startIndex: 12, type: 'identifier.cpp' },
+				{ startIndex: 15, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '	template <typename T>',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'keyword.template.cpp' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'delimiter.angle.cpp' },
+				{ startIndex: 11, type: 'keyword.typename.cpp' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'identifier.cpp' },
+				{ startIndex: 21, type: 'delimiter.angle.cpp' }
+			]
+		},
+		{
+			line: '	class CoolClass : protected BaseClass',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'keyword.class.cpp' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.cpp' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'delimiter.cpp' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'keyword.protected.cpp' },
+				{ startIndex: 28, type: '' },
+				{ startIndex: 29, type: 'identifier.cpp' }
+			]
+		},
+		{
+			line: '	{',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '		private:',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'keyword.private.cpp' },
+				{ startIndex: 9, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '		',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '		static T field;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'keyword.static.cpp' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'identifier.cpp' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'identifier.cpp' },
+				{ startIndex: 16, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '		',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '		public:',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'keyword.public.cpp' },
+				{ startIndex: 8, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '		',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '		[[deprecated]]',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'annotation.cpp' }
+			]
+		},
+		{
+			line: '		foo method() const override',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'identifier.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'identifier.cpp' },
+				{ startIndex: 12, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'keyword.const.cpp' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'keyword.override.cpp' }
+			]
+		},
+		{
+			line: '		{',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '			auto s = new Bar();',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.auto.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.cpp' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'delimiter.cpp' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'keyword.new.cpp' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'identifier.cpp' },
+				{ startIndex: 19, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 21, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '			',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '			if (s.field) {',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.if.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 7, type: 'identifier.cpp' },
+				{ startIndex: 8, type: 'delimiter.cpp' },
+				{ startIndex: 9, type: 'identifier.cpp' },
+				{ startIndex: 14, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '				for(const auto & b : s.field) {',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'keyword.for.cpp' },
+				{ startIndex: 7, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 8, type: 'keyword.const.cpp' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'keyword.auto.cpp' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'delimiter.cpp' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'identifier.cpp' },
+				{ startIndex: 22, type: '' },
+				{ startIndex: 23, type: 'delimiter.cpp' },
+				{ startIndex: 24, type: '' },
+				{ startIndex: 25, type: 'identifier.cpp' },
+				{ startIndex: 26, type: 'delimiter.cpp' },
+				{ startIndex: 27, type: 'identifier.cpp' },
+				{ startIndex: 32, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 33, type: '' },
+				{ startIndex: 34, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '					break;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 5, type: 'keyword.break.cpp' },
+				{ startIndex: 10, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '				}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '			}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '		}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '		',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '		std::string s = "hello wordld\\n";',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'identifier.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 7, type: 'identifier.cpp' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'identifier.cpp' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'delimiter.cpp' },
+				{ startIndex: 17, type: '' },
+				{ startIndex: 18, type: 'string.cpp' },
+				{ startIndex: 31, type: 'string.escape.cpp' },
+				{ startIndex: 33, type: 'string.cpp' },
+				{ startIndex: 34, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '		',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: "		int number = 123'123'123Ull;",
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'keyword.int.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'identifier.cpp' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'delimiter.cpp' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'number.cpp' },
+				{ startIndex: 29, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '	}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'delimiter.curly.cpp' }
+			]
+		},
+		{
+			line: '}',
+			tokens: [{ startIndex: 0, type: 'delimiter.curly.cpp' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '#endif',
+			tokens: [{ startIndex: 0, type: 'keyword.directive.cpp' }]
+		},
+		{
+			line: '#    ifdef VAR',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.cpp' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'identifier.cpp' }
+			]
+		},
+		{
+			line: '#	define SUM(A,B) (A) + (B)',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.cpp' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'identifier.cpp' },
+				{ startIndex: 12, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 13, type: 'identifier.cpp' },
+				{ startIndex: 14, type: 'delimiter.cpp' },
+				{ startIndex: 15, type: 'identifier.cpp' },
+				{ startIndex: 16, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 17, type: '' },
+				{ startIndex: 18, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 19, type: 'identifier.cpp' },
+				{ startIndex: 20, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'delimiter.cpp' },
+				{ startIndex: 23, type: '' },
+				{ startIndex: 24, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 25, type: 'identifier.cpp' },
+				{ startIndex: 26, type: 'delimiter.parenthesis.cpp' }
+			]
+		},
+		{
+			line: 'uR"V0G0N(abc)V0G0N"def',
+			tokens: [
+				{ startIndex: 0, type: 'string.raw.begin.cpp' },
+				{ startIndex: 9, type: 'string.raw.cpp' },
+				{ startIndex: 12, type: 'string.raw.end.cpp' },
+				{ startIndex: 19, type: 'identifier.cpp' }
+			]
+		}
+	],
+
+	// https://github.com/microsoft/monaco-editor/issues/1951
+	[
+		{
+			line: 'auto sv = R"({ "message": "Hello World" })""\\n"sv;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.auto.cpp' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'identifier.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'delimiter.cpp' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'string.raw.begin.cpp' },
+				{ startIndex: 13, type: 'string.raw.cpp' },
+				{ startIndex: 41, type: 'string.raw.end.cpp' },
+				{ startIndex: 43, type: 'string.cpp' },
+				{ startIndex: 44, type: 'string.escape.cpp' },
+				{ startIndex: 46, type: 'string.cpp' },
+				{ startIndex: 47, type: 'identifier.cpp' },
+				{ startIndex: 49, type: 'delimiter.cpp' }
+			]
+		},
+		{
+			line: '    // This is a comment, not a string',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.cpp' }
+			]
+		}
+	],
+
+	// Annotations
+	[
+		{
+			line: '[[nodiscard]]',
+			tokens: [{ startIndex: 0, type: 'annotation.cpp' }]
+		}
+	],
+	[
+		{
+			// Example from http://eel.is/c++draft/dcl.attr
+			line: '[[using CC: opt(1), debug]]',
+			tokens: [
+				{ startIndex: 0, type: 'annotation.cpp' }, // [[
+				{ startIndex: 2, type: 'keyword.cpp' }, // using
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'annotation.cpp' }, // CC
+				{ startIndex: 10, type: 'delimiter.cpp' }, // colon
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'annotation.cpp' }, // opt
+				{ startIndex: 15, type: 'delimiter.parenthesis.cpp' }, // (
+				{ startIndex: 16, type: 'annotation.cpp' }, // 1
+				{ startIndex: 17, type: 'delimiter.parenthesis.cpp' }, // )
+				{ startIndex: 18, type: 'delimiter.cpp' }, // ,
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'annotation.cpp' } // debug]]
+			]
+		}
+	],
+	[
+		// Multiline and comments.
+		{
+			line: '[[nodiscard /*commented*/',
+			tokens: [
+				{ startIndex: 0, type: 'annotation.cpp' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'comment.cpp' }
+			]
+		},
+		{
+			line: ']] int i;',
+			tokens: [
+				{ startIndex: 0, type: 'annotation.cpp' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'keyword.int.cpp' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.cpp' },
+				{ startIndex: 8, type: 'delimiter.cpp' }
+			]
+		}
+	],
+	[
+		// We don't support newlines between annotation square brackets, but we do support other whitespace.
+		{
+			line: '[ [nodiscard] ]',
+			tokens: [{ startIndex: 0, type: 'annotation.cpp' }]
+		}
+	],
+
+	// Preprocessor directives with whitespace inamongst the characters,
+	// and crucially checking with whitespace before the initial #.
+	[
+		{
+			line: ' # if defined(SOMETHING)',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'identifier.cpp' },
+				{ startIndex: 13, type: 'delimiter.parenthesis.cpp' },
+				{ startIndex: 14, type: 'identifier.cpp' },
+				{ startIndex: 23, type: 'delimiter.parenthesis.cpp' }
+			]
+		},
+		{
+			line: '        #include <io.h>',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.include.cpp' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'keyword.directive.include.begin.cpp' },
+				{ startIndex: 18, type: 'string.include.identifier.cpp' },
+				{ startIndex: 22, type: 'keyword.directive.include.end.cpp' }
+			]
+		},
+		{
+			line: '      #  include <io.h>',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.directive.include.cpp' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'keyword.directive.include.begin.cpp' },
+				{ startIndex: 18, type: 'string.include.identifier.cpp' },
+				{ startIndex: 22, type: 'keyword.directive.include.end.cpp' }
+			]
+		}
+	],
+
+	[
+		// microsoft/monaco-editor#2497 : comment continuation highlighting
+		{
+			line: '// this is a comment \\',
+			tokens: [{ startIndex: 0, type: 'comment.cpp' }]
+		},
+		{
+			line: 'this is still a comment',
+			tokens: [{ startIndex: 0, type: 'comment.cpp' }]
+		},
+		{
+			line: 'int x = 1;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.cpp' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cpp' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.cpp' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.cpp' },
+				{ startIndex: 9, type: 'delimiter.cpp' }
+			]
+		}
+	]
+]);

monaco-languages/src/cpp/cpp.ts

@@ -0,0 +1,428 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '[', close: ']' },
+		{ open: '{', close: '}' },
+		{ open: '(', close: ')' },
+		{ open: "'", close: "'", notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*#pragma\\s+region\\b'),
+			end: new RegExp('^\\s*#pragma\\s+endregion\\b')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.cpp',
+
+	brackets: [
+		{ token: 'delimiter.curly', open: '{', close: '}' },
+		{ token: 'delimiter.parenthesis', open: '(', close: ')' },
+		{ token: 'delimiter.square', open: '[', close: ']' },
+		{ token: 'delimiter.angle', open: '<', close: '>' }
+	],
+
+	keywords: [
+		'abstract',
+		'amp',
+		'array',
+		'auto',
+		'bool',
+		'break',
+		'case',
+		'catch',
+		'char',
+		'class',
+		'const',
+		'constexpr',
+		'const_cast',
+		'continue',
+		'cpu',
+		'decltype',
+		'default',
+		'delegate',
+		'delete',
+		'do',
+		'double',
+		'dynamic_cast',
+		'each',
+		'else',
+		'enum',
+		'event',
+		'explicit',
+		'export',
+		'extern',
+		'false',
+		'final',
+		'finally',
+		'float',
+		'for',
+		'friend',
+		'gcnew',
+		'generic',
+		'goto',
+		'if',
+		'in',
+		'initonly',
+		'inline',
+		'int',
+		'interface',
+		'interior_ptr',
+		'internal',
+		'literal',
+		'long',
+		'mutable',
+		'namespace',
+		'new',
+		'noexcept',
+		'nullptr',
+		'__nullptr',
+		'operator',
+		'override',
+		'partial',
+		'pascal',
+		'pin_ptr',
+		'private',
+		'property',
+		'protected',
+		'public',
+		'ref',
+		'register',
+		'reinterpret_cast',
+		'restrict',
+		'return',
+		'safe_cast',
+		'sealed',
+		'short',
+		'signed',
+		'sizeof',
+		'static',
+		'static_assert',
+		'static_cast',
+		'struct',
+		'switch',
+		'template',
+		'this',
+		'thread_local',
+		'throw',
+		'tile_static',
+		'true',
+		'try',
+		'typedef',
+		'typeid',
+		'typename',
+		'union',
+		'unsigned',
+		'using',
+		'virtual',
+		'void',
+		'volatile',
+		'wchar_t',
+		'where',
+		'while',
+
+		'_asm', // reserved word with one underscores
+		'_based',
+		'_cdecl',
+		'_declspec',
+		'_fastcall',
+		'_if_exists',
+		'_if_not_exists',
+		'_inline',
+		'_multiple_inheritance',
+		'_pascal',
+		'_single_inheritance',
+		'_stdcall',
+		'_virtual_inheritance',
+		'_w64',
+
+		'__abstract', // reserved word with two underscores
+		'__alignof',
+		'__asm',
+		'__assume',
+		'__based',
+		'__box',
+		'__builtin_alignof',
+		'__cdecl',
+		'__clrcall',
+		'__declspec',
+		'__delegate',
+		'__event',
+		'__except',
+		'__fastcall',
+		'__finally',
+		'__forceinline',
+		'__gc',
+		'__hook',
+		'__identifier',
+		'__if_exists',
+		'__if_not_exists',
+		'__inline',
+		'__int128',
+		'__int16',
+		'__int32',
+		'__int64',
+		'__int8',
+		'__interface',
+		'__leave',
+		'__m128',
+		'__m128d',
+		'__m128i',
+		'__m256',
+		'__m256d',
+		'__m256i',
+		'__m64',
+		'__multiple_inheritance',
+		'__newslot',
+		'__nogc',
+		'__noop',
+		'__nounwind',
+		'__novtordisp',
+		'__pascal',
+		'__pin',
+		'__pragma',
+		'__property',
+		'__ptr32',
+		'__ptr64',
+		'__raise',
+		'__restrict',
+		'__resume',
+		'__sealed',
+		'__single_inheritance',
+		'__stdcall',
+		'__super',
+		'__thiscall',
+		'__try',
+		'__try_cast',
+		'__typeof',
+		'__unaligned',
+		'__unhook',
+		'__uuidof',
+		'__value',
+		'__virtual_inheritance',
+		'__w64',
+		'__wchar_t'
+	],
+
+	operators: [
+		'=',
+		'>',
+		'<',
+		'!',
+		'~',
+		'?',
+		':',
+		'==',
+		'<=',
+		'>=',
+		'!=',
+		'&&',
+		'||',
+		'++',
+		'--',
+		'+',
+		'-',
+		'*',
+		'/',
+		'&',
+		'|',
+		'^',
+		'%',
+		'<<',
+		'>>',
+		'>>>',
+		'+=',
+		'-=',
+		'*=',
+		'/=',
+		'&=',
+		'|=',
+		'^=',
+		'%=',
+		'<<=',
+		'>>=',
+		'>>>='
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /([uU](ll|LL|l|L)|(ll|LL|l|L)?[uU]?)/,
+	floatsuffix: /[fFlL]?/,
+	encoding: /u|u8|U|L/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// C++ 11 Raw String
+			[/@encoding?R\"(?:([^ ()\\\t]*))\(/, { token: 'string.raw.begin', next: '@raw.$1' }],
+
+			// identifiers and keywords
+			[
+				/[a-zA-Z_]\w*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			// The preprocessor checks must be before whitespace as they check /^\s*#/ which
+			// otherwise fails to match later after other whitespace has been removed.
+
+			// Inclusion
+			[/^\s*#\s*include/, { token: 'keyword.directive.include', next: '@include' }],
+
+			// Preprocessor directive
+			[/^\s*#\s*\w+/, 'keyword.directive'],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [[ attributes ]].
+			[/\[\s*\[/, { token: 'annotation', next: '@annotation' }],
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/0[xX][0-9a-fA-F']*[0-9a-fA-F](@integersuffix)/, 'number.hex'],
+			[/0[0-7']*[0-7](@integersuffix)/, 'number.octal'],
+			[/0[bB][0-1']*[0-1](@integersuffix)/, 'number.binary'],
+			[/\d[\d']*\d(@integersuffix)/, 'number'],
+			[/\d(@integersuffix)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, 'string', '@string'],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@doccomment'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*\\$/, 'comment', '@linecomment'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		//For use with continuous line comments
+		linecomment: [
+			[/.*[^\\]$/, 'comment', '@pop'],
+			[/[^]+/, 'comment']
+		],
+
+		//Identical copy of comment above, except for the addition of .doc
+		doccomment: [
+			[/[^\/*]+/, 'comment.doc'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		],
+
+		raw: [
+			[
+				/(.*)(\))(?:([^ ()\\\t"]*))(\")/,
+				{
+					cases: {
+						'$3==$S2': [
+							'string.raw',
+							'string.raw.end',
+							'string.raw.end',
+							{ token: 'string.raw.end', next: '@pop' }
+						],
+						'@default': ['string.raw', 'string.raw', 'string.raw', 'string.raw']
+					}
+				}
+			],
+			[/.*/, 'string.raw']
+		],
+
+		annotation: [
+			{ include: '@whitespace' },
+			[/using|alignas/, 'keyword'],
+			[/[a-zA-Z0-9_]+/, 'annotation'],
+			[/[,:]/, 'delimiter'],
+			[/[()]/, '@brackets'],
+			[/\]\s*\]/, { token: 'annotation', next: '@pop' }]
+		],
+
+		include: [
+			[
+				/(\s*)(<)([^<>]*)(>)/,
+				[
+					'',
+					'keyword.directive.include.begin',
+					'string.include.identifier',
+					{ token: 'keyword.directive.include.end', next: '@pop' }
+				]
+			],
+			[
+				/(\s*)(")([^"]*)(")/,
+				[
+					'',
+					'keyword.directive.include.begin',
+					'string.include.identifier',
+					{ token: 'keyword.directive.include.end', next: '@pop' }
+				]
+			]
+		]
+	}
+};

monaco-languages/src/csharp/csharp.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'csharp',
+	extensions: ['.cs', '.csx', '.cake'],
+	aliases: ['C#', 'csharp'],
+	loader: () => import('./csharp')
+});

monaco-languages/src/csharp/csharp.test.ts

@@ -0,0 +1,976 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('csharp', [
+	// Generated from sample
+	[
+		{
+			line: 'using System;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.using.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'namespace.cs' },
+				{ startIndex: 12, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: 'using System.Collections.Generic;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.using.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'namespace.cs' },
+				{ startIndex: 12, type: 'delimiter.cs' },
+				{ startIndex: 13, type: 'namespace.cs' },
+				{ startIndex: 24, type: 'delimiter.cs' },
+				{ startIndex: 25, type: 'namespace.cs' },
+				{ startIndex: 32, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: 'using System.Diagnostics;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.using.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'namespace.cs' },
+				{ startIndex: 12, type: 'delimiter.cs' },
+				{ startIndex: 13, type: 'namespace.cs' },
+				{ startIndex: 24, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: 'using System.Linq;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.using.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'namespace.cs' },
+				{ startIndex: 12, type: 'delimiter.cs' },
+				{ startIndex: 13, type: 'namespace.cs' },
+				{ startIndex: 17, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: 'using System.Text;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.using.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'namespace.cs' },
+				{ startIndex: 12, type: 'delimiter.cs' },
+				{ startIndex: 13, type: 'namespace.cs' },
+				{ startIndex: 17, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: 'using System.Threading.Tasks;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.using.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'namespace.cs' },
+				{ startIndex: 12, type: 'delimiter.cs' },
+				{ startIndex: 13, type: 'namespace.cs' },
+				{ startIndex: 22, type: 'delimiter.cs' },
+				{ startIndex: 23, type: 'namespace.cs' },
+				{ startIndex: 28, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'namespace VS',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.namespace.cs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'namespace.cs' }
+			]
+		},
+		{
+			line: '{',
+			tokens: [{ startIndex: 0, type: 'delimiter.curly.cs' }]
+		},
+		{
+			line: '	class Program',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'keyword.class.cs' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.cs' }
+			]
+		},
+		{
+			line: '	{',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '		static void Main(string[] args)',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'keyword.static.cs' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.void.cs' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'identifier.cs' },
+				{ startIndex: 18, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 19, type: 'keyword.string.cs' },
+				{ startIndex: 25, type: 'delimiter.square.cs' },
+				{ startIndex: 27, type: '' },
+				{ startIndex: 28, type: 'identifier.cs' },
+				{ startIndex: 32, type: 'delimiter.parenthesis.cs' }
+			]
+		},
+		{
+			line: '		{',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '			ProcessStartInfo si = new ProcessStartInfo();',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'identifier.cs' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'identifier.cs' },
+				{ startIndex: 22, type: '' },
+				{ startIndex: 23, type: 'delimiter.cs' },
+				{ startIndex: 24, type: '' },
+				{ startIndex: 25, type: 'keyword.new.cs' },
+				{ startIndex: 28, type: '' },
+				{ startIndex: 29, type: 'identifier.cs' },
+				{ startIndex: 45, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 47, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '			float load= 3.2e02f;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.float.cs' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'identifier.cs' },
+				{ startIndex: 13, type: 'delimiter.cs' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'number.float.cs' },
+				{ startIndex: 22, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '			si.FileName = @"tools\\\\node.exe";',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'identifier.cs' },
+				{ startIndex: 5, type: 'delimiter.cs' },
+				{ startIndex: 6, type: 'identifier.cs' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'delimiter.cs' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'string.quote.cs' },
+				{ startIndex: 19, type: 'string.cs' },
+				{ startIndex: 34, type: 'string.quote.cs' },
+				{ startIndex: 35, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '			si.Arguments = "tools\\\\simpl3server.js";',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'identifier.cs' },
+				{ startIndex: 5, type: 'delimiter.cs' },
+				{ startIndex: 6, type: 'identifier.cs' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'delimiter.cs' },
+				{ startIndex: 17, type: '' },
+				{ startIndex: 18, type: 'string.quote.cs' },
+				{ startIndex: 19, type: 'string.cs' },
+				{ startIndex: 24, type: 'string.escape.cs' },
+				{ startIndex: 26, type: 'string.cs' },
+				{ startIndex: 41, type: 'string.quote.cs' },
+				{ startIndex: 42, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '			',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '			string someString = $"hello{outside+variable}the string again {{ escaped";',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.string.cs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.cs' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'delimiter.cs' },
+				{ startIndex: 22, type: '' },
+				{ startIndex: 23, type: 'string.quote.cs' },
+				{ startIndex: 25, type: 'string.cs' },
+				{ startIndex: 30, type: 'string.quote.cs' },
+				{ startIndex: 31, type: 'identifier.cs' },
+				{ startIndex: 38, type: 'delimiter.cs' },
+				{ startIndex: 39, type: 'identifier.cs' },
+				{ startIndex: 47, type: 'string.quote.cs' },
+				{ startIndex: 48, type: 'string.cs' },
+				{ startIndex: 65, type: 'string.escape.cs' },
+				{ startIndex: 67, type: 'string.cs' },
+				{ startIndex: 75, type: 'string.quote.cs' },
+				{ startIndex: 76, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '			var @string = 5;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.var.cs' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.cs' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'delimiter.cs' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'number.cs' },
+				{ startIndex: 18, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '			',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '			if (x == 4)',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.if.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 7, type: 'identifier.cs' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.cs' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'number.cs' },
+				{ startIndex: 13, type: 'delimiter.parenthesis.cs' }
+			]
+		},
+		{
+			line: '			{',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '				for (int i = 4; i<10; i++)',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'keyword.for.cs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 9, type: 'keyword.int.cs' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'identifier.cs' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'delimiter.cs' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'number.cs' },
+				{ startIndex: 18, type: 'delimiter.cs' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'identifier.cs' },
+				{ startIndex: 21, type: 'delimiter.angle.cs' },
+				{ startIndex: 22, type: 'number.cs' },
+				{ startIndex: 24, type: 'delimiter.cs' },
+				{ startIndex: 25, type: '' },
+				{ startIndex: 26, type: 'identifier.cs' },
+				{ startIndex: 27, type: 'delimiter.cs' },
+				{ startIndex: 29, type: 'delimiter.parenthesis.cs' }
+			]
+		},
+		{
+			line: '				{',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '					var d = i;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 5, type: 'keyword.var.cs' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'identifier.cs' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'delimiter.cs' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'identifier.cs' },
+				{ startIndex: 14, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '				}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '			}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '			else',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'keyword.else.cs' }
+			]
+		},
+		{
+			line: '			{',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '				return;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'keyword.return.cs' },
+				{ startIndex: 10, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '			}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '			',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '			Process.Start(si);',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 3, type: 'identifier.cs' },
+				{ startIndex: 10, type: 'delimiter.cs' },
+				{ startIndex: 11, type: 'identifier.cs' },
+				{ startIndex: 16, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 17, type: 'identifier.cs' },
+				{ startIndex: 19, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 20, type: 'delimiter.cs' }
+			]
+		},
+		{
+			line: '		}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '	}',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '}',
+			tokens: [{ startIndex: 0, type: 'delimiter.curly.cs' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '#pragma region /MapLayer/*Image* /// ',
+			tokens: [{ startIndex: 0, type: 'namespace.cpp.cs' }]
+		},
+		{
+			line: 'namespace ShouldNotBeAComment {}',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.namespace.cs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'namespace.cs' },
+				{ startIndex: 29, type: '' },
+				{ startIndex: 30, type: 'delimiter.curly.cs' }
+			]
+		},
+		{
+			line: '#pragma endregion Region_1',
+			tokens: [{ startIndex: 0, type: 'namespace.cpp.cs' }]
+		}
+	],
+
+	// Keywords
+	[
+		{
+			line: 'namespace VS { class Program { static void Main(string[] args) {} } }',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.namespace.cs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'namespace.cs' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'delimiter.curly.cs' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'keyword.class.cs' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'identifier.cs' },
+				{ startIndex: 28, type: '' },
+				{ startIndex: 29, type: 'delimiter.curly.cs' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'keyword.static.cs' },
+				{ startIndex: 37, type: '' },
+				{ startIndex: 38, type: 'keyword.void.cs' },
+				{ startIndex: 42, type: '' },
+				{ startIndex: 43, type: 'identifier.cs' },
+				{ startIndex: 47, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 48, type: 'keyword.string.cs' },
+				{ startIndex: 54, type: 'delimiter.square.cs' },
+				{ startIndex: 56, type: '' },
+				{ startIndex: 57, type: 'identifier.cs' },
+				{ startIndex: 61, type: 'delimiter.parenthesis.cs' },
+				{ startIndex: 62, type: '' },
+				{ startIndex: 63, type: 'delimiter.curly.cs' },
+				{ startIndex: 65, type: '' },
+				{ startIndex: 66, type: 'delimiter.curly.cs' },
+				{ startIndex: 67, type: '' },
+				{ startIndex: 68, type: 'delimiter.curly.cs' }
+			]
+		}
+	],
+
+	// Comments - single line
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '/almost a comment',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.cs' },
+				{ startIndex: 1, type: 'identifier.cs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.cs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1 / 2; /* comment',
+			tokens: [
+				{ startIndex: 0, type: 'number.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.cs' },
+				{ startIndex: 5, type: 'delimiter.cs' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'comment.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'var x = 1; // my comment // is a nice one',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.var.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.cs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.cs' },
+				{ startIndex: 9, type: 'delimiter.cs' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'comment.cs' }
+			]
+		}
+	],
+
+	// Comments - range comment, single line
+	[
+		{
+			line: '/* a simple comment */',
+			tokens: [{ startIndex: 0, type: 'comment.cs' }]
+		}
+	],
+
+	[
+		{
+			line: 'var x = /* a simple comment */ 1;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.var.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.cs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.cs' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'number.cs' },
+				{ startIndex: 32, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'var x = /* comment */ 1; */',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.var.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.cs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.cs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.cs' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'number.cs' },
+				{ startIndex: 23, type: 'delimiter.cs' },
+				{ startIndex: 24, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /**/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.cs' },
+				{ startIndex: 8, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /*/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.cs' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '123_456',
+			tokens: [{ startIndex: 0, type: 'number.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '0x',
+			tokens: [
+				{ startIndex: 0, type: 'number.cs' },
+				{ startIndex: 1, type: 'identifier.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0b',
+			tokens: [
+				{ startIndex: 0, type: 'number.cs' },
+				{ startIndex: 1, type: 'identifier.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '0x123_456',
+			tokens: [{ startIndex: 0, type: 'number.hex.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '0b101',
+			tokens: [{ startIndex: 0, type: 'number.hex.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '0b1010_0001',
+			tokens: [{ startIndex: 0, type: 'number.hex.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1_23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5F',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5f',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '2_3.5f',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3F',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3f',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3F',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3f',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5D',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '456_123.5D',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5d',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3D',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3d',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.720_123E3d',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3D',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3d',
+			tokens: [{ startIndex: 0, type: 'number.float.cs' }]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.cs' },
+				{ startIndex: 1, type: 'delimiter.cs' },
+				{ startIndex: 2, type: 'number.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.cs' },
+				{ startIndex: 3, type: 'delimiter.cs' },
+				{ startIndex: 4, type: 'number.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.cs' }
+			]
+		}
+	],
+
+	// Strings
+	[
+		{
+			line: 'x = "string";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.quote.cs' },
+				{ startIndex: 5, type: 'string.cs' },
+				{ startIndex: 11, type: 'string.quote.cs' },
+				{ startIndex: 12, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = "stri\\"ng";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.quote.cs' },
+				{ startIndex: 5, type: 'string.cs' },
+				{ startIndex: 9, type: 'string.escape.cs' },
+				{ startIndex: 11, type: 'string.cs' },
+				{ startIndex: 13, type: 'string.quote.cs' },
+				{ startIndex: 14, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	// Verbatim Strings
+	[
+		{
+			line: 'x = @"verbatimstring";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.quote.cs' },
+				{ startIndex: 6, type: 'string.cs' },
+				{ startIndex: 20, type: 'string.quote.cs' },
+				{ startIndex: 21, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = @"verbatim""string";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.quote.cs' },
+				{ startIndex: 6, type: 'string.cs' },
+				{ startIndex: 14, type: 'string.escape.cs' },
+				{ startIndex: 16, type: 'string.cs' },
+				{ startIndex: 22, type: 'string.quote.cs' },
+				{ startIndex: 23, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = @"verbatim\\string\\";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.quote.cs' },
+				{ startIndex: 6, type: 'string.cs' },
+				{ startIndex: 22, type: 'string.quote.cs' },
+				{ startIndex: 23, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = @"verbatim',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.quote.cs' },
+				{ startIndex: 6, type: 'string.cs' }
+			]
+		},
+		{
+			line: 'string";',
+			tokens: [
+				{ startIndex: 0, type: 'string.cs' },
+				{ startIndex: 6, type: 'string.quote.cs' },
+				{ startIndex: 7, type: 'delimiter.cs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = $@"verbatim {interpolated} string{{}}"" ";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.cs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.cs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.quote.cs' },
+				{ startIndex: 7, type: 'string.cs' },
+				{ startIndex: 16, type: 'string.quote.cs' },
+				{ startIndex: 17, type: 'identifier.cs' },
+				{ startIndex: 29, type: 'string.quote.cs' },
+				{ startIndex: 30, type: 'string.cs' },
+				{ startIndex: 37, type: 'string.escape.cs' },
+				{ startIndex: 39, type: 'string.cs' },
+				{ startIndex: 41, type: 'string.escape.cs' },
+				{ startIndex: 43, type: 'string.cs' },
+				{ startIndex: 44, type: 'string.quote.cs' },
+				{ startIndex: 45, type: 'delimiter.cs' }
+			]
+		}
+	]
+]);

monaco-languages/src/csharp/csharp.ts

@@ -0,0 +1,352 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	wordPattern:
+		/(-?\d*\.\d\w*)|([^\`\~\!\#\$\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: "'", close: "'", notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' },
+		{ open: "'", close: "'" },
+		{ open: '"', close: '"' }
+	],
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*#region\\b'),
+			end: new RegExp('^\\s*#endregion\\b')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.cs',
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '<', close: '>', token: 'delimiter.angle' }
+	],
+
+	keywords: [
+		'extern',
+		'alias',
+		'using',
+		'bool',
+		'decimal',
+		'sbyte',
+		'byte',
+		'short',
+		'ushort',
+		'int',
+		'uint',
+		'long',
+		'ulong',
+		'char',
+		'float',
+		'double',
+		'object',
+		'dynamic',
+		'string',
+		'assembly',
+		'is',
+		'as',
+		'ref',
+		'out',
+		'this',
+		'base',
+		'new',
+		'typeof',
+		'void',
+		'checked',
+		'unchecked',
+		'default',
+		'delegate',
+		'var',
+		'const',
+		'if',
+		'else',
+		'switch',
+		'case',
+		'while',
+		'do',
+		'for',
+		'foreach',
+		'in',
+		'break',
+		'continue',
+		'goto',
+		'return',
+		'throw',
+		'try',
+		'catch',
+		'finally',
+		'lock',
+		'yield',
+		'from',
+		'let',
+		'where',
+		'join',
+		'on',
+		'equals',
+		'into',
+		'orderby',
+		'ascending',
+		'descending',
+		'select',
+		'group',
+		'by',
+		'namespace',
+		'partial',
+		'class',
+		'field',
+		'event',
+		'method',
+		'param',
+		'public',
+		'protected',
+		'internal',
+		'private',
+		'abstract',
+		'sealed',
+		'static',
+		'struct',
+		'readonly',
+		'volatile',
+		'virtual',
+		'override',
+		'params',
+		'get',
+		'set',
+		'add',
+		'remove',
+		'operator',
+		'true',
+		'false',
+		'implicit',
+		'explicit',
+		'interface',
+		'enum',
+		'null',
+		'async',
+		'await',
+		'fixed',
+		'sizeof',
+		'stackalloc',
+		'unsafe',
+		'nameof',
+		'when'
+	],
+
+	namespaceFollows: ['namespace', 'using'],
+
+	parenFollows: ['if', 'for', 'while', 'switch', 'foreach', 'using', 'catch', 'when'],
+
+	operators: [
+		'=',
+		'??',
+		'||',
+		'&&',
+		'|',
+		'^',
+		'&',
+		'==',
+		'!=',
+		'<=',
+		'>=',
+		'<<',
+		'+',
+		'-',
+		'*',
+		'/',
+		'%',
+		'!',
+		'~',
+		'++',
+		'--',
+		'+=',
+		'-=',
+		'*=',
+		'/=',
+		'%=',
+		'&=',
+		'|=',
+		'^=',
+		'<<=',
+		'>>=',
+		'>>',
+		'=>'
+	],
+
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+
+	// escape sequences
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[
+				/\@?[a-zA-Z_]\w*/,
+				{
+					cases: {
+						'@namespaceFollows': {
+							token: 'keyword.$0',
+							next: '@namespace'
+						},
+						'@keywords': {
+							token: 'keyword.$0',
+							next: '@qualified'
+						},
+						'@default': { token: 'identifier', next: '@qualified' }
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[
+				/}/,
+				{
+					cases: {
+						'$S2==interpolatedstring': {
+							token: 'string.quote',
+							next: '@pop'
+						},
+						'$S2==litinterpstring': {
+							token: 'string.quote',
+							next: '@pop'
+						},
+						'@default': '@brackets'
+					}
+				}
+			],
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// numbers
+			[/[0-9_]*\.[0-9_]+([eE][\-+]?\d+)?[fFdD]?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]+/, 'number.hex'],
+			[/0[bB][01_]+/, 'number.hex'], // binary: use same theme style as hex
+			[/[0-9_]+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, { token: 'string.quote', next: '@string' }],
+			[/\$\@"/, { token: 'string.quote', next: '@litinterpstring' }],
+			[/\@"/, { token: 'string.quote', next: '@litstring' }],
+			[/\$"/, { token: 'string.quote', next: '@interpolatedstring' }],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		qualified: [
+			[
+				/[a-zA-Z_][\w]*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'identifier'
+					}
+				}
+			],
+			[/\./, 'delimiter'],
+			['', '', '@pop']
+		],
+
+		namespace: [
+			{ include: '@whitespace' },
+			[/[A-Z]\w*/, 'namespace'],
+			[/[\.=]/, 'delimiter'],
+			['', '', '@pop']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			// [/\/\*/,    'comment', '@push' ],    // no nested comments :-(
+			['\\*/', 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, { token: 'string.quote', next: '@pop' }]
+		],
+
+		litstring: [
+			[/[^"]+/, 'string'],
+			[/""/, 'string.escape'],
+			[/"/, { token: 'string.quote', next: '@pop' }]
+		],
+
+		litinterpstring: [
+			[/[^"{]+/, 'string'],
+			[/""/, 'string.escape'],
+			[/{{/, 'string.escape'],
+			[/}}/, 'string.escape'],
+			[/{/, { token: 'string.quote', next: 'root.litinterpstring' }],
+			[/"/, { token: 'string.quote', next: '@pop' }]
+		],
+
+		interpolatedstring: [
+			[/[^\\"{]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/{{/, 'string.escape'],
+			[/}}/, 'string.escape'],
+			[/{/, { token: 'string.quote', next: 'root.interpolatedstring' }],
+			[/"/, { token: 'string.quote', next: '@pop' }]
+		],
+
+		whitespace: [
+			[/^[ \t\v\f]*#((r)|(load))(?=\s)/, 'directive.csx'],
+			[/^[ \t\v\f]*#\w.*$/, 'namespace.cpp'],
+			[/[ \t\v\f\r\n]+/, ''],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		]
+	}
+};

monaco-languages/src/csp/csp.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'csp',
+	extensions: [],
+	aliases: ['CSP', 'csp'],
+	loader: () => import('./csp')
+});

monaco-languages/src/csp/csp.test.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('csp', []);

monaco-languages/src/csp/csp.ts

@@ -0,0 +1,58 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	brackets: [],
+	autoClosingPairs: [],
+	surroundingPairs: []
+};
+
+export const language = <languages.IMonarchLanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	// defaultToken: 'invalid',
+	keywords: [],
+	typeKeywords: [],
+	tokenPostfix: '.csp',
+	operators: [],
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	tokenizer: {
+		root: [
+			[/child-src/, 'string.quote'],
+			[/connect-src/, 'string.quote'],
+			[/default-src/, 'string.quote'],
+			[/font-src/, 'string.quote'],
+			[/frame-src/, 'string.quote'],
+			[/img-src/, 'string.quote'],
+			[/manifest-src/, 'string.quote'],
+			[/media-src/, 'string.quote'],
+			[/object-src/, 'string.quote'],
+			[/script-src/, 'string.quote'],
+			[/style-src/, 'string.quote'],
+			[/worker-src/, 'string.quote'],
+			[/base-uri/, 'string.quote'],
+			[/plugin-types/, 'string.quote'],
+			[/sandbox/, 'string.quote'],
+			[/disown-opener/, 'string.quote'],
+			[/form-action/, 'string.quote'],
+			[/frame-ancestors/, 'string.quote'],
+			[/report-uri/, 'string.quote'],
+			[/report-to/, 'string.quote'],
+			[/upgrade-insecure-requests/, 'string.quote'],
+			[/block-all-mixed-content/, 'string.quote'],
+			[/require-sri-for/, 'string.quote'],
+			[/reflected-xss/, 'string.quote'],
+			[/referrer/, 'string.quote'],
+			[/policy-uri/, 'string.quote'],
+			[/'self'/, 'string.quote'],
+			[/'unsafe-inline'/, 'string.quote'],
+			[/'unsafe-eval'/, 'string.quote'],
+			[/'strict-dynamic'/, 'string.quote'],
+			[/'unsafe-hashed-attributes'/, 'string.quote']
+		]
+	}
+};

monaco-languages/src/css/css.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'css',
+	extensions: ['.css'],
+	aliases: ['CSS', 'css'],
+	mimetypes: ['text/css'],
+	loader: () => import('./css')
+});

monaco-languages/src/css/css.test.ts

@@ -0,0 +1,591 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('css', [
+	// Skip whitespace
+	[
+		{
+			line: '      body',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 6, type: 'tag.css' }
+			]
+		}
+	],
+
+	// CSS rule
+	//	body {
+	//	  margin: 0;
+	//	  padding: 3em 6em;
+	//	  font-family: tahoma, arial, sans-serif;
+	//	  text-decoration: none !important;
+	//	  color: #000
+	//	}
+	[
+		{
+			line: 'body {',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  margin: 0;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'attribute.value.number.css' },
+				{ startIndex: 11, type: 'delimiter.css' }
+			]
+		},
+		{
+			line: '  padding: 3em 6em;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'attribute.value.number.css' },
+				{ startIndex: 12, type: 'attribute.value.unit.css' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'attribute.value.number.css' },
+				{ startIndex: 16, type: 'attribute.value.unit.css' },
+				{ startIndex: 18, type: 'delimiter.css' }
+			]
+		},
+		{
+			line: '  font-family: tahoma, arial, sans-serif;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'attribute.value.css' },
+				{ startIndex: 21, type: 'delimiter.css' },
+				{ startIndex: 22, type: '' },
+				{ startIndex: 23, type: 'attribute.value.css' },
+				{ startIndex: 28, type: 'delimiter.css' },
+				{ startIndex: 29, type: '' },
+				{ startIndex: 30, type: 'attribute.value.css' },
+				{ startIndex: 40, type: 'delimiter.css' }
+			]
+		},
+		{
+			line: '  text-decoration: none !important;',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'attribute.value.css' },
+				{ startIndex: 23, type: '' },
+				{ startIndex: 24, type: 'keyword.css' },
+				{ startIndex: 34, type: 'delimiter.css' }
+			]
+		},
+		{
+			line: '  color: #000',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'attribute.value.hex.css' }
+			]
+		},
+		{
+			line: '  }',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// CSS units and numbers
+	[
+		{
+			line: '* { padding: 3em -9pt -0.5px; }',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.bracket.css' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'attribute.name.css' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'attribute.value.number.css' },
+				{ startIndex: 14, type: 'attribute.value.unit.css' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'attribute.value.number.css' },
+				{ startIndex: 19, type: 'attribute.value.unit.css' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'attribute.value.number.css' },
+				{ startIndex: 26, type: 'attribute.value.unit.css' },
+				{ startIndex: 28, type: 'delimiter.css' },
+				{ startIndex: 29, type: '' },
+				{ startIndex: 30, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// CSS unfinished unit and numbers
+	[
+		{
+			line: '* { padding: -',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.bracket.css' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'attribute.name.css' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'delimiter.css' }
+			]
+		}
+	],
+
+	// CSS single line comment
+	//	h1 /*comment*/ p  {
+	[
+		{
+			line: 'h1 /*comment*/ p {',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'comment.css' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'tag.css' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// CSS multi line comment
+	//	h1 /*com
+	//  ment*/ p  {
+	[
+		{
+			line: 'h1 /*com',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'comment.css' }
+			]
+		},
+		{
+			line: 'ment*/ p',
+			tokens: [
+				{ startIndex: 0, type: 'comment.css' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'tag.css' }
+			]
+		}
+	],
+
+	// CSS ID rule
+	[
+		{
+			line: '#myID {',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// CSS Class rules
+	[
+		{
+			line: '.myID {',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// CSS @import etc
+	[
+		{
+			line: '@import url("something.css");',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.css' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'attribute.value.css' },
+				{ startIndex: 11, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 12, type: 'string.css' },
+				{ startIndex: 27, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 28, type: 'delimiter.css' }
+			]
+		}
+	],
+
+	// CSS multi-line string with an escaped newline
+	//	body {
+	//	content: 'con\
+	//  tent';
+	[
+		{
+			line: 'body {',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  content: "con\\',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.css' }
+			]
+		},
+		{
+			line: 'tent";',
+			tokens: [
+				{ startIndex: 0, type: 'string.css' },
+				{ startIndex: 5, type: 'delimiter.css' }
+			]
+		}
+	],
+
+	// CSS empty string value
+	//	body {
+	//	content: '';
+	[
+		{
+			line: 'body {',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  content: "";',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.css' },
+				{ startIndex: 13, type: 'delimiter.css' }
+			]
+		}
+	],
+
+	// CSS font face
+	// @font-face {
+	//     font-family: 'Opificio';
+	// }
+	[
+		{
+			line: '@font-face {',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.css' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  font-family: "Opificio";',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'string.css' },
+				{ startIndex: 25, type: 'delimiter.css' }
+			]
+		}
+	],
+
+	// CSS string with escaped quotes
+	//	's\"tr'
+	[
+		{
+			line: '"s\\"tr" ',
+			tokens: [
+				{ startIndex: 0, type: 'string.css' },
+				{ startIndex: 7, type: '' }
+			]
+		}
+	],
+
+	// CSS key frame animation syntax
+	//@-webkit-keyframes infinite-spinning {
+	//	 from {
+	//		-webkit-transform: rotate(0deg);
+	//	}
+	//	to {
+	//		-webkit-transform: rotate(360deg);
+	//	}
+	//}
+	[
+		{
+			line: '@-webkit-keyframes infinite-spinning {',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.css' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'attribute.value.css' },
+				{ startIndex: 36, type: '' },
+				{ startIndex: 37, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  from {',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.value.css' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  -webkit-transform: rotate(0deg);',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'attribute.value.css' },
+				{ startIndex: 28, type: 'attribute.value.number.css' },
+				{ startIndex: 29, type: 'attribute.value.unit.css' },
+				{ startIndex: 32, type: 'attribute.value.css' },
+				{ startIndex: 33, type: 'delimiter.css' }
+			]
+		},
+		{
+			line: '	 }',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  to {',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.value.css' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '  -webkit-transform: rotate(360deg);',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'attribute.name.css' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'attribute.value.css' },
+				{ startIndex: 28, type: 'attribute.value.number.css' },
+				{ startIndex: 31, type: 'attribute.value.unit.css' },
+				{ startIndex: 34, type: 'attribute.value.css' },
+				{ startIndex: 35, type: 'delimiter.css' }
+			]
+		},
+		{
+			line: '	 }',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '}',
+			tokens: [{ startIndex: 0, type: 'delimiter.bracket.css' }]
+		}
+	],
+
+	// CSS @import related coloring bug 9553
+	//		 @import url('something.css');
+	//		.rule1{}
+	//		.rule2{}
+	[
+		{
+			line: '@import url("something.css");',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.css' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'attribute.value.css' },
+				{ startIndex: 11, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 12, type: 'string.css' },
+				{ startIndex: 27, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 28, type: 'delimiter.css' }
+			]
+		},
+		{
+			line: '.rule1{}',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 6, type: 'delimiter.bracket.css' }
+			]
+		},
+		{
+			line: '.rule2{}',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 6, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// Triple quotes - bug #9870
+	[
+		{
+			line: '"""',
+			tokens: [{ startIndex: 0, type: 'string.css' }]
+		}
+	],
+
+	[
+		{
+			line: '""""',
+			tokens: [{ startIndex: 0, type: 'string.css' }]
+		}
+	],
+
+	[
+		{
+			line: '"""""',
+			tokens: [{ startIndex: 0, type: 'string.css' }]
+		}
+	],
+
+	// import statement - bug #10308
+	//	@import url('something.css');@import url('something.css');
+	[
+		{
+			line: '@import url("something.css");@import url("something.css");',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.css' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'attribute.value.css' },
+				{ startIndex: 11, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 12, type: 'string.css' },
+				{ startIndex: 27, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 28, type: 'delimiter.css' },
+				{ startIndex: 29, type: 'keyword.css' },
+				{ startIndex: 36, type: '' },
+				{ startIndex: 37, type: 'attribute.value.css' },
+				{ startIndex: 40, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 41, type: 'string.css' },
+				{ startIndex: 56, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 57, type: 'delimiter.css' }
+			]
+		}
+	],
+
+	// !important - bug #9578
+	// .a{background:#f5f9fc !important}.b{font-family:"Helvetica Neue", Helvetica;height:31px;}
+	[
+		{
+			line: '.a{background:#f5f9fc !important}.b{font-family:"Helvetica Neue", Helvetica;height:31px;}',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 2, type: 'delimiter.bracket.css' },
+				{ startIndex: 3, type: 'attribute.name.css' },
+				{ startIndex: 14, type: 'attribute.value.hex.css' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'keyword.css' },
+				{ startIndex: 32, type: 'delimiter.bracket.css' },
+				{ startIndex: 33, type: 'tag.css' },
+				{ startIndex: 35, type: 'delimiter.bracket.css' },
+				{ startIndex: 36, type: 'attribute.name.css' },
+				{ startIndex: 48, type: 'string.css' },
+				{ startIndex: 64, type: 'delimiter.css' },
+				{ startIndex: 65, type: '' },
+				{ startIndex: 66, type: 'attribute.value.css' },
+				{ startIndex: 75, type: 'delimiter.css' },
+				{ startIndex: 76, type: 'attribute.name.css' },
+				{ startIndex: 83, type: 'attribute.value.number.css' },
+				{ startIndex: 85, type: 'attribute.value.unit.css' },
+				{ startIndex: 87, type: 'delimiter.css' },
+				{ startIndex: 88, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// base64-encoded data uris - bug #9580
+	//.even { background: #fff url(data:image/gif;base64,R0lGODlhBgASALMAAOfn5+rq6uvr6+zs7O7u7vHx8fPz8/b29vj4+P39/f///wAAAAAAAAAAAAAAAAAAACwAAAAABgASAAAIMAAVCBxIsKDBgwgTDkzAsKGAhxARSJx4oKJFAxgzFtjIkYDHjwNCigxAsiSAkygDAgA7) repeat-x bottom}
+	[
+		{
+			line: '.even { background: #fff url(data:image/gif;base64,R0lGODlhBgASALMAAOfn5+rq6uvr6+zs7O7u7vHx8fPz8/b29vj4+P39/f///wAAAAAAAAAAAAAAAAAAACwAAAAABgASAAAIMAAVCBxIsKDBgwgTDkzAsKGAhxARSJx4oKJFAxgzFtjIkYDHjwNCigxAsiSAkygDAgA7) repeat-x bottom}',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.bracket.css' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'attribute.name.css' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'attribute.value.hex.css' },
+				{ startIndex: 24, type: '' },
+				{ startIndex: 25, type: 'attribute.value.css' },
+				{ startIndex: 28, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 29, type: 'string.css' },
+				{ startIndex: 215, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 216, type: '' },
+				{ startIndex: 217, type: 'attribute.value.css' },
+				{ startIndex: 225, type: '' },
+				{ startIndex: 226, type: 'attribute.value.css' },
+				{ startIndex: 232, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// /a colorization is incorrect in url - bug #9581
+	//.a{background:url(/a.jpg)}
+	[
+		{
+			line: '.a{background:url(/a.jpg)}',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 2, type: 'delimiter.bracket.css' },
+				{ startIndex: 3, type: 'attribute.name.css' },
+				{ startIndex: 14, type: 'attribute.value.css' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 18, type: 'string.css' },
+				{ startIndex: 24, type: 'delimiter.parenthesis.css' },
+				{ startIndex: 25, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	// Bracket Matching
+	[
+		{
+			line: 'p{}',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 1, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'p:nth() {}',
+			tokens: [
+				{ startIndex: 0, type: 'tag.css' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 8, type: 'delimiter.bracket.css' }
+			]
+		}
+	],
+
+	[
+		{
+			line: "@import 'https://example.com/test.css';",
+			tokens: [
+				{ startIndex: 0, type: 'keyword.css' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'string.css' },
+				{ startIndex: 38, type: 'delimiter.css' }
+			]
+		}
+	]
+]);

monaco-languages/src/css/css.ts

@@ -0,0 +1,221 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	wordPattern: /(#?-?\d*\.\d\w*%?)|((::|[@#.!:])?[\w-?]+%?)|::|[@#.!:]/g,
+
+	comments: {
+		blockComment: ['/*', '*/']
+	},
+
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string', 'comment'] },
+		{ open: '[', close: ']', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] },
+		{ open: "'", close: "'", notIn: ['string', 'comment'] }
+	],
+
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*\\/\\*\\s*#region\\b\\s*(.*?)\\s*\\*\\/'),
+			end: new RegExp('^\\s*\\/\\*\\s*#endregion\\b.*\\*\\/')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.css',
+
+	ws: '[ \t\n\r\f]*', // whitespaces (referenced in several rules)
+	identifier:
+		'-?-?([a-zA-Z]|(\\\\(([0-9a-fA-F]{1,6}\\s?)|[^[0-9a-fA-F])))([\\w\\-]|(\\\\(([0-9a-fA-F]{1,6}\\s?)|[^[0-9a-fA-F])))*',
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.bracket' },
+		{ open: '[', close: ']', token: 'delimiter.bracket' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '<', close: '>', token: 'delimiter.angle' }
+	],
+
+	tokenizer: {
+		root: [{ include: '@selector' }],
+
+		selector: [
+			{ include: '@comments' },
+			{ include: '@import' },
+			{ include: '@strings' },
+			[
+				'[@](keyframes|-webkit-keyframes|-moz-keyframes|-o-keyframes)',
+				{ token: 'keyword', next: '@keyframedeclaration' }
+			],
+			['[@](page|content|font-face|-moz-document)', { token: 'keyword' }],
+			['[@](charset|namespace)', { token: 'keyword', next: '@declarationbody' }],
+			[
+				'(url-prefix)(\\()',
+				['attribute.value', { token: 'delimiter.parenthesis', next: '@urldeclaration' }]
+			],
+			[
+				'(url)(\\()',
+				['attribute.value', { token: 'delimiter.parenthesis', next: '@urldeclaration' }]
+			],
+			{ include: '@selectorname' },
+			['[\\*]', 'tag'], // selector symbols
+			['[>\\+,]', 'delimiter'], // selector operators
+			['\\[', { token: 'delimiter.bracket', next: '@selectorattribute' }],
+			['{', { token: 'delimiter.bracket', next: '@selectorbody' }]
+		],
+
+		selectorbody: [
+			{ include: '@comments' },
+			['[*_]?@identifier@ws:(?=(\\s|\\d|[^{;}]*[;}]))', 'attribute.name', '@rulevalue'], // rule definition: to distinguish from a nested selector check for whitespace, number or a semicolon
+			['}', { token: 'delimiter.bracket', next: '@pop' }]
+		],
+
+		selectorname: [
+			['(\\.|#(?=[^{])|%|(@identifier)|:)+', 'tag'] // selector (.foo, div, ...)
+		],
+
+		selectorattribute: [
+			{ include: '@term' },
+			[']', { token: 'delimiter.bracket', next: '@pop' }]
+		],
+
+		term: [
+			{ include: '@comments' },
+			[
+				'(url-prefix)(\\()',
+				['attribute.value', { token: 'delimiter.parenthesis', next: '@urldeclaration' }]
+			],
+			[
+				'(url)(\\()',
+				['attribute.value', { token: 'delimiter.parenthesis', next: '@urldeclaration' }]
+			],
+			{ include: '@functioninvocation' },
+			{ include: '@numbers' },
+			{ include: '@name' },
+			{ include: '@strings' },
+			['([<>=\\+\\-\\*\\/\\^\\|\\~,])', 'delimiter'],
+			[',', 'delimiter']
+		],
+
+		rulevalue: [
+			{ include: '@comments' },
+			{ include: '@strings' },
+			{ include: '@term' },
+			['!important', 'keyword'],
+			[';', 'delimiter', '@pop'],
+			['(?=})', { token: '', next: '@pop' }] // missing semicolon
+		],
+
+		warndebug: [['[@](warn|debug)', { token: 'keyword', next: '@declarationbody' }]],
+
+		import: [['[@](import)', { token: 'keyword', next: '@declarationbody' }]],
+
+		urldeclaration: [
+			{ include: '@strings' },
+			['[^)\r\n]+', 'string'],
+			['\\)', { token: 'delimiter.parenthesis', next: '@pop' }]
+		],
+
+		parenthizedterm: [
+			{ include: '@term' },
+			['\\)', { token: 'delimiter.parenthesis', next: '@pop' }]
+		],
+
+		declarationbody: [
+			{ include: '@term' },
+			[';', 'delimiter', '@pop'],
+			['(?=})', { token: '', next: '@pop' }] // missing semicolon
+		],
+
+		comments: [
+			['\\/\\*', 'comment', '@comment'],
+			['\\/\\/+.*', 'comment']
+		],
+
+		comment: [
+			['\\*\\/', 'comment', '@pop'],
+			[/[^*/]+/, 'comment'],
+			[/./, 'comment']
+		],
+
+		name: [['@identifier', 'attribute.value']],
+
+		numbers: [
+			[
+				'-?(\\d*\\.)?\\d+([eE][\\-+]?\\d+)?',
+				{ token: 'attribute.value.number', next: '@units' }
+			],
+			['#[0-9a-fA-F_]+(?!\\w)', 'attribute.value.hex']
+		],
+
+		units: [
+			[
+				'(em|ex|ch|rem|vmin|vmax|vw|vh|vm|cm|mm|in|px|pt|pc|deg|grad|rad|turn|s|ms|Hz|kHz|%)?',
+				'attribute.value.unit',
+				'@pop'
+			]
+		],
+
+		keyframedeclaration: [
+			['@identifier', 'attribute.value'],
+			['{', { token: 'delimiter.bracket', switchTo: '@keyframebody' }]
+		],
+
+		keyframebody: [
+			{ include: '@term' },
+			['{', { token: 'delimiter.bracket', next: '@selectorbody' }],
+			['}', { token: 'delimiter.bracket', next: '@pop' }]
+		],
+
+		functioninvocation: [
+			['@identifier\\(', { token: 'attribute.value', next: '@functionarguments' }]
+		],
+
+		functionarguments: [
+			['\\$@identifier@ws:', 'attribute.name'],
+			['[,]', 'delimiter'],
+			{ include: '@term' },
+			['\\)', { token: 'attribute.value', next: '@pop' }]
+		],
+
+		strings: [
+			['~?"', { token: 'string', next: '@stringenddoublequote' }],
+			["~?'", { token: 'string', next: '@stringendquote' }]
+		],
+
+		stringenddoublequote: [
+			['\\\\.', 'string'],
+			['"', { token: 'string', next: '@pop' }],
+			[/[^\\"]+/, 'string'],
+			['.', 'string']
+		],
+
+		stringendquote: [
+			['\\\\.', 'string'],
+			["'", { token: 'string', next: '@pop' }],
+			[/[^\\']+/, 'string'],
+			['.', 'string']
+		]
+	}
+};

monaco-languages/src/dart/dart.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'dart',
+	extensions: ['.dart'],
+	aliases: ['Dart', 'dart'],
+	mimetypes: ['text/x-dart-source', 'text/x-dart'],
+	loader: () => import('./dart')
+});

monaco-languages/src/dart/dart.test.ts

@@ -0,0 +1,558 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('dart', [
+	// Comments - single line
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.dart' }
+			]
+		}
+	],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[
+		{
+			line: '/* //*/ a',
+			tokens: [
+				{ startIndex: 0, type: 'comment.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '/almost a comment',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.dart' },
+				{ startIndex: 1, type: 'identifier.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.dart' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1 / 2; /* comment',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.dart' },
+				{ startIndex: 5, type: 'delimiter.dart' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'comment.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'var x = 1; // my comment // is a nice one',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.dart' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.dart' },
+				{ startIndex: 9, type: 'delimiter.dart' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'comment.dart' }
+			]
+		}
+	],
+
+	// Comments - range comment, single line
+	[
+		{
+			line: '/* a simple comment */',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		}
+	],
+
+	[
+		{
+			line: 'var x = /* a simple comment */ 1;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.dart' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.dart' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'number.dart' },
+				{ startIndex: 32, type: 'delimiter.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'var x = /* comment */ 1; */',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.dart' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.dart' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'number.dart' },
+				{ startIndex: 23, type: 'delimiter.dart' },
+				{ startIndex: 24, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /**/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.dart' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.dart' },
+				{ startIndex: 8, type: 'delimiter.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /*/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.dart' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.dart' }
+			]
+		}
+	],
+
+	// Comments - range comment, multiple lines
+	[
+		{
+			line: '/* start of multiline comment',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		},
+		{
+			line: 'a comment between without a star',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		},
+		{
+			line: 'end of multiline comment*/',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		}
+	],
+
+	[
+		{
+			line: 'var x = /* start a comment',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.dart' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.dart' }
+			]
+		},
+		{
+			line: ' a ',
+			tokens: [{ startIndex: 0, type: 'comment.dart' }]
+		},
+		{
+			line: 'and end it */ 2;',
+			tokens: [
+				{ startIndex: 0, type: 'comment.dart' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'number.dart' },
+				{ startIndex: 15, type: 'delimiter.dart' }
+			]
+		}
+	],
+
+	// Keywords
+	[
+		{
+			line: 'var x = function() { };',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.dart' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.dart' },
+				{ startIndex: 16, type: 'delimiter.parenthesis.dart' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'delimiter.bracket.dart' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'delimiter.bracket.dart' },
+				{ startIndex: 22, type: 'delimiter.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '    var    ',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'keyword.dart' },
+				{ startIndex: 7, type: '' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '0.10',
+			tokens: [{ startIndex: 0, type: 'number.float.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '0x',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 1, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '0x5_2',
+			tokens: [{ startIndex: 0, type: 'number.hex.dart' }]
+		}
+	],
+	[
+		{
+			line: '0b1010_0101',
+			tokens: [{ startIndex: 0, type: 'number.binary.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '0B001',
+			tokens: [{ startIndex: 0, type: 'number.binary.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '10e3',
+			tokens: [{ startIndex: 0, type: 'number.float.dart' }]
+		}
+	],
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e-3',
+			tokens: [{ startIndex: 0, type: 'number.float.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E-3',
+			tokens: [{ startIndex: 0, type: 'number.float.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '0_52',
+			tokens: [{ startIndex: 0, type: 'number.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '5_2',
+			tokens: [{ startIndex: 0, type: 'number.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '5_______2',
+			tokens: [{ startIndex: 0, type: 'number.dart' }]
+		}
+	],
+	[
+		{
+			line: '3._1415F',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 1, type: 'delimiter.dart' },
+				{ startIndex: 2, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '999_99_9999_L',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 11, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '52_',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 2, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0_x52',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 1, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x_52',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 1, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x52_',
+			tokens: [
+				{ startIndex: 0, type: 'number.hex.dart' },
+				{ startIndex: 4, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '052_',
+			tokens: [
+				{ startIndex: 0, type: 'number.octal.dart' },
+				{ startIndex: 3, type: 'identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '23.5L',
+			tokens: [
+				{ startIndex: 0, type: 'number.float.dart' },
+				{ startIndex: 4, type: 'type.identifier.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 1, type: 'delimiter.dart' },
+				{ startIndex: 2, type: 'number.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 3, type: 'delimiter.dart' },
+				{ startIndex: 4, type: 'number.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.dart' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.dart' }
+			]
+		}
+	],
+
+	// Strings
+	[
+		{
+			line: "var s = 's';",
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dart' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.dart' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.dart' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'string.dart' },
+				{ startIndex: 11, type: 'delimiter.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'String s = "concatenated" + " String" ;',
+			tokens: [
+				{ startIndex: 0, type: 'type.identifier.dart' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.dart' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.dart' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.dart' },
+				{ startIndex: 25, type: '' },
+				{ startIndex: 26, type: 'delimiter.dart' },
+				{ startIndex: 27, type: '' },
+				{ startIndex: 28, type: 'string.dart' },
+				{ startIndex: 37, type: '' },
+				{ startIndex: 38, type: 'delimiter.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"quote in a string"',
+			tokens: [{ startIndex: 0, type: 'string.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '"escaping \\"quotes\\" is cool"',
+			tokens: [
+				{ startIndex: 0, type: 'string.dart' },
+				{ startIndex: 10, type: 'string.escape.dart' },
+				{ startIndex: 12, type: 'string.dart' },
+				{ startIndex: 18, type: 'string.escape.dart' },
+				{ startIndex: 20, type: 'string.dart' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"\\"',
+			tokens: [{ startIndex: 0, type: 'string.invalid.dart' }]
+		}
+	],
+
+	// Annotations
+	[
+		{
+			line: '@',
+			tokens: [{ startIndex: 0, type: 'invalid.dart' }]
+		}
+	],
+
+	[
+		{
+			line: '@Override',
+			tokens: [{ startIndex: 0, type: 'annotation.dart' }]
+		}
+	]
+]);

monaco-languages/src/dart/dart.ts

@@ -0,0 +1,308 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: "'", close: "'", notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string'] },
+		{ open: '`', close: '`', notIn: ['string', 'comment'] },
+		{ open: '/**', close: ' */', notIn: ['string'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+
+		{ open: '<', close: '>' },
+		{ open: "'", close: "'" },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: '`', close: '`' }
+	],
+	folding: {
+		markers: {
+			start: /^\s*\s*#?region\b/,
+			end: /^\s*\s*#?endregion\b/
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: 'invalid',
+	tokenPostfix: '.dart',
+
+	keywords: [
+		'abstract',
+		'dynamic',
+		'implements',
+		'show',
+		'as',
+		'else',
+		'import',
+		'static',
+		'assert',
+		'enum',
+		'in',
+		'super',
+		'async',
+		'export',
+		'interface',
+		'switch',
+		'await',
+		'extends',
+		'is',
+		'sync',
+		'break',
+		'external',
+		'library',
+		'this',
+		'case',
+		'factory',
+		'mixin',
+		'throw',
+		'catch',
+		'false',
+		'new',
+		'true',
+		'class',
+		'final',
+		'null',
+		'try',
+		'const',
+		'finally',
+		'on',
+		'typedef',
+		'continue',
+		'for',
+		'operator',
+		'var',
+		'covariant',
+		'Function',
+		'part',
+		'void',
+		'default',
+		'get',
+		'rethrow',
+		'while',
+		'deferred',
+		'hide',
+		'return',
+		'with',
+		'do',
+		'if',
+		'set',
+		'yield'
+	],
+	typeKeywords: ['int', 'double', 'String', 'bool'],
+
+	operators: [
+		'+',
+		'-',
+		'*',
+		'/',
+		'~/',
+		'%',
+		'++',
+		'--',
+		'==',
+		'!=',
+		'>',
+		'<',
+		'>=',
+		'<=',
+		'=',
+		'-=',
+		'/=',
+		'%=',
+		'>>=',
+		'^=',
+		'+=',
+		'*=',
+		'~/=',
+		'<<=',
+		'&=',
+		'!=',
+		'||',
+		'&&',
+		'&',
+		'|',
+		'^',
+		'~',
+		'<<',
+		'>>',
+		'!',
+		'>>>',
+		'??',
+		'?',
+		':',
+		'|='
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	regexpctl: /[(){}\[\]\$\^|\-*+?\.]/,
+	regexpesc: /\\(?:[bBdDfnrstvwWn0\\\/]|@regexpctl|c[A-Z]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [[/[{}]/, 'delimiter.bracket'], { include: 'common' }],
+
+		common: [
+			// identifiers and keywords
+			[
+				/[a-z_$][\w$]*/,
+				{
+					cases: {
+						'@typeKeywords': 'type.identifier',
+						'@keywords': 'keyword',
+						'@default': 'identifier'
+					}
+				}
+			],
+			[/[A-Z_$][\w\$]*/, 'type.identifier'], // show class names
+			// [/[A-Z][\w\$]*/, 'identifier'],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// regular expression: ensure it is terminated before beginning (otherwise it is an opeator)
+			[
+				/\/(?=([^\\\/]|\\.)+\/([gimsuy]*)(\s*)(\.|;|,|\)|\]|\}|$))/,
+				{ token: 'regexp', bracket: '@open', next: '@regexp' }
+			],
+
+			// @ annotations.
+			[/@[a-zA-Z]+/, 'annotation'],
+			// variable
+
+			// delimiters and operators
+			[/[()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/!(?=([^=]|$))/, 'delimiter'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?/, 'number.float'],
+			[/0[xX](@hexdigits)n?/, 'number.hex'],
+			[/0[oO]?(@octaldigits)n?/, 'number.octal'],
+			[/0[bB](@binarydigits)n?/, 'number.binary'],
+			[/(@digits)n?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, 'string', '@string_double'],
+			[/'/, 'string', '@string_single']
+
+			//   [/[a-zA-Z]+/, "variable"]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@jsdoc'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/\/.*$/, 'comment.doc'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		jsdoc: [
+			[/[^\/*]+/, 'comment.doc'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		// We match regular expression quite precisely
+		regexp: [
+			[
+				/(\{)(\d+(?:,\d*)?)(\})/,
+				['regexp.escape.control', 'regexp.escape.control', 'regexp.escape.control']
+			],
+			[
+				/(\[)(\^?)(?=(?:[^\]\\\/]|\\.)+)/,
+				['regexp.escape.control', { token: 'regexp.escape.control', next: '@regexrange' }]
+			],
+			[/(\()(\?:|\?=|\?!)/, ['regexp.escape.control', 'regexp.escape.control']],
+			[/[()]/, 'regexp.escape.control'],
+			[/@regexpctl/, 'regexp.escape.control'],
+			[/[^\\\/]/, 'regexp'],
+			[/@regexpesc/, 'regexp.escape'],
+			[/\\\./, 'regexp.invalid'],
+			[
+				/(\/)([gimsuy]*)/,
+				[{ token: 'regexp', bracket: '@close', next: '@pop' }, 'keyword.other']
+			]
+		],
+
+		regexrange: [
+			[/-/, 'regexp.escape.control'],
+			[/\^/, 'regexp.invalid'],
+			[/@regexpesc/, 'regexp.escape'],
+			[/[^\]]/, 'regexp'],
+			[
+				/\]/,
+				{
+					token: 'regexp.escape.control',
+					next: '@pop',
+					bracket: '@close'
+				}
+			]
+		],
+
+		string_double: [
+			[/[^\\"\$]+/, 'string'],
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop'],
+			[/\$\w+/, 'identifier']
+		],
+
+		string_single: [
+			[/[^\\'\$]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/'/, 'string', '@pop'],
+			[/\$\w+/, 'identifier']
+		]
+	}
+};

monaco-languages/src/dockerfile/dockerfile.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'dockerfile',
+	extensions: ['.dockerfile'],
+	filenames: ['Dockerfile'],
+	aliases: ['Dockerfile'],
+	loader: () => import('./dockerfile')
+});

monaco-languages/src/dockerfile/dockerfile.test.ts

@@ -0,0 +1,238 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('dockerfile', [
+	// All
+	[
+		{
+			line: 'FROM mono:3.12',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 4, type: '' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'ENV KRE_FEED https://www.myget.org/F/aspnetvnext/api/v2',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'variable.dockerfile' },
+				{ startIndex: 12, type: '' }
+			]
+		},
+		{
+			line: 'ENV KRE_USER_HOME /opt/kre',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'variable.dockerfile' },
+				{ startIndex: 17, type: '' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'RUN apt-get -qq update && apt-get -qqy install unzip ',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'ONBUILD RUN curl -sSL https://raw.githubusercontent.com/aspnet/Home/dev/kvminstall.sh | sh',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'keyword.dockerfile' },
+				{ startIndex: 11, type: '' }
+			]
+		},
+		{
+			line: 'ONBUILD RUN bash -c "source $KRE_USER_HOME/kvm/kvm.sh \\',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'keyword.dockerfile' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 20, type: 'string.dockerfile' },
+				{ startIndex: 28, type: 'variable.dockerfile' },
+				{ startIndex: 42, type: 'string.dockerfile' }
+			]
+		},
+		{
+			line: '    && kvm install latest -a default \\',
+			tokens: [{ startIndex: 0, type: 'string.dockerfile' }]
+		},
+		{
+			line: '    && kvm alias default | xargs -i ln -s $KRE_USER_HOME/packages/{} $KRE_USER_HOME/packages/default"',
+			tokens: [
+				{ startIndex: 0, type: 'string.dockerfile' },
+				{ startIndex: 42, type: 'variable.dockerfile' },
+				{ startIndex: 56, type: 'string.dockerfile' },
+				{ startIndex: 69, type: 'variable.dockerfile' },
+				{ startIndex: 83, type: 'string.dockerfile' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '# Install libuv for Kestrel from source code (binary is not in wheezy and one in jessie is still too old)',
+			tokens: [{ startIndex: 0, type: 'comment.dockerfile' }]
+		},
+		{
+			line: 'RUN apt-get -qqy install \\',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' }
+			]
+		},
+		{
+			line: '    autoconf \\',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '    automake \\',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '    build-essential \\',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '    libtool ',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: 'RUN LIBUV_VERSION=1.0.0-rc2 \\',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' }
+			]
+		},
+		{
+			line: '    && curl -sSL https://github.com/joyent/libuv/archive/v${LIBUV_VERSION}.tar.gz | tar zxfv - -C /usr/local/src \\',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 58, type: 'variable.dockerfile' },
+				{ startIndex: 74, type: '' }
+			]
+		},
+		{
+			line: '    && cd /usr/local/src/libuv-$LIBUV_VERSION \\',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 31, type: 'variable.dockerfile' },
+				{ startIndex: 45, type: '' }
+			]
+		},
+		{
+			line: '    && sh autogen.sh && ./configure && make && make install \\',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '    && rm -rf /usr/local/src/libuv-$LIBUV_VERSION \\',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 35, type: 'variable.dockerfile' },
+				{ startIndex: 49, type: '' }
+			]
+		},
+		{
+			line: '    && ldconfig',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: 'ENV PATH $PATH:$KRE_USER_HOME/packages/default/bin',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'variable.dockerfile' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'variable.dockerfile' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'variable.dockerfile' },
+				{ startIndex: 29, type: '' }
+			]
+		},
+		{
+			line: '',
+			tokens: []
+		},
+		{
+			line: '# Extra things to test',
+			tokens: [{ startIndex: 0, type: 'comment.dockerfile' }]
+		},
+		{
+			line: 'RUN echo "string at end"',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 9, type: 'string.dockerfile' }
+			]
+		},
+		{
+			line: "RUN echo must work 'some str' and some more",
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 19, type: 'string.dockerfile' },
+				{ startIndex: 29, type: '' }
+			]
+		},
+		{
+			line: 'RUN echo hi this is # not a comment',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' }
+			]
+		},
+		{
+			line: "RUN echo 'String with ${VAR} and another $one here'",
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 9, type: 'string.dockerfile' },
+				{ startIndex: 22, type: 'variable.dockerfile' },
+				{ startIndex: 28, type: 'string.dockerfile' },
+				{ startIndex: 41, type: 'variable.dockerfile' },
+				{ startIndex: 45, type: 'string.dockerfile' }
+			]
+		}
+	],
+	[
+		{
+			line: "RUN \\'e\\'",
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' }
+			]
+		},
+		{
+			line: 'RUN echo hi this is # not a comment',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.dockerfile' },
+				{ startIndex: 3, type: '' }
+			]
+		}
+	]
+]);

monaco-languages/src/dockerfile/dockerfile.ts

@@ -0,0 +1,145 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.dockerfile',
+
+	variable: /\${?[\w]+}?/,
+
+	tokenizer: {
+		root: [
+			{ include: '@whitespace' },
+			{ include: '@comment' },
+
+			[/(ONBUILD)(\s+)/, ['keyword', '']],
+			[/(ENV)(\s+)([\w]+)/, ['keyword', '', { token: 'variable', next: '@arguments' }]],
+			[
+				/(FROM|MAINTAINER|RUN|EXPOSE|ENV|ADD|ARG|VOLUME|LABEL|USER|WORKDIR|COPY|CMD|STOPSIGNAL|SHELL|HEALTHCHECK|ENTRYPOINT)/,
+				{ token: 'keyword', next: '@arguments' }
+			]
+		],
+
+		arguments: [
+			{ include: '@whitespace' },
+			{ include: '@strings' },
+
+			[
+				/(@variable)/,
+				{
+					cases: {
+						'@eos': { token: 'variable', next: '@popall' },
+						'@default': 'variable'
+					}
+				}
+			],
+			[
+				/\\/,
+				{
+					cases: {
+						'@eos': '',
+						'@default': ''
+					}
+				}
+			],
+			[
+				/./,
+				{
+					cases: {
+						'@eos': { token: '', next: '@popall' },
+						'@default': ''
+					}
+				}
+			]
+		],
+
+		// Deal with white space, including comments
+		whitespace: [
+			[
+				/\s+/,
+				{
+					cases: {
+						'@eos': { token: '', next: '@popall' },
+						'@default': ''
+					}
+				}
+			]
+		],
+
+		comment: [[/(^#.*$)/, 'comment', '@popall']],
+
+		// Recognize strings, including those broken across lines with \ (but not without)
+		strings: [
+			[/\\'$/, '', '@popall'], // \' leaves @arguments at eol
+			[/\\'/, ''], // \' is not a string
+			[/'$/, 'string', '@popall'],
+			[/'/, 'string', '@stringBody'],
+			[/"$/, 'string', '@popall'],
+			[/"/, 'string', '@dblStringBody']
+		],
+		stringBody: [
+			[
+				/[^\\\$']/,
+				{
+					cases: {
+						'@eos': { token: 'string', next: '@popall' },
+						'@default': 'string'
+					}
+				}
+			],
+
+			[/\\./, 'string.escape'],
+			[/'$/, 'string', '@popall'],
+			[/'/, 'string', '@pop'],
+			[/(@variable)/, 'variable'],
+
+			[/\\$/, 'string'],
+			[/$/, 'string', '@popall']
+		],
+		dblStringBody: [
+			[
+				/[^\\\$"]/,
+				{
+					cases: {
+						'@eos': { token: 'string', next: '@popall' },
+						'@default': 'string'
+					}
+				}
+			],
+
+			[/\\./, 'string.escape'],
+			[/"$/, 'string', '@popall'],
+			[/"/, 'string', '@pop'],
+			[/(@variable)/, 'variable'],
+
+			[/\\$/, 'string'],
+			[/$/, 'string', '@popall']
+		]
+	}
+};

monaco-languages/src/ecl/ecl.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'ecl',
+	extensions: ['.ecl'],
+	aliases: ['ECL', 'Ecl', 'ecl'],
+	loader: () => import('./ecl')
+});

monaco-languages/src/ecl/ecl.test.ts

@@ -0,0 +1,8 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('ecl', []);

monaco-languages/src/ecl/ecl.ts

@@ -0,0 +1,481 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: "'", close: "'", notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' },
+		{ open: "'", close: "'" },
+		{ open: '"', close: '"' }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.ecl',
+	ignoreCase: true,
+
+	brackets: [
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '<', close: '>', token: 'delimiter.angle' }
+	],
+
+	pounds: [
+		'append',
+		'break',
+		'declare',
+		'demangle',
+		'end',
+		'for',
+		'getdatatype',
+		'if',
+		'inmodule',
+		'loop',
+		'mangle',
+		'onwarning',
+		'option',
+		'set',
+		'stored',
+		'uniquename'
+	].join('|'),
+
+	keywords: [
+		'__compressed__',
+		'after',
+		'all',
+		'and',
+		'any',
+		'as',
+		'atmost',
+		'before',
+		'beginc',
+		'best',
+		'between',
+		'case',
+		'cluster',
+		'compressed',
+		'compression',
+		'const',
+		'counter',
+		'csv',
+		'default',
+		'descend',
+		'embed',
+		'encoding',
+		'encrypt',
+		'end',
+		'endc',
+		'endembed',
+		'endmacro',
+		'enum',
+		'escape',
+		'except',
+		'exclusive',
+		'expire',
+		'export',
+		'extend',
+		'fail',
+		'few',
+		'fileposition',
+		'first',
+		'flat',
+		'forward',
+		'from',
+		'full',
+		'function',
+		'functionmacro',
+		'group',
+		'grouped',
+		'heading',
+		'hole',
+		'ifblock',
+		'import',
+		'in',
+		'inner',
+		'interface',
+		'internal',
+		'joined',
+		'keep',
+		'keyed',
+		'last',
+		'left',
+		'limit',
+		'linkcounted',
+		'literal',
+		'little_endian',
+		'load',
+		'local',
+		'locale',
+		'lookup',
+		'lzw',
+		'macro',
+		'many',
+		'maxcount',
+		'maxlength',
+		'min skew',
+		'module',
+		'mofn',
+		'multiple',
+		'named',
+		'namespace',
+		'nocase',
+		'noroot',
+		'noscan',
+		'nosort',
+		'not',
+		'noxpath',
+		'of',
+		'onfail',
+		'only',
+		'opt',
+		'or',
+		'outer',
+		'overwrite',
+		'packed',
+		'partition',
+		'penalty',
+		'physicallength',
+		'pipe',
+		'prefetch',
+		'quote',
+		'record',
+		'repeat',
+		'retry',
+		'return',
+		'right',
+		'right1',
+		'right2',
+		'rows',
+		'rowset',
+		'scan',
+		'scope',
+		'self',
+		'separator',
+		'service',
+		'shared',
+		'skew',
+		'skip',
+		'smart',
+		'soapaction',
+		'sql',
+		'stable',
+		'store',
+		'terminator',
+		'thor',
+		'threshold',
+		'timelimit',
+		'timeout',
+		'token',
+		'transform',
+		'trim',
+		'type',
+		'unicodeorder',
+		'unordered',
+		'unsorted',
+		'unstable',
+		'update',
+		'use',
+		'validate',
+		'virtual',
+		'whole',
+		'width',
+		'wild',
+		'within',
+		'wnotrim',
+		'xml',
+		'xpath'
+	],
+
+	functions: [
+		'abs',
+		'acos',
+		'aggregate',
+		'allnodes',
+		'apply',
+		'ascii',
+		'asin',
+		'assert',
+		'asstring',
+		'atan',
+		'atan2',
+		'ave',
+		'build',
+		'buildindex',
+		'case',
+		'catch',
+		'choose',
+		'choosen',
+		'choosesets',
+		'clustersize',
+		'combine',
+		'correlation',
+		'cos',
+		'cosh',
+		'count',
+		'covariance',
+		'cron',
+		'dataset',
+		'dedup',
+		'define',
+		'denormalize',
+		'dictionary',
+		'distribute',
+		'distributed',
+		'distribution',
+		'ebcdic',
+		'enth',
+		'error',
+		'evaluate',
+		'event',
+		'eventextra',
+		'eventname',
+		'exists',
+		'exp',
+		'fail',
+		'failcode',
+		'failmessage',
+		'fetch',
+		'fromunicode',
+		'fromxml',
+		'getenv',
+		'getisvalid',
+		'global',
+		'graph',
+		'group',
+		'hash',
+		'hash32',
+		'hash64',
+		'hashcrc',
+		'hashmd5',
+		'having',
+		'httpcall',
+		'httpheader',
+		'if',
+		'iff',
+		'index',
+		'intformat',
+		'isvalid',
+		'iterate',
+		'join',
+		'keydiff',
+		'keypatch',
+		'keyunicode',
+		'length',
+		'library',
+		'limit',
+		'ln',
+		'loadxml',
+		'local',
+		'log',
+		'loop',
+		'map',
+		'matched',
+		'matchlength',
+		'matchposition',
+		'matchtext',
+		'matchunicode',
+		'max',
+		'merge',
+		'mergejoin',
+		'min',
+		'nofold',
+		'nolocal',
+		'nonempty',
+		'normalize',
+		'nothor',
+		'notify',
+		'output',
+		'parallel',
+		'parse',
+		'pipe',
+		'power',
+		'preload',
+		'process',
+		'project',
+		'pull',
+		'random',
+		'range',
+		'rank',
+		'ranked',
+		'realformat',
+		'recordof',
+		'regexfind',
+		'regexreplace',
+		'regroup',
+		'rejected',
+		'rollup',
+		'round',
+		'roundup',
+		'row',
+		'rowdiff',
+		'sample',
+		'sequential',
+		'set',
+		'sin',
+		'sinh',
+		'sizeof',
+		'soapcall',
+		'sort',
+		'sorted',
+		'sqrt',
+		'stepped',
+		'stored',
+		'sum',
+		'table',
+		'tan',
+		'tanh',
+		'thisnode',
+		'topn',
+		'tounicode',
+		'toxml',
+		'transfer',
+		'transform',
+		'trim',
+		'truncate',
+		'typeof',
+		'ungroup',
+		'unicodeorder',
+		'variance',
+		'wait',
+		'which',
+		'workunit',
+		'xmldecode',
+		'xmlencode',
+		'xmltext',
+		'xmlunicode'
+	],
+
+	typesint: ['integer', 'unsigned'].join('|'),
+
+	typesnum: ['data', 'qstring', 'string', 'unicode', 'utf8', 'varstring', 'varunicode'],
+
+	typesone: [
+		'ascii',
+		'big_endian',
+		'boolean',
+		'data',
+		'decimal',
+		'ebcdic',
+		'grouped',
+		'integer',
+		'linkcounted',
+		'pattern',
+		'qstring',
+		'real',
+		'record',
+		'rule',
+		'set of',
+		'streamed',
+		'string',
+		'token',
+		'udecimal',
+		'unicode',
+		'unsigned',
+		'utf8',
+		'varstring',
+		'varunicode'
+	].join('|'),
+
+	operators: ['+', '-', '/', ':=', '<', '<>', '=', '>', '\\', 'and', 'in', 'not', 'or'],
+
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+
+	// escape sequences
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			[/@typesint[4|8]/, 'type'],
+			[/#(@pounds)/, 'type'],
+			[/@typesone/, 'type'],
+
+			[
+				/[a-zA-Z_$][\w-$]*/,
+				{
+					cases: {
+						'@functions': 'keyword.function',
+						'@keywords': 'keyword',
+						'@operators': 'operator'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// numbers
+			[/[0-9_]*\.[0-9_]+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F_]+/, 'number.hex'],
+			[/0[bB][01]+/, 'number.hex'], // binary: use same theme style as hex
+			[/[0-9_]+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'],
+			[/"/, 'string', '@string'],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\v\f\r\n]+/, ''],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		string: [
+			[/[^\\']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/'/, 'string', '@pop']
+		]
+	}
+};

monaco-languages/src/elixir/elixir.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'elixir',
+	extensions: ['.ex', '.exs'],
+	aliases: ['Elixir', 'elixir', 'ex'],
+	loader: () => import('./elixir')
+});

monaco-languages/src/elixir/elixir.test.ts

@@ -0,0 +1,376 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('elixir', [
+	// Keywords - module definition
+	[
+		{
+			line: 'defmodule Foo do end',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.declaration.elixir' },
+				{ startIndex: 9, type: 'white.elixir' },
+				{ startIndex: 10, type: 'type.identifier.elixir' },
+				{ startIndex: 13, type: 'white.elixir' },
+				{ startIndex: 14, type: 'keyword.elixir' },
+				{ startIndex: 16, type: 'white.elixir' },
+				{ startIndex: 17, type: 'keyword.elixir' }
+			]
+		}
+	],
+	// Keywords - function definition
+	[
+		{
+			line: 'def foo(x) do end',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.declaration.elixir' },
+				{ startIndex: 3, type: 'white.elixir' },
+				{ startIndex: 4, type: 'function.elixir' },
+				{ startIndex: 7, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 8, type: 'identifier.elixir' },
+				{ startIndex: 9, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 10, type: 'white.elixir' },
+				{ startIndex: 11, type: 'keyword.elixir' },
+				{ startIndex: 13, type: 'white.elixir' },
+				{ startIndex: 14, type: 'keyword.elixir' }
+			]
+		}
+	],
+	// Keywords - macro
+	[
+		{
+			line: 'defmacro mac(name) do quote do def unquote(name)() do nil end end end',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.declaration.elixir' },
+				{ startIndex: 8, type: 'white.elixir' },
+				{ startIndex: 9, type: 'function.elixir' },
+				{ startIndex: 12, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 13, type: 'identifier.elixir' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 18, type: 'white.elixir' },
+				{ startIndex: 19, type: 'keyword.elixir' },
+				{ startIndex: 21, type: 'white.elixir' },
+				{ startIndex: 22, type: 'keyword.elixir' },
+				{ startIndex: 27, type: 'white.elixir' },
+				{ startIndex: 28, type: 'keyword.elixir' },
+				{ startIndex: 30, type: 'white.elixir' },
+				{ startIndex: 31, type: 'keyword.declaration.elixir' },
+				{ startIndex: 34, type: 'white.elixir' },
+				{ startIndex: 35, type: 'keyword.elixir' },
+				{ startIndex: 42, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 43, type: 'identifier.elixir' },
+				{ startIndex: 47, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 50, type: 'white.elixir' },
+				{ startIndex: 51, type: 'keyword.elixir' },
+				{ startIndex: 53, type: 'white.elixir' },
+				{ startIndex: 54, type: 'constant.language.elixir' },
+				{ startIndex: 57, type: 'white.elixir' },
+				{ startIndex: 58, type: 'keyword.elixir' },
+				{ startIndex: 61, type: 'white.elixir' },
+				{ startIndex: 62, type: 'keyword.elixir' },
+				{ startIndex: 65, type: 'white.elixir' },
+				{ startIndex: 66, type: 'keyword.elixir' }
+			]
+		}
+	],
+	// Comments
+	[
+		{
+			line: 'nil # comment',
+			tokens: [
+				{ startIndex: 0, type: 'constant.language.elixir' },
+				{ startIndex: 3, type: 'white.elixir' },
+				{ startIndex: 4, type: 'comment.punctuation.elixir' },
+				{ startIndex: 5, type: 'comment.elixir' }
+			]
+		}
+	],
+	// Keyword list shorthand
+	[
+		{
+			line: '["key": value]',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.square.elixir' },
+				{ startIndex: 1, type: 'constant.delimiter.elixir' },
+				{ startIndex: 2, type: 'constant.elixir' },
+				{ startIndex: 5, type: 'constant.delimiter.elixir' },
+				{ startIndex: 7, type: 'white.elixir' },
+				{ startIndex: 8, type: 'identifier.elixir' },
+				{ startIndex: 13, type: 'delimiter.square.elixir' }
+			]
+		}
+	],
+	// Numbers
+	[
+		{
+			line: '[1,1.23,1.23e-10,0xab,0o171,0b01001]',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.square.elixir' },
+				{ startIndex: 1, type: 'number.elixir' },
+				{ startIndex: 2, type: 'punctuation.elixir' },
+				{ startIndex: 3, type: 'number.float.elixir' },
+				{ startIndex: 7, type: 'punctuation.elixir' },
+				{ startIndex: 8, type: 'number.float.elixir' },
+				{ startIndex: 16, type: 'punctuation.elixir' },
+				{ startIndex: 17, type: 'number.hex.elixir' },
+				{ startIndex: 21, type: 'punctuation.elixir' },
+				{ startIndex: 22, type: 'number.octal.elixir' },
+				{ startIndex: 27, type: 'punctuation.elixir' },
+				{ startIndex: 28, type: 'number.binary.elixir' },
+				{ startIndex: 35, type: 'delimiter.square.elixir' }
+			]
+		}
+	],
+	// Unused bindings
+	[
+		{
+			line: 'def foo(_x) do _y = 1 end',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.declaration.elixir' },
+				{ startIndex: 3, type: 'white.elixir' },
+				{ startIndex: 4, type: 'function.elixir' },
+				{ startIndex: 7, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 8, type: 'comment.unused.elixir' },
+				{ startIndex: 10, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 11, type: 'white.elixir' },
+				{ startIndex: 12, type: 'keyword.elixir' },
+				{ startIndex: 14, type: 'white.elixir' },
+				{ startIndex: 15, type: 'comment.unused.elixir' },
+				{ startIndex: 17, type: 'white.elixir' },
+				{ startIndex: 18, type: 'operator.elixir' },
+				{ startIndex: 19, type: 'white.elixir' },
+				{ startIndex: 20, type: 'number.elixir' },
+				{ startIndex: 21, type: 'white.elixir' },
+				{ startIndex: 22, type: 'keyword.elixir' }
+			]
+		}
+	],
+	// Function calls
+	[
+		{
+			line: 'foo(x)',
+			tokens: [
+				{ startIndex: 0, type: 'function.call.elixir' },
+				{ startIndex: 3, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 4, type: 'identifier.elixir' },
+				{ startIndex: 5, type: 'delimiter.parenthesis.elixir' }
+			]
+		}
+	],
+	[
+		{
+			line: 'foo.()',
+			tokens: [
+				{ startIndex: 0, type: 'function.call.elixir' },
+				{ startIndex: 3, type: 'operator.elixir' },
+				{ startIndex: 4, type: 'delimiter.parenthesis.elixir' }
+			]
+		}
+	],
+	[
+		{
+			line: 'Mod.foo()',
+			tokens: [
+				{ startIndex: 0, type: 'type.identifier.elixir' },
+				{ startIndex: 3, type: 'operator.elixir' },
+				{ startIndex: 4, type: 'function.call.elixir' },
+				{ startIndex: 7, type: 'delimiter.parenthesis.elixir' }
+			]
+		}
+	],
+	// Function call (Erlang module)
+	[
+		{
+			line: ':mo.foo()',
+			tokens: [
+				{ startIndex: 0, type: 'constant.punctuation.elixir' },
+				{ startIndex: 1, type: 'constant.elixir' },
+				{ startIndex: 3, type: 'operator.elixir' },
+				{ startIndex: 4, type: 'function.call.elixir' },
+				{ startIndex: 7, type: 'delimiter.parenthesis.elixir' }
+			]
+		}
+	],
+	// Function call (pipe)
+	[
+		{
+			line: '1 |> abs()',
+			tokens: [
+				{ startIndex: 0, type: 'number.elixir' },
+				{ startIndex: 1, type: 'white.elixir' },
+				{ startIndex: 2, type: 'operator.elixir' },
+				{ startIndex: 4, type: 'white.elixir' },
+				{ startIndex: 5, type: 'function.call.elixir' },
+				{ startIndex: 8, type: 'delimiter.parenthesis.elixir' }
+			]
+		}
+	],
+	// Function reference
+	[
+		{
+			line: '&max(&1,&2)',
+			tokens: [
+				{ startIndex: 0, type: 'operator.elixir' },
+				{ startIndex: 1, type: 'function.call.elixir' },
+				{ startIndex: 4, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 5, type: 'operator.elixir' },
+				{ startIndex: 7, type: 'punctuation.elixir' },
+				{ startIndex: 8, type: 'operator.elixir' },
+				{ startIndex: 10, type: 'delimiter.parenthesis.elixir' }
+			]
+		}
+	],
+	// Strings
+	[
+		{
+			line: '"foo"',
+			tokens: [
+				{ startIndex: 0, type: 'string.delimiter.elixir' },
+				{ startIndex: 1, type: 'string.elixir' },
+				{ startIndex: 4, type: 'string.delimiter.elixir' }
+			]
+		}
+	],
+	[
+		{
+			line: '"foo \\u0065\\u0301 #{1}"',
+			tokens: [
+				{ startIndex: 0, type: 'string.delimiter.elixir' },
+				{ startIndex: 1, type: 'string.elixir' },
+				{ startIndex: 5, type: 'constant.character.escape.elixir' },
+				{ startIndex: 17, type: 'string.elixir' },
+				{ startIndex: 18, type: 'delimiter.bracket.embed.elixir' },
+				{ startIndex: 20, type: 'number.elixir' },
+				{ startIndex: 21, type: 'delimiter.bracket.embed.elixir' },
+				{ startIndex: 22, type: 'string.delimiter.elixir' }
+			]
+		}
+	],
+	[
+		{
+			line: '"""heredoc"""',
+			tokens: [
+				{ startIndex: 0, type: 'string.delimiter.elixir' },
+				{ startIndex: 3, type: 'string.elixir' },
+				{ startIndex: 10, type: 'string.delimiter.elixir' }
+			]
+		}
+	],
+	// Atom strings
+	[
+		{
+			line: ':"atom"',
+			tokens: [
+				{ startIndex: 0, type: 'constant.delimiter.elixir' },
+				{ startIndex: 2, type: 'constant.elixir' },
+				{ startIndex: 6, type: 'constant.delimiter.elixir' }
+			]
+		}
+	],
+	// Sigils (string)
+	[
+		{
+			line: '~s{foo}',
+			tokens: [
+				{ startIndex: 0, type: 'string.delimiter.elixir' },
+				{ startIndex: 3, type: 'string.elixir' },
+				{ startIndex: 6, type: 'string.delimiter.elixir' }
+			]
+		}
+	],
+	// Sigils (regexp)
+	[
+		{
+			line: '~r/foo/',
+			tokens: [
+				{ startIndex: 0, type: 'regexp.delimiter.elixir' },
+				{ startIndex: 3, type: 'regexp.elixir' },
+				{ startIndex: 6, type: 'regexp.delimiter.elixir' }
+			]
+		}
+	],
+	// Sigils (other)
+	[
+		{
+			line: '~D/foo/',
+			tokens: [
+				{ startIndex: 0, type: 'sigil.delimiter.elixir' },
+				{ startIndex: 3, type: 'sigil.elixir' },
+				{ startIndex: 6, type: 'sigil.delimiter.elixir' }
+			]
+		}
+	],
+	// Sigils (no interpolation)
+	[
+		{
+			line: '~W/foo#{1}/',
+			tokens: [
+				{ startIndex: 0, type: 'sigil.delimiter.elixir' },
+				{ startIndex: 3, type: 'sigil.elixir' },
+				{ startIndex: 10, type: 'sigil.delimiter.elixir' }
+			]
+		}
+	],
+	// Module attributes
+	[
+		{
+			line: '@attr 1',
+			tokens: [
+				{ startIndex: 0, type: 'variable.elixir' },
+				{ startIndex: 5, type: 'white.elixir' },
+				{ startIndex: 6, type: 'number.elixir' }
+			]
+		}
+	],
+	// Module attributes (docs)
+	[
+		{
+			line: '@doc "foo"',
+			tokens: [{ startIndex: 0, type: 'comment.block.documentation.elixir' }]
+		}
+	],
+	// Operator definition
+	[
+		{
+			line: 'def a ~> b, do: max(a,b)',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.declaration.elixir' },
+				{ startIndex: 3, type: 'white.elixir' },
+				{ startIndex: 4, type: 'identifier.elixir' },
+				{ startIndex: 5, type: 'white.elixir' },
+				{ startIndex: 6, type: 'operator.elixir' },
+				{ startIndex: 8, type: 'white.elixir' },
+				{ startIndex: 9, type: 'identifier.elixir' },
+				{ startIndex: 10, type: 'punctuation.elixir' },
+				{ startIndex: 11, type: 'white.elixir' },
+				{ startIndex: 12, type: 'constant.elixir' },
+				{ startIndex: 14, type: 'constant.punctuation.elixir' },
+				{ startIndex: 15, type: 'white.elixir' },
+				{ startIndex: 16, type: 'function.call.elixir' },
+				{ startIndex: 19, type: 'delimiter.parenthesis.elixir' },
+				{ startIndex: 20, type: 'identifier.elixir' },
+				{ startIndex: 21, type: 'punctuation.elixir' },
+				{ startIndex: 22, type: 'identifier.elixir' },
+				{ startIndex: 23, type: 'delimiter.parenthesis.elixir' }
+			]
+		}
+	],
+	// Constants
+	[
+		{
+			line: '[true,false,nil]',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.square.elixir' },
+				{ startIndex: 1, type: 'constant.language.elixir' },
+				{ startIndex: 5, type: 'punctuation.elixir' },
+				{ startIndex: 6, type: 'constant.language.elixir' },
+				{ startIndex: 11, type: 'punctuation.elixir' },
+				{ startIndex: 12, type: 'constant.language.elixir' },
+				{ startIndex: 15, type: 'delimiter.square.elixir' }
+			]
+		}
+	]
+]);

monaco-languages/src/elixir/elixir.ts

@@ -0,0 +1,632 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: "'", close: "'" },
+		{ open: '"', close: '"' }
+	],
+	autoClosingPairs: [
+		{ open: "'", close: "'", notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['comment'] },
+		{ open: '"""', close: '"""' },
+		{ open: '`', close: '`', notIn: ['string', 'comment'] },
+		{ open: '(', close: ')' },
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '<<', close: '>>' }
+	],
+	indentationRules: {
+		increaseIndentPattern: /^\s*(after|else|catch|rescue|fn|[^#]*(do|<\-|\->|\{|\[|\=))\s*$/,
+		decreaseIndentPattern: /^\s*((\}|\])\s*$|(after|else|catch|rescue|end)\b)/
+	}
+};
+
+/**
+ * A Monarch lexer for the Elixir language.
+ *
+ * References:
+ *
+ * * Monarch documentation - https://microsoft.github.io/monaco-editor/monarch.html
+ * * Elixir lexer - https://github.com/elixir-makeup/makeup_elixir/blob/master/lib/makeup/lexers/elixir_lexer.ex
+ * * TextMate lexer (elixir-tmbundle) - https://github.com/elixir-editors/elixir-tmbundle/blob/master/Syntaxes/Elixir.tmLanguage
+ * * TextMate lexer (vscode-elixir-ls) - https://github.com/elixir-lsp/vscode-elixir-ls/blob/master/syntaxes/elixir.json
+ */
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: 'source',
+	tokenPostfix: '.elixir',
+
+	brackets: [
+		{ open: '[', close: ']', token: 'delimiter.square' },
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '<<', close: '>>', token: 'delimiter.angle.special' }
+	],
+
+	// Below are lists/regexps to which we reference later.
+
+	declarationKeywords: [
+		'def',
+		'defp',
+		'defn',
+		'defnp',
+		'defguard',
+		'defguardp',
+		'defmacro',
+		'defmacrop',
+		'defdelegate',
+		'defcallback',
+		'defmacrocallback',
+		'defmodule',
+		'defprotocol',
+		'defexception',
+		'defimpl',
+		'defstruct'
+	],
+	operatorKeywords: ['and', 'in', 'not', 'or', 'when'],
+	namespaceKeywords: ['alias', 'import', 'require', 'use'],
+	otherKeywords: [
+		'after',
+		'case',
+		'catch',
+		'cond',
+		'do',
+		'else',
+		'end',
+		'fn',
+		'for',
+		'if',
+		'quote',
+		'raise',
+		'receive',
+		'rescue',
+		'super',
+		'throw',
+		'try',
+		'unless',
+		'unquote_splicing',
+		'unquote',
+		'with'
+	],
+	constants: ['true', 'false', 'nil'],
+	nameBuiltin: ['__MODULE__', '__DIR__', '__ENV__', '__CALLER__', '__STACKTRACE__'],
+
+	// Matches any of the operator names:
+	// <<< >>> ||| &&& ^^^ ~~~ === !== ~>> <~> |~> <|> == != <= >= && || \\ <> ++ -- |> =~ -> <- ~> <~ :: .. = < > + - * / | . ^ & !
+	operator:
+		/-[->]?|!={0,2}|\*|\/|\\\\|&{1,3}|\.\.?|\^(?:\^\^)?|\+\+?|<(?:-|<<|=|>|\|>|~>?)?|=~|={1,3}|>(?:=|>>)?|\|~>|\|>|\|{1,3}|~>>?|~~~|::/,
+
+	// See https://hexdocs.pm/elixir/syntax-reference.html#variables
+	variableName: /[a-z_][a-zA-Z0-9_]*[?!]?/,
+
+	// See https://hexdocs.pm/elixir/syntax-reference.html#atoms
+	atomName: /[a-zA-Z_][a-zA-Z0-9_@]*[?!]?|@specialAtomName|@operator/,
+	specialAtomName: /\.\.\.|<<>>|%\{\}|%|\{\}/,
+
+	aliasPart: /[A-Z][a-zA-Z0-9_]*/,
+	moduleName: /@aliasPart(?:\.@aliasPart)*/,
+
+	// Sigil pairs are: """ """, ''' ''', " ", ' ', / /, | |, < >, { }, [ ], ( )
+	sigilSymmetricDelimiter: /"""|'''|"|'|\/|\|/,
+	sigilStartDelimiter: /@sigilSymmetricDelimiter|<|\{|\[|\(/,
+	sigilEndDelimiter: /@sigilSymmetricDelimiter|>|\}|\]|\)/,
+
+	decimal: /\d(?:_?\d)*/,
+	hex: /[0-9a-fA-F](_?[0-9a-fA-F])*/,
+	octal: /[0-7](_?[0-7])*/,
+	binary: /[01](_?[01])*/,
+
+	// See https://hexdocs.pm/elixir/master/String.html#module-escape-characters
+	escape: /\\u[0-9a-fA-F]{4}|\\x[0-9a-fA-F]{2}|\\./,
+
+	// The keys below correspond to tokenizer states.
+	// We start from the root state and match against its rules
+	// until we explicitly transition into another state.
+	// The `include` simply brings in all operations from the given state
+	// and is useful for improving readability.
+	tokenizer: {
+		root: [
+			{ include: '@whitespace' },
+			{ include: '@comments' },
+			// Keywords start as either an identifier or a string,
+			// but end with a : so it's important to match this first.
+			{ include: '@keywordsShorthand' },
+			{ include: '@numbers' },
+			{ include: '@identifiers' },
+			{ include: '@strings' },
+			{ include: '@atoms' },
+			{ include: '@sigils' },
+			{ include: '@attributes' },
+			{ include: '@symbols' }
+		],
+
+		// Whitespace
+
+		whitespace: [[/\s+/, 'white']],
+
+		// Comments
+
+		comments: [[/(#)(.*)/, ['comment.punctuation', 'comment']]],
+
+		// Keyword list shorthand
+
+		keywordsShorthand: [
+			[/(@atomName)(:)/, ['constant', 'constant.punctuation']],
+			// Use positive look-ahead to ensure the string is followed by :
+			// and should be considered a keyword.
+			[
+				/"(?=([^"]|#\{.*?\}|\\")*":)/,
+				{ token: 'constant.delimiter', next: '@doubleQuotedStringKeyword' }
+			],
+			[
+				/'(?=([^']|#\{.*?\}|\\')*':)/,
+				{ token: 'constant.delimiter', next: '@singleQuotedStringKeyword' }
+			]
+		],
+
+		doubleQuotedStringKeyword: [
+			[/":/, { token: 'constant.delimiter', next: '@pop' }],
+			{ include: '@stringConstantContentInterpol' }
+		],
+
+		singleQuotedStringKeyword: [
+			[/':/, { token: 'constant.delimiter', next: '@pop' }],
+			{ include: '@stringConstantContentInterpol' }
+		],
+
+		// Numbers
+
+		numbers: [
+			[/0b@binary/, 'number.binary'],
+			[/0o@octal/, 'number.octal'],
+			[/0x@hex/, 'number.hex'],
+			[/@decimal\.@decimal([eE]-?@decimal)?/, 'number.float'],
+			[/@decimal/, 'number']
+		],
+
+		// Identifiers
+
+		identifiers: [
+			// Tokenize identifier name in function-like definitions.
+			// Note: given `def a + b, do: nil`, `a` is not a function name,
+			// so we use negative look-ahead to ensure there's no operator.
+			[
+				/\b(defp?|defnp?|defmacrop?|defguardp?|defdelegate)(\s+)(@variableName)(?!\s+@operator)/,
+				[
+					'keyword.declaration',
+					'white',
+					{
+						cases: {
+							unquote: 'keyword',
+							'@default': 'function'
+						}
+					}
+				]
+			],
+			// Tokenize function calls
+			[
+				// In-scope call - an identifier followed by ( or .(
+				/(@variableName)(?=\s*\.?\s*\()/,
+				{
+					cases: {
+						// Tokenize as keyword in cases like `if(..., do: ..., else: ...)`
+						'@declarationKeywords': 'keyword.declaration',
+						'@namespaceKeywords': 'keyword',
+						'@otherKeywords': 'keyword',
+						'@default': 'function.call'
+					}
+				}
+			],
+			[
+				// Referencing function in a module
+				/(@moduleName)(\s*)(\.)(\s*)(@variableName)/,
+				['type.identifier', 'white', 'operator', 'white', 'function.call']
+			],
+			[
+				// Referencing function in an Erlang module
+				/(:)(@atomName)(\s*)(\.)(\s*)(@variableName)/,
+				['constant.punctuation', 'constant', 'white', 'operator', 'white', 'function.call']
+			],
+			[
+				// Piping into a function (tokenized separately as it may not have parentheses)
+				/(\|>)(\s*)(@variableName)/,
+				[
+					'operator',
+					'white',
+					{
+						cases: {
+							'@otherKeywords': 'keyword',
+							'@default': 'function.call'
+						}
+					}
+				]
+			],
+			[
+				// Function reference passed to another function
+				/(&)(\s*)(@variableName)/,
+				['operator', 'white', 'function.call']
+			],
+			// Language keywords, builtins, constants and variables
+			[
+				/@variableName/,
+				{
+					cases: {
+						'@declarationKeywords': 'keyword.declaration',
+						'@operatorKeywords': 'keyword.operator',
+						'@namespaceKeywords': 'keyword',
+						'@otherKeywords': 'keyword',
+						'@constants': 'constant.language',
+						'@nameBuiltin': 'variable.language',
+						'_.*': 'comment.unused',
+						'@default': 'identifier'
+					}
+				}
+			],
+			// Module names
+			[/@moduleName/, 'type.identifier']
+		],
+
+		// Strings
+
+		strings: [
+			[/"""/, { token: 'string.delimiter', next: '@doubleQuotedHeredoc' }],
+			[/'''/, { token: 'string.delimiter', next: '@singleQuotedHeredoc' }],
+			[/"/, { token: 'string.delimiter', next: '@doubleQuotedString' }],
+			[/'/, { token: 'string.delimiter', next: '@singleQuotedString' }]
+		],
+
+		doubleQuotedHeredoc: [
+			[/"""/, { token: 'string.delimiter', next: '@pop' }],
+			{ include: '@stringContentInterpol' }
+		],
+
+		singleQuotedHeredoc: [
+			[/'''/, { token: 'string.delimiter', next: '@pop' }],
+			{ include: '@stringContentInterpol' }
+		],
+
+		doubleQuotedString: [
+			[/"/, { token: 'string.delimiter', next: '@pop' }],
+			{ include: '@stringContentInterpol' }
+		],
+
+		singleQuotedString: [
+			[/'/, { token: 'string.delimiter', next: '@pop' }],
+			{ include: '@stringContentInterpol' }
+		],
+
+		// Atoms
+
+		atoms: [
+			[/(:)(@atomName)/, ['constant.punctuation', 'constant']],
+			[/:"/, { token: 'constant.delimiter', next: '@doubleQuotedStringAtom' }],
+			[/:'/, { token: 'constant.delimiter', next: '@singleQuotedStringAtom' }]
+		],
+
+		doubleQuotedStringAtom: [
+			[/"/, { token: 'constant.delimiter', next: '@pop' }],
+			{ include: '@stringConstantContentInterpol' }
+		],
+
+		singleQuotedStringAtom: [
+			[/'/, { token: 'constant.delimiter', next: '@pop' }],
+			{ include: '@stringConstantContentInterpol' }
+		],
+
+		// Sigils
+
+		// See https://elixir-lang.org/getting-started/sigils.html
+		// Sigils allow for typing values using their textual representation.
+		// All sigils start with ~ followed by a letter indicating sigil type
+		// and then a delimiter pair enclosing the textual representation.
+		// Optional modifiers are allowed after the closing delimiter.
+		// For instance a regular expressions can be written as:
+		// ~r/foo|bar/ ~r{foo|bar} ~r/foo|bar/g
+		//
+		// In general lowercase sigils allow for interpolation
+		// and escaped characters, whereas uppercase sigils don't
+		//
+		// During tokenization we want to distinguish some
+		// specific sigil types, namely string and regexp,
+		// so that they cen be themed separately.
+		//
+		// To reasonably handle all those combinations we leverage
+		// dot-separated states, so if we transition to @sigilStart.interpol.s.{.}
+		// then "sigilStart.interpol.s" state will match and also all
+		// the individual dot-separated parameters can be accessed.
+
+		sigils: [
+			[/~[a-z]@sigilStartDelimiter/, { token: '@rematch', next: '@sigil.interpol' }],
+			[/~[A-Z]@sigilStartDelimiter/, { token: '@rematch', next: '@sigil.noInterpol' }]
+		],
+
+		sigil: [
+			[/~([a-zA-Z])\{/, { token: '@rematch', switchTo: '@sigilStart.$S2.$1.{.}' }],
+			[/~([a-zA-Z])\[/, { token: '@rematch', switchTo: '@sigilStart.$S2.$1.[.]' }],
+			[/~([a-zA-Z])\(/, { token: '@rematch', switchTo: '@sigilStart.$S2.$1.(.)' }],
+			[/~([a-zA-Z])\</, { token: '@rematch', switchTo: '@sigilStart.$S2.$1.<.>' }],
+			[
+				/~([a-zA-Z])(@sigilSymmetricDelimiter)/,
+				{ token: '@rematch', switchTo: '@sigilStart.$S2.$1.$2.$2' }
+			]
+		],
+
+		// The definitions below expect states to be of the form:
+		//
+		// sigilStart.<interpol-or-noInterpol>.<sigil-letter>.<start-delimiter>.<end-delimiter>
+		// sigilContinue.<interpol-or-noInterpol>.<sigil-letter>.<start-delimiter>.<end-delimiter>
+		//
+		// The sigilStart state is used only to properly classify the token (as string/regex/sigil)
+		// and immediately switches to the sigilContinue sate, which handles the actual content
+		// and waits for the corresponding end delimiter.
+
+		'sigilStart.interpol.s': [
+			[
+				/~s@sigilStartDelimiter/,
+				{
+					token: 'string.delimiter',
+					switchTo: '@sigilContinue.$S2.$S3.$S4.$S5'
+				}
+			]
+		],
+
+		'sigilContinue.interpol.s': [
+			[
+				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				{
+					cases: {
+						'$1==$S5': { token: 'string.delimiter', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			],
+			{ include: '@stringContentInterpol' }
+		],
+
+		'sigilStart.noInterpol.S': [
+			[
+				/~S@sigilStartDelimiter/,
+				{
+					token: 'string.delimiter',
+					switchTo: '@sigilContinue.$S2.$S3.$S4.$S5'
+				}
+			]
+		],
+
+		'sigilContinue.noInterpol.S': [
+			// Ignore escaped sigil end
+			[/(^|[^\\])\\@sigilEndDelimiter/, 'string'],
+			[
+				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				{
+					cases: {
+						'$1==$S5': { token: 'string.delimiter', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			],
+			{ include: '@stringContent' }
+		],
+
+		'sigilStart.interpol.r': [
+			[
+				/~r@sigilStartDelimiter/,
+				{
+					token: 'regexp.delimiter',
+					switchTo: '@sigilContinue.$S2.$S3.$S4.$S5'
+				}
+			]
+		],
+
+		'sigilContinue.interpol.r': [
+			[
+				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				{
+					cases: {
+						'$1==$S5': { token: 'regexp.delimiter', next: '@pop' },
+						'@default': 'regexp'
+					}
+				}
+			],
+			{ include: '@regexpContentInterpol' }
+		],
+
+		'sigilStart.noInterpol.R': [
+			[
+				/~R@sigilStartDelimiter/,
+				{
+					token: 'regexp.delimiter',
+					switchTo: '@sigilContinue.$S2.$S3.$S4.$S5'
+				}
+			]
+		],
+
+		'sigilContinue.noInterpol.R': [
+			// Ignore escaped sigil end
+			[/(^|[^\\])\\@sigilEndDelimiter/, 'regexp'],
+			[
+				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				{
+					cases: {
+						'$1==$S5': { token: 'regexp.delimiter', next: '@pop' },
+						'@default': 'regexp'
+					}
+				}
+			],
+			{ include: '@regexpContent' }
+		],
+
+		// Fallback to the generic sigil by default
+		'sigilStart.interpol': [
+			[
+				/~([a-zA-Z])@sigilStartDelimiter/,
+				{
+					token: 'sigil.delimiter',
+					switchTo: '@sigilContinue.$S2.$S3.$S4.$S5'
+				}
+			]
+		],
+
+		'sigilContinue.interpol': [
+			[
+				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				{
+					cases: {
+						'$1==$S5': { token: 'sigil.delimiter', next: '@pop' },
+						'@default': 'sigil'
+					}
+				}
+			],
+			{ include: '@sigilContentInterpol' }
+		],
+
+		'sigilStart.noInterpol': [
+			[
+				/~([a-zA-Z])@sigilStartDelimiter/,
+				{
+					token: 'sigil.delimiter',
+					switchTo: '@sigilContinue.$S2.$S3.$S4.$S5'
+				}
+			]
+		],
+
+		'sigilContinue.noInterpol': [
+			// Ignore escaped sigil end
+			[/(^|[^\\])\\@sigilEndDelimiter/, 'sigil'],
+			[
+				/(@sigilEndDelimiter)[a-zA-Z]*/,
+				{
+					cases: {
+						'$1==$S5': { token: 'sigil.delimiter', next: '@pop' },
+						'@default': 'sigil'
+					}
+				}
+			],
+			{ include: '@sigilContent' }
+		],
+
+		// Attributes
+
+		attributes: [
+			// Module @doc* attributes - tokenized as comments
+			[
+				/\@(module|type)?doc (~[sS])?"""/,
+				{
+					token: 'comment.block.documentation',
+					next: '@doubleQuotedHeredocDocstring'
+				}
+			],
+			[
+				/\@(module|type)?doc (~[sS])?"/,
+				{
+					token: 'comment.block.documentation',
+					next: '@doubleQuotedStringDocstring'
+				}
+			],
+			[/\@(module|type)?doc false/, 'comment.block.documentation'],
+			// Module attributes
+			[/\@(@variableName)/, 'variable']
+		],
+
+		doubleQuotedHeredocDocstring: [
+			[/"""/, { token: 'comment.block.documentation', next: '@pop' }],
+			{ include: '@docstringContent' }
+		],
+
+		doubleQuotedStringDocstring: [
+			[/"/, { token: 'comment.block.documentation', next: '@pop' }],
+			{ include: '@docstringContent' }
+		],
+
+		// Operators, punctuation, brackets
+
+		symbols: [
+			// Code point operator (either with regular character ?a or an escaped one ?\n)
+			[/\?(\\.|[^\\\s])/, 'number.constant'],
+			// Anonymous function arguments
+			[/&\d+/, 'operator'],
+			// Bitshift operators (must go before delimiters, so that << >> don't match first)
+			[/<<<|>>>/, 'operator'],
+			// Delimiter pairs
+			[/[()\[\]\{\}]|<<|>>/, '@brackets'],
+			// Triple dot is a valid name (must go before operators, so that .. doesn't match instead)
+			[/\.\.\./, 'identifier'],
+			// Punctuation => (must go before operators, so it's not tokenized as = then >)
+			[/=>/, 'punctuation'],
+			// Operators
+			[/@operator/, 'operator'],
+			// Punctuation
+			[/[:;,.%]/, 'punctuation']
+		],
+
+		// Generic helpers
+
+		stringContentInterpol: [
+			{ include: '@interpolation' },
+			{ include: '@escapeChar' },
+			{ include: '@stringContent' }
+		],
+
+		stringContent: [[/./, 'string']],
+
+		stringConstantContentInterpol: [
+			{ include: '@interpolation' },
+			{ include: '@escapeChar' },
+			{ include: '@stringConstantContent' }
+		],
+
+		stringConstantContent: [[/./, 'constant']],
+
+		regexpContentInterpol: [
+			{ include: '@interpolation' },
+			{ include: '@escapeChar' },
+			{ include: '@regexpContent' }
+		],
+
+		regexpContent: [
+			// # may be a regular regexp char, so we use a heuristic
+			// assuming a # surrounded by whitespace is actually a comment.
+			[/(\s)(#)(\s.*)$/, ['white', 'comment.punctuation', 'comment']],
+			[/./, 'regexp']
+		],
+
+		sigilContentInterpol: [
+			{ include: '@interpolation' },
+			{ include: '@escapeChar' },
+			{ include: '@sigilContent' }
+		],
+
+		sigilContent: [[/./, 'sigil']],
+
+		docstringContent: [[/./, 'comment.block.documentation']],
+
+		escapeChar: [[/@escape/, 'constant.character.escape']],
+
+		interpolation: [
+			[/#{/, { token: 'delimiter.bracket.embed', next: '@interpolationContinue' }]
+		],
+
+		interpolationContinue: [
+			[/}/, { token: 'delimiter.bracket.embed', next: '@pop' }],
+			// Interpolation brackets may contain arbitrary code,
+			// so we simply match against all the root rules,
+			// until we reach interpolation end (the above matches).
+			{ include: '@root' }
+		]
+	}
+};

monaco-languages/src/fillers/monaco-editor-core-amd.ts

@@ -0,0 +1,12 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+// Resolves with the global monaco API
+
+declare const define: any;
+
+define([], function () {
+	return (<any>self).monaco;
+});

monaco-languages/src/fillers/monaco-editor-core.ts

@@ -0,0 +1,6 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+export * from 'monaco-editor-core';

monaco-languages/src/flow9/flow9.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'flow9',
+	extensions: ['.flow'],
+	aliases: ['Flow9', 'Flow', 'flow9', 'flow'],
+	loader: () => import('./flow9')
+});

monaco-languages/src/flow9/flow9.test.ts

@@ -0,0 +1,148 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('flow9', [
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.flow' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.flow' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '/* //*/ a',
+			tokens: [
+				{ startIndex: 0, type: 'comment.flow' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.flow' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '/import file 1',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.flow' },
+				{ startIndex: 1, type: 'keyword.flow' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.flow' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'number.flow' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'getDefaults() -> [int] {}',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.flow' },
+				{ startIndex: 11, type: 'delimiter.flow' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'delimiter.flow' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'delimiter.flow' },
+				{ startIndex: 18, type: 'type.flow' },
+				{ startIndex: 21, type: 'delimiter.flow' },
+				{ startIndex: 22, type: '' },
+				{ startIndex: 23, type: 'delimiter.flow' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.flow' }]
+		}
+	],
+
+	[
+		{
+			line: '0.10',
+			tokens: [{ startIndex: 0, type: 'number.flow' }]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.flow' }]
+		}
+	],
+
+	[
+		{
+			line: '052_',
+			tokens: [
+				{ startIndex: 0, type: 'number.flow' },
+				{ startIndex: 3, type: 'identifier.flow' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.flow' },
+				{ startIndex: 1, type: 'delimiter.flow' },
+				{ startIndex: 2, type: 'number.flow' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.flow' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.flow' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.flow' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"simple string"',
+			tokens: [{ startIndex: 0, type: 'string.flow' }]
+		}
+	],
+
+	[
+		{
+			line: '""',
+			tokens: [{ startIndex: 0, type: 'string.flow' }]
+		}
+	],
+
+	[
+		{
+			line: '"""',
+			tokens: [
+				{ startIndex: 0, type: 'string.flow' },
+				{ startIndex: 2, type: 'string.invalid.flow' }
+			]
+		}
+	]
+]);

monaco-languages/src/flow9/flow9.ts

@@ -0,0 +1,163 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		blockComment: ['/*', '*/'],
+		lineComment: '//'
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}', notIn: ['string'] },
+		{ open: '[', close: ']', notIn: ['string'] },
+		{ open: '(', close: ')', notIn: ['string'] },
+		{ open: '"', close: '"', notIn: ['string'] },
+		{ open: "'", close: "'", notIn: ['string'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '<', close: '>' }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.flow',
+
+	keywords: [
+		'import',
+		'require',
+		'export',
+		'forbid',
+		'native',
+		'if',
+		'else',
+		'cast',
+		'unsafe',
+		'switch',
+		'default'
+	],
+
+	types: [
+		'io',
+		'mutable',
+		'bool',
+		'int',
+		'double',
+		'string',
+		'flow',
+		'void',
+		'ref',
+		'true',
+		'false',
+		'with'
+	],
+
+	operators: [
+		'=',
+		'>',
+		'<',
+		'<=',
+		'>=',
+		'==',
+		'!',
+		'!=',
+		':=',
+		'::=',
+		'&&',
+		'||',
+		'+',
+		'-',
+		'*',
+		'/',
+		'@',
+		'&',
+		'%',
+		':',
+		'->',
+		'\\',
+		'$',
+		'??',
+		'^'
+	],
+
+	symbols: /[@$=><!~?:&|+\-*\\\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[
+				/[a-zA-Z_]\w*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@types': 'type',
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, 'delimiter'],
+			[/[<>](?!@symbols)/, 'delimiter'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// numbers
+			[
+				/((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)/,
+				'number'
+			],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'],
+			[/"/, 'string', '@string']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		]
+	}
+};

monaco-languages/src/fsharp/fsharp.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'fsharp',
+	extensions: ['.fs', '.fsi', '.ml', '.mli', '.fsx', '.fsscript'],
+	aliases: ['F#', 'FSharp', 'fsharp'],
+	loader: () => import('./fsharp')
+});

monaco-languages/src/fsharp/fsharp.test.ts

@@ -0,0 +1,491 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('fsharp', [
+	// comments - single line
+	[
+		{
+			line: '// one line comment',
+			tokens: [{ startIndex: 0, type: 'comment.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '/almost a comment',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.fs' },
+				{ startIndex: 1, type: 'identifier.fs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.fs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '(/*almost a comment',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.parenthesis.fs' },
+				{ startIndex: 1, type: 'delimiter.fs' },
+				{ startIndex: 3, type: 'identifier.fs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.fs' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'identifier.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1 / 2; (* comment',
+			tokens: [
+				{ startIndex: 0, type: 'number.fs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.fs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.fs' },
+				{ startIndex: 5, type: 'delimiter.fs' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'comment.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'let x = 1; // my comment // is a nice one',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.let.fs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.fs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.fs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.fs' },
+				{ startIndex: 9, type: 'delimiter.fs' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'comment.fs' }
+			]
+		}
+	],
+
+	// Keywords
+	[
+		{
+			line: 'namespace Application1',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.namespace.fs' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'type MyType',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.type.fs' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'identifier.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'module App =',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.module.fs' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.fs' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'delimiter.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'let AppName = "App1"',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.let.fs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.fs' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'delimiter.fs' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'string.fs' }
+			]
+		}
+	],
+
+	// Comments - range comment
+	[
+		{
+			line: '(* a simple comment *)',
+			tokens: [{ startIndex: 0, type: 'comment.fs' }]
+		}
+	],
+
+	[
+		{
+			line: 'let x = (* a simple comment *) 1',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.let.fs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.fs' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.fs' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.fs' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'number.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = (**)',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.fs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.fs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = (*)',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.fs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.fs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'delimiter.parenthesis.fs' },
+				{ startIndex: 5, type: 'delimiter.fs' },
+				{ startIndex: 6, type: 'delimiter.parenthesis.fs' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5F',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5f',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3F',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3f',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3F',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3f',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5M',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5m',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3M',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3m',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3M',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3m',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.fs' },
+				{ startIndex: 1, type: 'delimiter.fs' },
+				{ startIndex: 2, type: 'number.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.fs' },
+				{ startIndex: 3, type: 'delimiter.fs' },
+				{ startIndex: 4, type: 'number.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.fs' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.fs' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.fs' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0b00000101',
+			tokens: [{ startIndex: 0, type: 'number.bin.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86y',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '0b00000101y',
+			tokens: [{ startIndex: 0, type: 'number.bin.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86s',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86us',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86l',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86u',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86ul',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '0x00002D3Fn',
+			tokens: [{ startIndex: 0, type: 'number.hex.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '0x00002D3Fun',
+			tokens: [{ startIndex: 0, type: 'number.hex.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86L',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '86UL',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '9999999999999999999999999999I',
+			tokens: [{ startIndex: 0, type: 'number.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '0x00002D3FLF',
+			tokens: [{ startIndex: 0, type: 'number.float.fs' }]
+		}
+	],
+
+	[
+		{
+			line: '(* This operator -> (*) should be ignored *) let i = 0',
+			tokens: [
+				{ startIndex: 0, type: 'comment.fs' },
+				{ startIndex: 44, type: '' },
+				{ startIndex: 45, type: 'keyword.let.fs' },
+				{ startIndex: 48, type: '' },
+				{ startIndex: 49, type: 'identifier.fs' },
+				{ startIndex: 50, type: '' },
+				{ startIndex: 51, type: 'delimiter.fs' },
+				{ startIndex: 52, type: '' },
+				{ startIndex: 53, type: 'number.fs' }
+			]
+		}
+	]
+]);

monaco-languages/src/fsharp/fsharp.ts

@@ -0,0 +1,237 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['(*', '*)']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*//\\s*#region\\b|^\\s*\\(\\*\\s*#region(.*)\\*\\)'),
+			end: new RegExp('^\\s*//\\s*#endregion\\b|^\\s*\\(\\*\\s*#endregion\\s*\\*\\)')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.fs',
+
+	keywords: [
+		'abstract',
+		'and',
+		'atomic',
+		'as',
+		'assert',
+		'asr',
+		'base',
+		'begin',
+		'break',
+		'checked',
+		'component',
+		'const',
+		'constraint',
+		'constructor',
+		'continue',
+		'class',
+		'default',
+		'delegate',
+		'do',
+		'done',
+		'downcast',
+		'downto',
+		'elif',
+		'else',
+		'end',
+		'exception',
+		'eager',
+		'event',
+		'external',
+		'extern',
+		'false',
+		'finally',
+		'for',
+		'fun',
+		'function',
+		'fixed',
+		'functor',
+		'global',
+		'if',
+		'in',
+		'include',
+		'inherit',
+		'inline',
+		'interface',
+		'internal',
+		'land',
+		'lor',
+		'lsl',
+		'lsr',
+		'lxor',
+		'lazy',
+		'let',
+		'match',
+		'member',
+		'mod',
+		'module',
+		'mutable',
+		'namespace',
+		'method',
+		'mixin',
+		'new',
+		'not',
+		'null',
+		'of',
+		'open',
+		'or',
+		'object',
+		'override',
+		'private',
+		'parallel',
+		'process',
+		'protected',
+		'pure',
+		'public',
+		'rec',
+		'return',
+		'static',
+		'sealed',
+		'struct',
+		'sig',
+		'then',
+		'to',
+		'true',
+		'tailcall',
+		'trait',
+		'try',
+		'type',
+		'upcast',
+		'use',
+		'val',
+		'void',
+		'virtual',
+		'volatile',
+		'when',
+		'while',
+		'with',
+		'yield'
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\^%;\.,\/]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	integersuffix: /[uU]?[yslnLI]?/,
+	floatsuffix: /[fFmM]?/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[
+				/[a-zA-Z_]\w*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [< attributes >].
+			[/\[<.*>\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^#(if|else|endif)/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[/@symbols/, 'delimiter'],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, 'number.float'],
+			[/0x[0-9a-fA-F]+LF/, 'number.float'],
+			[/0x[0-9a-fA-F]+(@integersuffix)/, 'number.hex'],
+			[/0b[0-1]+(@integersuffix)/, 'number.bin'],
+			[/\d+(@integersuffix)/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"""/, 'string', '@string."""'],
+			[/"/, 'string', '@string."'],
+
+			// literal string
+			[/\@"/, { token: 'string.quote', next: '@litstring' }],
+
+			// characters
+			[/'[^\\']'B?/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\(\*(?!\))/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^*(]+/, 'comment'],
+			[/\*\)/, 'comment', '@pop'],
+			[/\*/, 'comment'],
+			[/\(\*\)/, 'comment'],
+			[/\(/, 'comment']
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[
+				/("""|"B?)/,
+				{
+					cases: {
+						'$#==$S2': { token: 'string', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			]
+		],
+
+		litstring: [
+			[/[^"]+/, 'string'],
+			[/""/, 'string.escape'],
+			[/"/, { token: 'string.quote', next: '@pop' }]
+		]
+	}
+};

monaco-languages/src/go/go.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'go',
+	extensions: ['.go'],
+	aliases: ['Go'],
+	loader: () => import('./go')
+});

monaco-languages/src/go/go.ts

@@ -0,0 +1,238 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '`', close: '`', notIn: ['string'] },
+		{ open: '"', close: '"', notIn: ['string'] },
+		{ open: "'", close: "'", notIn: ['string', 'comment'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '`', close: '`' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.go',
+
+	keywords: [
+		'break',
+		'case',
+		'chan',
+		'const',
+		'continue',
+		'default',
+		'defer',
+		'else',
+		'fallthrough',
+		'for',
+		'func',
+		'go',
+		'goto',
+		'if',
+		'import',
+		'interface',
+		'map',
+		'package',
+		'range',
+		'return',
+		'select',
+		'struct',
+		'switch',
+		'type',
+		'var',
+		'bool',
+		'true',
+		'false',
+		'uint8',
+		'uint16',
+		'uint32',
+		'uint64',
+		'int8',
+		'int16',
+		'int32',
+		'int64',
+		'float32',
+		'float64',
+		'complex64',
+		'complex128',
+		'byte',
+		'rune',
+		'uint',
+		'int',
+		'uintptr',
+		'string',
+		'nil'
+	],
+
+	operators: [
+		'+',
+		'-',
+		'*',
+		'/',
+		'%',
+		'&',
+		'|',
+		'^',
+		'<<',
+		'>>',
+		'&^',
+		'+=',
+		'-=',
+		'*=',
+		'/=',
+		'%=',
+		'&=',
+		'|=',
+		'^=',
+		'<<=',
+		'>>=',
+		'&^=',
+		'&&',
+		'||',
+		'<-',
+		'++',
+		'--',
+		'==',
+		'<',
+		'>',
+		'=',
+		'!',
+		'!=',
+		'<=',
+		'>=',
+		':=',
+		'...',
+		'(',
+		')',
+		'',
+		']',
+		'{',
+		'}',
+		',',
+		';',
+		'.',
+		':'
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// identifiers and keywords
+			[
+				/[a-zA-Z_]\w*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// [[ attributes ]].
+			[/\[\[.*\]\]/, 'annotation'],
+
+			// Preprocessor directive
+			[/^\s*#\w+/, 'keyword'],
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F']*[0-9a-fA-F]/, 'number.hex'],
+			[/0[0-7']*[0-7]/, 'number.octal'],
+			[/0[bB][0-1']*[0-1]/, 'number.binary'],
+			[/\d[\d']*/, 'number'],
+			[/\d/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, 'string', '@string'],
+			[/`/, 'string', '@rawstring'],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@doccomment'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		doccomment: [
+			[/[^\/*]+/, 'comment.doc'],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/, 'comment.doc.invalid'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		],
+
+		rawstring: [
+			[/[^\`]/, 'string'],
+			[/`/, 'string', '@pop']
+		]
+	}
+};

monaco-languages/src/graphql/graphql.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'graphql',
+	extensions: ['.graphql', '.gql'],
+	aliases: ['GraphQL', 'graphql', 'gql'],
+	mimetypes: ['application/graphql'],
+	loader: () => import('./graphql')
+});

monaco-languages/src/graphql/graphql.test.ts

@@ -0,0 +1,138 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('graphql', [
+	// Keywords
+	[
+		{
+			line: 'scalar Date',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.gql' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'type.identifier.gql' }
+			]
+		}
+	],
+
+	// Root schema definition
+	[
+		{
+			line: 'schema { query: Query, mutation: Mutation subscription: Subscription }',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.gql' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'delimiter.curly.gql' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'keyword.gql' }, // this should be identifier
+				{ startIndex: 14, type: 'operator.gql' },
+				{ startIndex: 15, type: '' },
+				{ startIndex: 16, type: 'type.identifier.gql' },
+				{ startIndex: 21, type: 'delimiter.gql' },
+				{ startIndex: 22, type: '' },
+				{ startIndex: 23, type: 'keyword.gql' }, // this should be identifier
+				{ startIndex: 31, type: 'operator.gql' },
+				{ startIndex: 32, type: '' },
+				{ startIndex: 33, type: 'type.identifier.gql' },
+				{ startIndex: 41, type: '' },
+				{ startIndex: 42, type: 'keyword.gql' }, // this should be identifier
+				{ startIndex: 54, type: 'operator.gql' },
+				{ startIndex: 55, type: '' },
+				{ startIndex: 56, type: 'type.identifier.gql' },
+				{ startIndex: 68, type: '' },
+				{ startIndex: 69, type: 'delimiter.curly.gql' }
+			]
+		}
+	],
+
+	[
+		{
+			line: `query testQuery($intValue:Int=3){value(arg:{string:"string" int:$intValue}){field1 field2}}`,
+			tokens: [
+				{ startIndex: 0, type: 'keyword.gql' }, // 'query'
+				{ startIndex: 5, type: '' }, // ' '
+				{ startIndex: 6, type: 'key.identifier.gql' }, // 'testQuery'
+				{ startIndex: 15, type: 'delimiter.parenthesis.gql' }, // '('
+				{ startIndex: 16, type: 'argument.identifier.gql' }, // '$intValue'
+				{ startIndex: 25, type: 'operator.gql' }, // ':'
+				{ startIndex: 26, type: 'keyword.gql' }, // 'Int'
+				{ startIndex: 29, type: 'operator.gql' }, // '='
+				{ startIndex: 30, type: 'number.gql' }, // '3'
+				{ startIndex: 31, type: 'delimiter.parenthesis.gql' }, // ')'
+				{ startIndex: 32, type: 'delimiter.curly.gql' }, // '{'
+				{ startIndex: 33, type: 'key.identifier.gql' }, // 'value'
+				{ startIndex: 38, type: 'delimiter.parenthesis.gql' }, // '('
+				{ startIndex: 39, type: 'key.identifier.gql' }, // 'arg'
+				{ startIndex: 42, type: 'operator.gql' }, // ':'
+				{ startIndex: 43, type: 'delimiter.curly.gql' }, // '{'
+				{ startIndex: 44, type: 'key.identifier.gql' }, // 'string'
+				{ startIndex: 50, type: 'operator.gql' }, // ':'
+				{ startIndex: 51, type: 'string.quote.gql' }, // '"'
+				{ startIndex: 52, type: 'string.gql' }, // 'string'
+				{ startIndex: 58, type: 'string.quote.gql' }, // '"'
+				{ startIndex: 59, type: '' }, // ' '
+				{ startIndex: 60, type: 'key.identifier.gql' }, // 'int'
+				{ startIndex: 63, type: 'operator.gql' }, // ':'
+				{ startIndex: 64, type: 'argument.identifier.gql' }, // '$intValue'
+				{ startIndex: 73, type: 'delimiter.curly.gql' }, // '}'
+				{ startIndex: 74, type: 'delimiter.parenthesis.gql' }, // ')'
+				{ startIndex: 75, type: 'delimiter.curly.gql' }, // '{'
+				{ startIndex: 76, type: 'key.identifier.gql' }, // 'field1'
+				{ startIndex: 82, type: '' }, // ' '
+				{ startIndex: 83, type: 'key.identifier.gql' }, // 'field2'
+				{ startIndex: 89, type: 'delimiter.curly.gql' } // '}}'
+			]
+		}
+	],
+
+	// More complex test:
+	//   """
+	//   Node interface
+	//   - allows (re)fetch arbitrary entity only by ID
+	//   """
+	//   interface Node {
+	//     id: ID!
+	//   }
+	[
+		{
+			line: '"""',
+			tokens: [{ startIndex: 0, type: 'string.gql' }]
+		},
+		{
+			line: 'This is MarkDown',
+			tokens: [{ startIndex: 0, type: '' }]
+		},
+		{
+			line: '"""',
+			tokens: [{ startIndex: 0, type: 'string.gql' }]
+		},
+		{
+			line: 'interface Node {',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.gql' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'type.identifier.gql' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'delimiter.curly.gql' }
+			]
+		},
+		{
+			line: '  id: ID!',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 2, type: 'key.identifier.gql' },
+				{ startIndex: 4, type: 'operator.gql' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'keyword.gql' },
+				{ startIndex: 8, type: 'operator.gql' }
+			]
+		},
+		{
+			line: '}',
+			tokens: [{ startIndex: 0, type: 'delimiter.curly.gql' }]
+		}
+	]
+]);

monaco-languages/src/graphql/graphql.ts

@@ -0,0 +1,174 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"""', close: '"""', notIn: ['string', 'comment'] },
+		{ open: '"', close: '"', notIn: ['string', 'comment'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"""', close: '"""' },
+		{ open: '"', close: '"' }
+	],
+	folding: {
+		offSide: true
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	defaultToken: 'invalid',
+	tokenPostfix: '.gql',
+
+	keywords: [
+		'null',
+		'true',
+		'false',
+		'query',
+		'mutation',
+		'subscription',
+		'extend',
+		'schema',
+		'directive',
+		'scalar',
+		'type',
+		'interface',
+		'union',
+		'enum',
+		'input',
+		'implements',
+		'fragment',
+		'on'
+	],
+
+	typeKeywords: ['Int', 'Float', 'String', 'Boolean', 'ID'],
+
+	directiveLocations: [
+		'SCHEMA',
+		'SCALAR',
+		'OBJECT',
+		'FIELD_DEFINITION',
+		'ARGUMENT_DEFINITION',
+		'INTERFACE',
+		'UNION',
+		'ENUM',
+		'ENUM_VALUE',
+		'INPUT_OBJECT',
+		'INPUT_FIELD_DEFINITION',
+		'QUERY',
+		'MUTATION',
+		'SUBSCRIPTION',
+		'FIELD',
+		'FRAGMENT_DEFINITION',
+		'FRAGMENT_SPREAD',
+		'INLINE_FRAGMENT',
+		'VARIABLE_DEFINITION'
+	],
+
+	operators: ['=', '!', '?', ':', '&', '|'],
+
+	// we include these common regular expressions
+	symbols: /[=!?:&|]+/,
+
+	// https://facebook.github.io/graphql/draft/#sec-String-Value
+	escapes: /\\(?:["\\\/bfnrt]|u[0-9A-Fa-f]{4})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// fields and argument names
+			[
+				/[a-z_][\w$]*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@default': 'key.identifier'
+					}
+				}
+			],
+
+			// identify typed input variables
+			[
+				/[$][\w$]*/,
+				{
+					cases: {
+						'@keywords': 'keyword',
+						'@default': 'argument.identifier'
+					}
+				}
+			],
+
+			// to show class names nicely
+			[
+				/[A-Z][\w\$]*/,
+				{
+					cases: {
+						'@typeKeywords': 'keyword',
+						'@default': 'type.identifier'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/@symbols/, { cases: { '@operators': 'operator', '@default': '' } }],
+
+			// @ annotations.
+			// As an example, we emit a debugging log message on these tokens.
+			// Note: message are supressed during the first load -- change some lines to see them.
+			[/@\s*[a-zA-Z_\$][\w\$]*/, { token: 'annotation', log: 'annotation token: $0' }],
+
+			// numbers
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/0[xX][0-9a-fA-F]+/, 'number.hex'],
+			[/\d+/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			[/"""/, { token: 'string', next: '@mlstring', nextEmbedded: 'markdown' }],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, { token: 'string.quote', bracket: '@open', next: '@string' }]
+		],
+
+		mlstring: [
+			[/[^"]+/, 'string'],
+			['"""', { token: 'string', next: '@pop', nextEmbedded: '@pop' }]
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, { token: 'string.quote', bracket: '@close', next: '@pop' }]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/#.*$/, 'comment']
+		]
+	}
+};

monaco-languages/src/handlebars/handlebars.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'handlebars',
+	extensions: ['.handlebars', '.hbs'],
+	aliases: ['Handlebars', 'handlebars', 'hbs'],
+	mimetypes: ['text/x-handlebars-template'],
+	loader: () => import('./handlebars')
+});

monaco-languages/src/handlebars/handlebars.test.ts

@@ -0,0 +1,354 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization(
+	['handlebars', 'css'],
+	[
+		// Just HTML
+		[
+			{
+				line: '<h1>handlebars!</h1>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 3, type: 'delimiter.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 15, type: 'delimiter.html' },
+					{ startIndex: 17, type: 'tag.html' },
+					{ startIndex: 19, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Expressions
+		[
+			{
+				line: '<h1>{{ title }}</h1>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 3, type: 'delimiter.html' },
+					{ startIndex: 4, type: 'delimiter.handlebars' },
+					{ startIndex: 6, type: '' },
+					{ startIndex: 7, type: 'variable.parameter.handlebars' },
+					{ startIndex: 12, type: '' },
+					{ startIndex: 13, type: 'delimiter.handlebars' },
+					{ startIndex: 15, type: 'delimiter.html' },
+					{ startIndex: 17, type: 'tag.html' },
+					{ startIndex: 19, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Expressions Sans Whitespace
+		[
+			{
+				line: '<h1>{{title}}</h1>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 3, type: 'delimiter.html' },
+					{ startIndex: 4, type: 'delimiter.handlebars' },
+					{ startIndex: 6, type: 'variable.parameter.handlebars' },
+					{ startIndex: 11, type: 'delimiter.handlebars' },
+					{ startIndex: 13, type: 'delimiter.html' },
+					{ startIndex: 15, type: 'tag.html' },
+					{ startIndex: 17, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Unescaped Expressions
+		[
+			{
+				line: '<h1>{{{ title }}}</h1>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 3, type: 'delimiter.html' },
+					{ startIndex: 4, type: 'delimiter.handlebars' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'variable.parameter.handlebars' },
+					{ startIndex: 13, type: '' },
+					{ startIndex: 14, type: 'delimiter.handlebars' },
+					{ startIndex: 17, type: 'delimiter.html' },
+					{ startIndex: 19, type: 'tag.html' },
+					{ startIndex: 21, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Blocks
+		[
+			{
+				line: '<ul>{{#each items}}<li>{{item}}</li>{{/each}}</ul>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 3, type: 'delimiter.html' },
+					{ startIndex: 4, type: 'delimiter.handlebars' },
+					{ startIndex: 6, type: 'keyword.helper.handlebars' },
+					{ startIndex: 11, type: '' },
+					{ startIndex: 12, type: 'variable.parameter.handlebars' },
+					{ startIndex: 17, type: 'delimiter.handlebars' },
+					{ startIndex: 19, type: 'delimiter.html' },
+					{ startIndex: 20, type: 'tag.html' },
+					{ startIndex: 22, type: 'delimiter.html' },
+					{ startIndex: 23, type: 'delimiter.handlebars' },
+					{ startIndex: 25, type: 'variable.parameter.handlebars' },
+					{ startIndex: 29, type: 'delimiter.handlebars' },
+					{ startIndex: 31, type: 'delimiter.html' },
+					{ startIndex: 33, type: 'tag.html' },
+					{ startIndex: 35, type: 'delimiter.html' },
+					{ startIndex: 36, type: 'delimiter.handlebars' },
+					{ startIndex: 38, type: 'keyword.helper.handlebars' },
+					{ startIndex: 43, type: 'delimiter.handlebars' },
+					{ startIndex: 45, type: 'delimiter.html' },
+					{ startIndex: 47, type: 'tag.html' },
+					{ startIndex: 49, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Multiline
+		[
+			{
+				line: '<div>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: 'delimiter.html' }
+				]
+			},
+			{
+				line: '{{#if foo}}',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.handlebars' },
+					{ startIndex: 2, type: 'keyword.helper.handlebars' },
+					{ startIndex: 5, type: '' },
+					{ startIndex: 6, type: 'variable.parameter.handlebars' },
+					{ startIndex: 9, type: 'delimiter.handlebars' }
+				]
+			},
+			{
+				line: '<span>{{bar}}</span>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 5, type: 'delimiter.html' },
+					{ startIndex: 6, type: 'delimiter.handlebars' },
+					{ startIndex: 8, type: 'variable.parameter.handlebars' },
+					{ startIndex: 11, type: 'delimiter.handlebars' },
+					{ startIndex: 13, type: 'delimiter.html' },
+					{ startIndex: 15, type: 'tag.html' },
+					{ startIndex: 19, type: 'delimiter.html' }
+				]
+			},
+			{
+				line: '{{/if}}',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.handlebars' },
+					{ startIndex: 2, type: 'keyword.helper.handlebars' },
+					{ startIndex: 5, type: 'delimiter.handlebars' }
+				]
+			}
+		],
+
+		// Div end
+		[
+			{
+				line: '</div>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 2, type: 'tag.html' },
+					{ startIndex: 5, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// HTML Expressions
+		[
+			{
+				line: '<script type="text/x-handlebars-template"><h1>{{ title }}</h1></script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name' },
+					{ startIndex: 12, type: 'delimiter' },
+					{ startIndex: 13, type: 'attribute.value' },
+					{ startIndex: 41, type: 'delimiter.html' },
+					{ startIndex: 42, type: 'delimiter.html' },
+					{ startIndex: 43, type: 'tag.html' },
+					{ startIndex: 45, type: 'delimiter.html' },
+					{ startIndex: 46, type: 'delimiter.handlebars' },
+					{ startIndex: 48, type: '' },
+					{ startIndex: 49, type: 'variable.parameter.handlebars' },
+					{ startIndex: 54, type: '' },
+					{ startIndex: 55, type: 'delimiter.handlebars' },
+					{ startIndex: 57, type: 'delimiter.html' },
+					{ startIndex: 59, type: 'tag.html' },
+					{ startIndex: 61, type: 'delimiter.html' },
+					{ startIndex: 62, type: 'delimiter.html' },
+					{ startIndex: 64, type: 'tag.html' },
+					{ startIndex: 70, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Multi-line HTML Expressions
+		[
+			{
+				line: '<script type="text/x-handlebars-template">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name' },
+					{ startIndex: 12, type: 'delimiter' },
+					{ startIndex: 13, type: 'attribute.value' },
+					{ startIndex: 41, type: 'delimiter.html' }
+				]
+			},
+			{
+				line: '<h1>{{ title }}</h1>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 3, type: 'delimiter.html' },
+					{ startIndex: 4, type: 'delimiter.handlebars' },
+					{ startIndex: 6, type: '' },
+					{ startIndex: 7, type: 'variable.parameter.handlebars' },
+					{ startIndex: 12, type: '' },
+					{ startIndex: 13, type: 'delimiter.handlebars' },
+					{ startIndex: 15, type: 'delimiter.html' },
+					{ startIndex: 17, type: 'tag.html' },
+					{ startIndex: 19, type: 'delimiter.html' }
+				]
+			},
+			{
+				line: '</script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 2, type: 'tag.html' },
+					{ startIndex: 8, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// HTML Nested Modes
+		[
+			{
+				line: '{{foo}}<script></script>{{bar}}',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.handlebars' },
+					{ startIndex: 2, type: 'variable.parameter.handlebars' },
+					{ startIndex: 5, type: 'delimiter.handlebars' },
+					{ startIndex: 7, type: 'delimiter.html' },
+					{ startIndex: 8, type: 'tag.html' },
+					{ startIndex: 14, type: 'delimiter.html' },
+					// { startIndex:15, type: 'delimiter.html' },
+					{ startIndex: 17, type: 'tag.html' },
+					{ startIndex: 23, type: 'delimiter.html' },
+					{ startIndex: 24, type: 'delimiter.handlebars' },
+					{ startIndex: 26, type: 'variable.parameter.handlebars' },
+					{ startIndex: 29, type: 'delimiter.handlebars' }
+				]
+			}
+		],
+
+		// else keyword
+		[
+			{
+				line: '{{else}}',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.handlebars' },
+					{ startIndex: 2, type: 'keyword.helper.handlebars' },
+					{ startIndex: 6, type: 'delimiter.handlebars' }
+				]
+			}
+		],
+
+		// else keyword #2
+		[
+			{
+				line: '{{elseFoo}}',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.handlebars' },
+					{ startIndex: 2, type: 'variable.parameter.handlebars' },
+					{ startIndex: 9, type: 'delimiter.handlebars' }
+				]
+			}
+		],
+
+		// Token inside attribute
+		[
+			{
+				line: '<a href="/posts/{{permalink}}">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 2, type: '' },
+					{ startIndex: 3, type: 'attribute.name' },
+					{ startIndex: 7, type: 'delimiter' },
+					{ startIndex: 8, type: 'attribute.value' },
+					{ startIndex: 30, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		[
+			{
+				line: '{{test "coloring/looks broken"}}">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.handlebars' },
+					{ startIndex: 2, type: 'variable.parameter.handlebars' },
+					{ startIndex: 6, type: '' },
+					{ startIndex: 7, type: 'string.handlebars' },
+					{ startIndex: 30, type: 'delimiter.handlebars' },
+					{ startIndex: 32, type: '' }
+				]
+			}
+		],
+
+		// Block comment
+		[
+			{
+				line: '{{!-- block comment --}}',
+				tokens: [
+					{ startIndex: 0, type: 'comment.block.start.handlebars' },
+					{ startIndex: 5, type: 'comment.content.handlebars' },
+					{ startIndex: 20, type: 'comment.block.end.handlebars' }
+				]
+			}
+		],
+
+		// Block comment with mustache
+		[
+			{
+				line: '{{!-- block comment }} with mustache --}}',
+				tokens: [
+					{ startIndex: 0, type: 'comment.block.start.handlebars' },
+					{ startIndex: 5, type: 'comment.content.handlebars' },
+					{ startIndex: 37, type: 'comment.block.end.handlebars' }
+				]
+			}
+		],
+
+		// Handlebars comment
+		[
+			{
+				line: '{{! comment }}',
+				tokens: [
+					{ startIndex: 0, type: 'comment.start.handlebars' },
+					{ startIndex: 3, type: 'comment.content.handlebars' },
+					{ startIndex: 12, type: 'comment.end.handlebars' }
+				]
+			}
+		]
+	]
+);

monaco-languages/src/handlebars/handlebars.ts

@@ -0,0 +1,432 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { languages } from '../fillers/monaco-editor-core';
+
+const EMPTY_ELEMENTS: string[] = [
+	'area',
+	'base',
+	'br',
+	'col',
+	'embed',
+	'hr',
+	'img',
+	'input',
+	'keygen',
+	'link',
+	'menuitem',
+	'meta',
+	'param',
+	'source',
+	'track',
+	'wbr'
+];
+
+export const conf: languages.LanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g,
+
+	comments: {
+		blockComment: ['{{!--', '--}}']
+	},
+
+	brackets: [
+		['<!--', '-->'],
+		['<', '>'],
+		['{{', '}}'],
+		['{', '}'],
+		['(', ')']
+	],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+
+	surroundingPairs: [
+		{ open: '<', close: '>' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+
+	onEnterRules: [
+		{
+			beforeText: new RegExp(
+				`<(?!(?:${EMPTY_ELEMENTS.join('|')}))(\\w[\\w\\d]*)([^/>]*(?!/)>)[^<]*$`,
+				'i'
+			),
+			afterText: /^<\/(\w[\w\d]*)\s*>$/i,
+			action: {
+				indentAction: languages.IndentAction.IndentOutdent
+			}
+		},
+		{
+			beforeText: new RegExp(
+				`<(?!(?:${EMPTY_ELEMENTS.join('|')}))(\\w[\\w\\d]*)([^/>]*(?!/)>)[^<]*$`,
+				'i'
+			),
+			action: { indentAction: languages.IndentAction.Indent }
+		}
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '',
+	// ignoreCase: true,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			[/\{\{!--/, 'comment.block.start.handlebars', '@commentBlock'],
+			[/\{\{!/, 'comment.start.handlebars', '@comment'],
+			[/\{\{/, { token: '@rematch', switchTo: '@handlebarsInSimpleState.root' }],
+			[/<!DOCTYPE/, 'metatag.html', '@doctype'],
+			[/<!--/, 'comment.html', '@commentHtml'],
+			[/(<)(\w+)(\/>)/, ['delimiter.html', 'tag.html', 'delimiter.html']],
+			[/(<)(script)/, ['delimiter.html', { token: 'tag.html', next: '@script' }]],
+			[/(<)(style)/, ['delimiter.html', { token: 'tag.html', next: '@style' }]],
+			[/(<)([:\w]+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/(<\/)(\w+)/, ['delimiter.html', { token: 'tag.html', next: '@otherTag' }]],
+			[/</, 'delimiter.html'],
+			[/\{/, 'delimiter.html'],
+			[/[^<{]+/] // text
+		],
+
+		doctype: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.comment'
+				}
+			],
+			[/[^>]+/, 'metatag.content.html'],
+			[/>/, 'metatag.html', '@pop']
+		],
+
+		comment: [
+			[/\}\}/, 'comment.end.handlebars', '@pop'],
+			[/./, 'comment.content.handlebars']
+		],
+
+		commentBlock: [
+			[/--\}\}/, 'comment.block.end.handlebars', '@pop'],
+			[/./, 'comment.content.handlebars']
+		],
+
+		commentHtml: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.comment'
+				}
+			],
+			[/-->/, 'comment.html', '@pop'],
+			[/[^-]+/, 'comment.content.html'],
+			[/./, 'comment.content.html']
+		],
+
+		otherTag: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.otherTag'
+				}
+			],
+			[/\/?>/, 'delimiter.html', '@pop'],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[/[ \t\r\n]+/] // whitespace
+		],
+
+		// -- BEGIN <script> tags handling
+
+		// After <script
+		script: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.script'
+				}
+			],
+			[/type/, 'attribute.name', '@scriptAfterType'],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@scriptEmbedded.text/javascript',
+					nextEmbedded: 'text/javascript'
+				}
+			],
+			[/[ \t\r\n]+/], // whitespace
+			[
+				/(<\/)(script\s*)(>)/,
+				['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]
+			]
+		],
+
+		// After <script ... type
+		scriptAfterType: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.scriptAfterType'
+				}
+			],
+			[/=/, 'delimiter', '@scriptAfterTypeEquals'],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@scriptEmbedded.text/javascript',
+					nextEmbedded: 'text/javascript'
+				}
+			], // cover invalid e.g. <script type>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <script ... type =
+		scriptAfterTypeEquals: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.scriptAfterTypeEquals'
+				}
+			],
+			[
+				/"([^"]*)"/,
+				{
+					token: 'attribute.value',
+					switchTo: '@scriptWithCustomType.$1'
+				}
+			],
+			[
+				/'([^']*)'/,
+				{
+					token: 'attribute.value',
+					switchTo: '@scriptWithCustomType.$1'
+				}
+			],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@scriptEmbedded.text/javascript',
+					nextEmbedded: 'text/javascript'
+				}
+			], // cover invalid e.g. <script type=>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <script ... type = $S2
+		scriptWithCustomType: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.scriptWithCustomType.$S2'
+				}
+			],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@scriptEmbedded.$S2',
+					nextEmbedded: '$S2'
+				}
+			],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		scriptEmbedded: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInEmbeddedState.scriptEmbedded.$S2',
+					nextEmbedded: '@pop'
+				}
+			],
+			[/<\/script/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }]
+		],
+
+		// -- END <script> tags handling
+
+		// -- BEGIN <style> tags handling
+
+		// After <style
+		style: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.style'
+				}
+			],
+			[/type/, 'attribute.name', '@styleAfterType'],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@styleEmbedded.text/css',
+					nextEmbedded: 'text/css'
+				}
+			],
+			[/[ \t\r\n]+/], // whitespace
+			[
+				/(<\/)(style\s*)(>)/,
+				['delimiter.html', 'tag.html', { token: 'delimiter.html', next: '@pop' }]
+			]
+		],
+
+		// After <style ... type
+		styleAfterType: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.styleAfterType'
+				}
+			],
+			[/=/, 'delimiter', '@styleAfterTypeEquals'],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@styleEmbedded.text/css',
+					nextEmbedded: 'text/css'
+				}
+			], // cover invalid e.g. <style type>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <style ... type =
+		styleAfterTypeEquals: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.styleAfterTypeEquals'
+				}
+			],
+			[
+				/"([^"]*)"/,
+				{
+					token: 'attribute.value',
+					switchTo: '@styleWithCustomType.$1'
+				}
+			],
+			[
+				/'([^']*)'/,
+				{
+					token: 'attribute.value',
+					switchTo: '@styleWithCustomType.$1'
+				}
+			],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@styleEmbedded.text/css',
+					nextEmbedded: 'text/css'
+				}
+			], // cover invalid e.g. <style type=>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <style ... type = $S2
+		styleWithCustomType: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInSimpleState.styleWithCustomType.$S2'
+				}
+			],
+			[
+				/>/,
+				{
+					token: 'delimiter.html',
+					next: '@styleEmbedded.$S2',
+					nextEmbedded: '$S2'
+				}
+			],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		styleEmbedded: [
+			[
+				/\{\{/,
+				{
+					token: '@rematch',
+					switchTo: '@handlebarsInEmbeddedState.styleEmbedded.$S2',
+					nextEmbedded: '@pop'
+				}
+			],
+			[/<\/style/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }]
+		],
+
+		// -- END <style> tags handling
+
+		handlebarsInSimpleState: [
+			[/\{\{\{?/, 'delimiter.handlebars'],
+			[/\}\}\}?/, { token: 'delimiter.handlebars', switchTo: '@$S2.$S3' }],
+			{ include: 'handlebarsRoot' }
+		],
+
+		handlebarsInEmbeddedState: [
+			[/\{\{\{?/, 'delimiter.handlebars'],
+			[
+				/\}\}\}?/,
+				{
+					token: 'delimiter.handlebars',
+					switchTo: '@$S2.$S3',
+					nextEmbedded: '$S3'
+				}
+			],
+			{ include: 'handlebarsRoot' }
+		],
+
+		handlebarsRoot: [
+			[/"[^"]*"/, 'string.handlebars'],
+			[/[#/][^\s}]+/, 'keyword.helper.handlebars'],
+			[/else\b/, 'keyword.helper.handlebars'],
+			[/[\s]+/],
+			[/[^}]/, 'variable.parameter.handlebars']
+		]
+	}
+};

monaco-languages/src/hcl/hcl.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'hcl',
+	extensions: ['.tf', '.tfvars', '.hcl'],
+	aliases: ['Terraform', 'tf', 'HCL', 'hcl'],
+	loader: () => import('./hcl')
+});

monaco-languages/src/hcl/hcl.ts

@@ -0,0 +1,192 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '#',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"', notIn: ['string'] }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.hcl',
+
+	keywords: [
+		'var',
+		'local',
+		'path',
+		'for_each',
+		'any',
+		'string',
+		'number',
+		'bool',
+		'true',
+		'false',
+		'null',
+		'if ',
+		'else ',
+		'endif ',
+		'for ',
+		'in',
+		'endfor'
+	],
+
+	operators: [
+		'=',
+		'>=',
+		'<=',
+		'==',
+		'!=',
+		'+',
+		'-',
+		'*',
+		'/',
+		'%',
+		'&&',
+		'||',
+		'!',
+		'<',
+		'>',
+		'?',
+		'...',
+		':'
+	],
+
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	terraformFunctions:
+		/(abs|ceil|floor|log|max|min|pow|signum|chomp|format|formatlist|indent|join|lower|regex|regexall|replace|split|strrev|substr|title|trimspace|upper|chunklist|coalesce|coalescelist|compact|concat|contains|distinct|element|flatten|index|keys|length|list|lookup|map|matchkeys|merge|range|reverse|setintersection|setproduct|setunion|slice|sort|transpose|values|zipmap|base64decode|base64encode|base64gzip|csvdecode|jsondecode|jsonencode|urlencode|yamldecode|yamlencode|abspath|dirname|pathexpand|basename|file|fileexists|fileset|filebase64|templatefile|formatdate|timeadd|timestamp|base64sha256|base64sha512|bcrypt|filebase64sha256|filebase64sha512|filemd5|filemd1|filesha256|filesha512|md5|rsadecrypt|sha1|sha256|sha512|uuid|uuidv5|cidrhost|cidrnetmask|cidrsubnet|tobool|tolist|tomap|tonumber|toset|tostring)/,
+	terraformMainBlocks: /(module|data|terraform|resource|provider|variable|output|locals)/,
+	tokenizer: {
+		root: [
+			// highlight main blocks
+			[
+				/^@terraformMainBlocks([ \t]*)([\w-]+|"[\w-]+"|)([ \t]*)([\w-]+|"[\w-]+"|)([ \t]*)(\{)/,
+				['type', '', 'string', '', 'string', '', '@brackets']
+			],
+			// highlight all the remaining blocks
+			[
+				/(\w+[ \t]+)([ \t]*)([\w-]+|"[\w-]+"|)([ \t]*)([\w-]+|"[\w-]+"|)([ \t]*)(\{)/,
+				['identifier', '', 'string', '', 'string', '', '@brackets']
+			],
+			// highlight block
+			[
+				/(\w+[ \t]+)([ \t]*)([\w-]+|"[\w-]+"|)([ \t]*)([\w-]+|"[\w-]+"|)(=)(\{)/,
+				['identifier', '', 'string', '', 'operator', '', '@brackets']
+			],
+			// terraform general highlight - shared with expressions
+			{ include: '@terraform' }
+		],
+		terraform: [
+			// highlight terraform functions
+			[/@terraformFunctions(\()/, ['type', '@brackets']],
+			// all other words are variables or keywords
+			[
+				/[a-zA-Z_]\w*-*/, // must work with variables such as foo-bar and also with negative numbers
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'variable'
+					}
+				}
+			],
+			{ include: '@whitespace' },
+			{ include: '@heredoc' },
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'operator',
+						'@default': ''
+					}
+				}
+			],
+			// numbers
+			[/\d*\d+[eE]([\-+]?\d+)?/, 'number.float'],
+			[/\d*\.\d+([eE][\-+]?\d+)?/, 'number.float'],
+			[/\d[\d']*/, 'number'],
+			[/\d/, 'number'],
+			[/[;,.]/, 'delimiter'], // delimiter: after number because of .\d floats
+			// strings
+			[/"/, 'string', '@string'], // this will include expressions
+			[/'/, 'invalid']
+		],
+		heredoc: [
+			[
+				/<<[-]*\s*["]?([\w\-]+)["]?/,
+				{ token: 'string.heredoc.delimiter', next: '@heredocBody.$1' }
+			]
+		],
+		heredocBody: [
+			[
+				/([\w\-]+)$/,
+				{
+					cases: {
+						'$1==$S2': [
+							{
+								token: 'string.heredoc.delimiter',
+								next: '@popall'
+							}
+						],
+						'@default': 'string.heredoc'
+					}
+				}
+			],
+			[/./, 'string.heredoc']
+		],
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment'],
+			[/#.*$/, 'comment']
+		],
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+		string: [
+			[/\$\{/, { token: 'delimiter', next: '@stringExpression' }],
+			[/[^\\"\$]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@popall']
+		],
+		stringInsideExpression: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		],
+		stringExpression: [
+			[/\}/, { token: 'delimiter', next: '@pop' }],
+			[/"/, 'string', '@stringInsideExpression'],
+			{ include: '@terraform' }
+		]
+	}
+};

monaco-languages/src/html/html.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'html',
+	extensions: ['.html', '.htm', '.shtml', '.xhtml', '.mdoc', '.jsp', '.asp', '.aspx', '.jshtm'],
+	aliases: ['HTML', 'htm', 'html', 'xhtml'],
+	mimetypes: ['text/html', 'text/x-jshtm', 'text/template', 'text/ng-template'],
+	loader: () => import('./html')
+});

monaco-languages/src/html/html.test.ts

@@ -0,0 +1,717 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization(
+	['html', 'css', 'javascript'],
+	[
+		// Open Start Tag #1'
+		[
+			{
+				line: '<abc',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' }
+				]
+			}
+		],
+
+		// Open Start Tag #2
+		[
+			{
+				line: '<input',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' }
+				]
+			}
+		],
+
+		// Open Start Tag with Invalid Tag
+		[
+			{
+				line: '< abc',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: '' }
+				]
+			}
+		],
+
+		// Open Start Tag #3
+		[
+			{
+				line: '< abc>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: '' }
+				]
+			}
+		],
+
+		// Open Start Tag #4
+		[
+			{
+				line: 'i <len;',
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 2, type: 'delimiter.html' },
+					{ startIndex: 3, type: 'tag.html' },
+					{ startIndex: 6, type: '' }
+				]
+			}
+		],
+
+		// Open Start Tag #5
+		[
+			{
+				line: '<',
+				tokens: [{ startIndex: 0, type: 'delimiter.html' }]
+			}
+		],
+
+		// Open End Tag
+		[
+			{
+				line: '</a',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 2, type: 'tag.html' }
+				]
+			}
+		],
+
+		// Complete Start Tag
+		[
+			{
+				line: '<abc>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Complete Start Tag with Whitespace
+		[
+			{
+				line: '<abc >',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// bug 9809 - Complete Start Tag with Namespaceprefix
+		[
+			{
+				line: '<foo:bar>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 8, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Complete End Tag
+		[
+			{
+				line: '</abc>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 2, type: 'tag.html' },
+					{ startIndex: 5, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Complete End Tag with Whitespace
+		[
+			{
+				line: '</abc  >',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 2, type: 'tag.html' },
+					{ startIndex: 5, type: '' },
+					{ startIndex: 7, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Empty Tag
+		[
+			{
+				line: '<abc />',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #1
+		[
+			{
+				line: '<script type="text/javascript">var i= 10;</script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' },
+					{ startIndex: 13, type: 'attribute.value.html' },
+					{ startIndex: 30, type: 'delimiter.html' },
+					{ startIndex: 31, type: 'keyword.js' },
+					{ startIndex: 34, type: '' },
+					{ startIndex: 35, type: 'identifier.js' },
+					{ startIndex: 36, type: 'delimiter.js' },
+					{ startIndex: 37, type: '' },
+					{ startIndex: 38, type: 'number.js' },
+					{ startIndex: 40, type: 'delimiter.js' },
+					{ startIndex: 41, type: 'delimiter.html' },
+					{ startIndex: 43, type: 'tag.html' },
+					{ startIndex: 49, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #2
+		[
+			{
+				line: '<script type="text/javascript">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' },
+					{ startIndex: 13, type: 'attribute.value.html' },
+					{ startIndex: 30, type: 'delimiter.html' }
+				]
+			},
+			{
+				line: 'var i= 10;',
+				tokens: [
+					{ startIndex: 0, type: 'keyword.js' },
+					{ startIndex: 3, type: '' },
+					{ startIndex: 4, type: 'identifier.js' },
+					{ startIndex: 5, type: 'delimiter.js' },
+					{ startIndex: 6, type: '' },
+					{ startIndex: 7, type: 'number.js' },
+					{ startIndex: 9, type: 'delimiter.js' }
+				]
+			},
+			{
+				line: '</script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 2, type: 'tag.html' },
+					{ startIndex: 8, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #3
+		[
+			{
+				line: '<script type="text/javascript">var i= 10;',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' },
+					{ startIndex: 13, type: 'attribute.value.html' },
+					{ startIndex: 30, type: 'delimiter.html' },
+					{ startIndex: 31, type: 'keyword.js' },
+					{ startIndex: 34, type: '' },
+					{ startIndex: 35, type: 'identifier.js' },
+					{ startIndex: 36, type: 'delimiter.js' },
+					{ startIndex: 37, type: '' },
+					{ startIndex: 38, type: 'number.js' },
+					{ startIndex: 40, type: 'delimiter.js' }
+				]
+			},
+			{
+				line: '</script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 2, type: 'tag.html' },
+					{ startIndex: 8, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #4
+		[
+			{
+				line: '<script type="text/javascript">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' },
+					{ startIndex: 13, type: 'attribute.value.html' },
+					{ startIndex: 30, type: 'delimiter.html' }
+				]
+			},
+			{
+				line: 'var i= 10;</script>',
+				tokens: [
+					{ startIndex: 0, type: 'keyword.js' },
+					{ startIndex: 3, type: '' },
+					{ startIndex: 4, type: 'identifier.js' },
+					{ startIndex: 5, type: 'delimiter.js' },
+					{ startIndex: 6, type: '' },
+					{ startIndex: 7, type: 'number.js' },
+					{ startIndex: 9, type: 'delimiter.js' },
+					{ startIndex: 10, type: 'delimiter.html' },
+					{ startIndex: 12, type: 'tag.html' },
+					{ startIndex: 18, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #5
+		[
+			{
+				line: '<script type="text/plain">a',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' },
+					{ startIndex: 13, type: 'attribute.value.html' },
+					{ startIndex: 25, type: 'delimiter.html' },
+					{ startIndex: 26, type: '' }
+				]
+			},
+			{
+				line: '<a</script>',
+				tokens: [
+					{ startIndex: 0, type: '' },
+					{ startIndex: 2, type: 'delimiter.html' },
+					{ startIndex: 4, type: 'tag.html' },
+					{ startIndex: 10, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #6
+		[
+			{
+				line: '<script>a</script><script>b</script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: 'delimiter.html' },
+					{ startIndex: 8, type: 'identifier.js' },
+					{ startIndex: 9, type: 'delimiter.html' },
+					{ startIndex: 11, type: 'tag.html' },
+					{ startIndex: 17, type: 'delimiter.html' },
+					// { startIndex:18, type: 'delimiter.html' },
+					{ startIndex: 19, type: 'tag.html' },
+					{ startIndex: 25, type: 'delimiter.html' },
+					{ startIndex: 26, type: 'identifier.js' },
+					{ startIndex: 27, type: 'delimiter.html' },
+					{ startIndex: 29, type: 'tag.html' },
+					{ startIndex: 35, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #7
+		[
+			{
+				line: '<script type="text/javascript"></script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' },
+					{ startIndex: 13, type: 'attribute.value.html' },
+					{ startIndex: 30, type: 'delimiter.html' },
+					// { startIndex:31, type: 'delimiter.html' },
+					{ startIndex: 33, type: 'tag.html' },
+					{ startIndex: 39, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #8
+		[
+			{
+				line: '<script>var i= 10;</script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: 'delimiter.html' },
+					{ startIndex: 8, type: 'keyword.js' },
+					{ startIndex: 11, type: '' },
+					{ startIndex: 12, type: 'identifier.js' },
+					{ startIndex: 13, type: 'delimiter.js' },
+					{ startIndex: 14, type: '' },
+					{ startIndex: 15, type: 'number.js' },
+					{ startIndex: 17, type: 'delimiter.js' },
+					{ startIndex: 18, type: 'delimiter.html' },
+					{ startIndex: 20, type: 'tag.html' },
+					{ startIndex: 26, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Embedded Content #9
+		[
+			{
+				line: '<script type="text/javascript" src="main.js"></script>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 7, type: '' },
+					{ startIndex: 8, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' },
+					{ startIndex: 13, type: 'attribute.value.html' },
+					{ startIndex: 30, type: '' },
+					{ startIndex: 31, type: 'attribute.name.html' },
+					{ startIndex: 34, type: 'delimiter.html' },
+					{ startIndex: 35, type: 'attribute.value.html' },
+					{ startIndex: 44, type: 'delimiter.html' },
+					// { startIndex:45, type: 'delimiter.html' },
+					{ startIndex: 47, type: 'tag.html' },
+					{ startIndex: 53, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Attribute
+		[
+			{
+				line: '<abc foo="bar">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' },
+					{ startIndex: 9, type: 'attribute.value.html' },
+					{ startIndex: 14, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Empty Attribute Value
+		[
+			{
+				line: "<abc foo='bar'>",
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' },
+					{ startIndex: 9, type: 'attribute.value.html' },
+					{ startIndex: 14, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with empty attributes
+		[
+			{
+				line: '<abc foo="">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' },
+					{ startIndex: 9, type: 'attribute.value.html' },
+					{ startIndex: 11, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Attributes
+		[
+			{
+				line: '<abc foo="bar" bar=\'foo\'>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' },
+					{ startIndex: 9, type: 'attribute.value.html' },
+					{ startIndex: 14, type: '' },
+					{ startIndex: 15, type: 'attribute.name.html' },
+					{ startIndex: 18, type: 'delimiter.html' },
+					{ startIndex: 19, type: 'attribute.value.html' },
+					{ startIndex: 24, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Attributes, no quotes
+		[
+			{
+				line: '<abc foo=bar bar=help-me>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' },
+					{ startIndex: 9, type: 'attribute.name.html' }, // slightly incorrect
+					{ startIndex: 12, type: '' },
+					{ startIndex: 13, type: 'attribute.name.html' },
+					{ startIndex: 16, type: 'delimiter.html' },
+					{ startIndex: 17, type: 'attribute.name.html' }, // slightly incorrect
+					{ startIndex: 24, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Attribute And Whitespace
+		[
+			{
+				line: '<abc foo=  "bar">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' },
+					{ startIndex: 9, type: '' },
+					{ startIndex: 11, type: 'attribute.value.html' },
+					{ startIndex: 16, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Attribute And Whitespace #2
+		[
+			{
+				line: '<abc foo = "bar">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: '' },
+					{ startIndex: 9, type: 'delimiter.html' },
+					{ startIndex: 10, type: '' },
+					{ startIndex: 11, type: 'attribute.value.html' },
+					{ startIndex: 16, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Name-Only-Attribute #1
+		[
+			{
+				line: '<abc foo>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Name-Only-Attribute #2
+		[
+			{
+				line: '<abc foo bar>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: '' },
+					{ startIndex: 9, type: 'attribute.name.html' },
+					{ startIndex: 12, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Interesting Attribute Name
+		[
+			{
+				line: '<abc foo!@#="bar">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: '' },
+					{ startIndex: 11, type: 'delimiter.html' },
+					{ startIndex: 12, type: 'attribute.value.html' },
+					{ startIndex: 17, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Angular Attribute Name
+		[
+			{
+				line: '<abc #myinput (click)="bar" [value]="someProperty" *ngIf="someCondition">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 6, type: 'attribute.name.html' },
+					{ startIndex: 13, type: '' },
+					{ startIndex: 15, type: 'attribute.name.html' },
+					{ startIndex: 20, type: '' },
+					{ startIndex: 21, type: 'delimiter.html' },
+					{ startIndex: 22, type: 'attribute.value.html' },
+					{ startIndex: 27, type: '' },
+					{ startIndex: 29, type: 'attribute.name.html' },
+					{ startIndex: 34, type: '' },
+					{ startIndex: 35, type: 'delimiter.html' },
+					{ startIndex: 36, type: 'attribute.value.html' },
+					{ startIndex: 50, type: '' },
+					{ startIndex: 52, type: 'attribute.name.html' },
+					{ startIndex: 56, type: 'delimiter.html' },
+					{ startIndex: 57, type: 'attribute.value.html' },
+					{ startIndex: 72, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Tag with Invalid Attribute Value
+		[
+			{
+				line: '<abc foo=">',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 4, type: '' },
+					{ startIndex: 5, type: 'attribute.name.html' },
+					{ startIndex: 8, type: 'delimiter.html' },
+					{ startIndex: 9, type: '' },
+					{ startIndex: 10, type: 'delimiter.html' }
+				]
+			}
+		],
+
+		// Simple Comment 1
+		[
+			{
+				line: '<!--a-->',
+				tokens: [
+					{ startIndex: 0, type: 'comment.html' },
+					{ startIndex: 4, type: 'comment.content.html' },
+					{ startIndex: 5, type: 'comment.html' }
+				]
+			}
+		],
+
+		// Simple Comment 2
+		[
+			{
+				line: '<!--a>foo bar</a -->',
+				tokens: [
+					{ startIndex: 0, type: 'comment.html' },
+					{ startIndex: 4, type: 'comment.content.html' },
+					{ startIndex: 17, type: 'comment.html' }
+				]
+			}
+		],
+
+		// Multiline Comment
+		[
+			{
+				line: '<!--a>',
+				tokens: [
+					{ startIndex: 0, type: 'comment.html' },
+					{ startIndex: 4, type: 'comment.content.html' }
+				]
+			},
+			{
+				line: 'foo ',
+				tokens: [{ startIndex: 0, type: 'comment.content.html' }]
+			},
+			{
+				line: 'bar</a -->',
+				tokens: [
+					{ startIndex: 0, type: 'comment.content.html' },
+					{ startIndex: 7, type: 'comment.html' }
+				]
+			}
+		],
+
+		// Simple Doctype
+		[
+			{
+				line: '<!DOCTYPE a>',
+				tokens: [
+					{ startIndex: 0, type: 'metatag.html' },
+					{ startIndex: 9, type: 'metatag.content.html' },
+					{ startIndex: 11, type: 'metatag.html' }
+				]
+			}
+		],
+
+		// Simple Doctype #2
+		[
+			{
+				line: '<!doctype a>',
+				tokens: [
+					{ startIndex: 0, type: 'metatag.html' },
+					{ startIndex: 9, type: 'metatag.content.html' },
+					{ startIndex: 11, type: 'metatag.html' }
+				]
+			}
+		],
+
+		// Simple Doctype #4
+		[
+			{
+				line: '<!DOCTYPE a',
+				tokens: [
+					{ startIndex: 0, type: 'metatag.html' },
+					{ startIndex: 9, type: 'metatag.content.html' }
+				]
+			},
+			{
+				line: '"foo" \'bar\'>',
+				tokens: [
+					{ startIndex: 0, type: 'metatag.content.html' },
+					{ startIndex: 11, type: 'metatag.html' }
+				]
+			}
+		],
+
+		// PR #14
+		[
+			{
+				line: '<asdf:bar>asd</asdf:bar>',
+				tokens: [
+					{ startIndex: 0, type: 'delimiter.html' },
+					{ startIndex: 1, type: 'tag.html' },
+					{ startIndex: 9, type: 'delimiter.html' },
+					{ startIndex: 10, type: '' },
+					{ startIndex: 13, type: 'delimiter.html' },
+					{ startIndex: 15, type: 'tag.html' },
+					{ startIndex: 23, type: 'delimiter.html' }
+				]
+			}
+		]
+	]
+);

monaco-languages/src/html/html.ts

@@ -0,0 +1,341 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { languages } from '../fillers/monaco-editor-core';
+
+const EMPTY_ELEMENTS: string[] = [
+	'area',
+	'base',
+	'br',
+	'col',
+	'embed',
+	'hr',
+	'img',
+	'input',
+	'keygen',
+	'link',
+	'menuitem',
+	'meta',
+	'param',
+	'source',
+	'track',
+	'wbr'
+];
+
+export const conf: languages.LanguageConfiguration = {
+	wordPattern: /(-?\d*\.\d\w*)|([^\`\~\!\@\$\^\&\*\(\)\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\s]+)/g,
+
+	comments: {
+		blockComment: ['<!--', '-->']
+	},
+
+	brackets: [
+		['<!--', '-->'],
+		['<', '>'],
+		['{', '}'],
+		['(', ')']
+	],
+
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+
+	surroundingPairs: [
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '<', close: '>' }
+	],
+
+	onEnterRules: [
+		{
+			beforeText: new RegExp(
+				`<(?!(?:${EMPTY_ELEMENTS.join('|')}))([_:\\w][_:\\w-.\\d]*)([^/>]*(?!/)>)[^<]*$`,
+				'i'
+			),
+			afterText: /^<\/([_:\w][_:\w-.\d]*)\s*>$/i,
+			action: {
+				indentAction: languages.IndentAction.IndentOutdent
+			}
+		},
+		{
+			beforeText: new RegExp(
+				`<(?!(?:${EMPTY_ELEMENTS.join('|')}))(\\w[\\w\\d]*)([^/>]*(?!/)>)[^<]*$`,
+				'i'
+			),
+			action: { indentAction: languages.IndentAction.Indent }
+		}
+	],
+
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*<!--\\s*#region\\b.*-->'),
+			end: new RegExp('^\\s*<!--\\s*#endregion\\b.*-->')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.html',
+	ignoreCase: true,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			[/<!DOCTYPE/, 'metatag', '@doctype'],
+			[/<!--/, 'comment', '@comment'],
+			[/(<)((?:[\w\-]+:)?[\w\-]+)(\s*)(\/>)/, ['delimiter', 'tag', '', 'delimiter']],
+			[/(<)(script)/, ['delimiter', { token: 'tag', next: '@script' }]],
+			[/(<)(style)/, ['delimiter', { token: 'tag', next: '@style' }]],
+			[/(<)((?:[\w\-]+:)?[\w\-]+)/, ['delimiter', { token: 'tag', next: '@otherTag' }]],
+			[/(<\/)((?:[\w\-]+:)?[\w\-]+)/, ['delimiter', { token: 'tag', next: '@otherTag' }]],
+			[/</, 'delimiter'],
+			[/[^<]+/] // text
+		],
+
+		doctype: [
+			[/[^>]+/, 'metatag.content'],
+			[/>/, 'metatag', '@pop']
+		],
+
+		comment: [
+			[/-->/, 'comment', '@pop'],
+			[/[^-]+/, 'comment.content'],
+			[/./, 'comment.content']
+		],
+
+		otherTag: [
+			[/\/?>/, 'delimiter', '@pop'],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[/[ \t\r\n]+/] // whitespace
+		],
+
+		// -- BEGIN <script> tags handling
+
+		// After <script
+		script: [
+			[/type/, 'attribute.name', '@scriptAfterType'],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@scriptEmbedded',
+					nextEmbedded: 'text/javascript'
+				}
+			],
+			[/[ \t\r\n]+/], // whitespace
+			[/(<\/)(script\s*)(>)/, ['delimiter', 'tag', { token: 'delimiter', next: '@pop' }]]
+		],
+
+		// After <script ... type
+		scriptAfterType: [
+			[/=/, 'delimiter', '@scriptAfterTypeEquals'],
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@scriptEmbedded',
+					nextEmbedded: 'text/javascript'
+				}
+			], // cover invalid e.g. <script type>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <script ... type =
+		scriptAfterTypeEquals: [
+			[
+				/"([^"]*)"/,
+				{
+					token: 'attribute.value',
+					switchTo: '@scriptWithCustomType.$1'
+				}
+			],
+			[
+				/'([^']*)'/,
+				{
+					token: 'attribute.value',
+					switchTo: '@scriptWithCustomType.$1'
+				}
+			],
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@scriptEmbedded',
+					nextEmbedded: 'text/javascript'
+				}
+			], // cover invalid e.g. <script type=>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <script ... type = $S2
+		scriptWithCustomType: [
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@scriptEmbedded.$S2',
+					nextEmbedded: '$S2'
+				}
+			],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/script\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		scriptEmbedded: [
+			[/<\/script/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }],
+			[/[^<]+/, '']
+		],
+
+		// -- END <script> tags handling
+
+		// -- BEGIN <style> tags handling
+
+		// After <style
+		style: [
+			[/type/, 'attribute.name', '@styleAfterType'],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@styleEmbedded',
+					nextEmbedded: 'text/css'
+				}
+			],
+			[/[ \t\r\n]+/], // whitespace
+			[/(<\/)(style\s*)(>)/, ['delimiter', 'tag', { token: 'delimiter', next: '@pop' }]]
+		],
+
+		// After <style ... type
+		styleAfterType: [
+			[/=/, 'delimiter', '@styleAfterTypeEquals'],
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@styleEmbedded',
+					nextEmbedded: 'text/css'
+				}
+			], // cover invalid e.g. <style type>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <style ... type =
+		styleAfterTypeEquals: [
+			[
+				/"([^"]*)"/,
+				{
+					token: 'attribute.value',
+					switchTo: '@styleWithCustomType.$1'
+				}
+			],
+			[
+				/'([^']*)'/,
+				{
+					token: 'attribute.value',
+					switchTo: '@styleWithCustomType.$1'
+				}
+			],
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@styleEmbedded',
+					nextEmbedded: 'text/css'
+				}
+			], // cover invalid e.g. <style type=>
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		// After <style ... type = $S2
+		styleWithCustomType: [
+			[
+				/>/,
+				{
+					token: 'delimiter',
+					next: '@styleEmbedded.$S2',
+					nextEmbedded: '$S2'
+				}
+			],
+			[/"([^"]*)"/, 'attribute.value'],
+			[/'([^']*)'/, 'attribute.value'],
+			[/[\w\-]+/, 'attribute.name'],
+			[/=/, 'delimiter'],
+			[/[ \t\r\n]+/], // whitespace
+			[/<\/style\s*>/, { token: '@rematch', next: '@pop' }]
+		],
+
+		styleEmbedded: [
+			[/<\/style/, { token: '@rematch', next: '@pop', nextEmbedded: '@pop' }],
+			[/[^<]+/, '']
+		]
+
+		// -- END <style> tags handling
+	}
+};
+
+// TESTED WITH:
+
+// <!DOCTYPE html>
+// <html>
+// <head>
+//   <title>Monarch Workbench</title>
+//   <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+//   <!----
+//   -- -- -- a comment -- -- --
+//   ---->
+//   <style bah="bah">
+//     body { font-family: Consolas; } /* nice */
+//   </style>
+// </head
+// >
+// a = "asd"
+// <body>
+//   <br/>
+//   <div
+//   class
+//   =
+//   "test"
+//   >
+//     <script>
+//       function() {
+//         alert("hi </ script>"); // javascript
+//       };
+//     </script>
+//     <script
+// 	bah="asdfg"
+// 	type="text/css"
+// 	>
+//   .bar { text-decoration: underline; }
+//     </script>
+//   </div>
+// </body>
+// </html>

monaco-languages/src/ini/ini.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'ini',
+	extensions: ['.ini', '.properties', '.gitconfig'],
+	filenames: ['config', '.gitattributes', '.gitconfig', '.editorconfig'],
+	aliases: ['Ini', 'ini'],
+	loader: () => import('./ini')
+});

monaco-languages/src/ini/ini.ts

@@ -0,0 +1,82 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	comments: {
+		lineComment: '#'
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.ini',
+
+	// we include these common regular expressions
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// sections
+			[/^\[[^\]]*\]/, 'metatag'],
+
+			// keys
+			[/(^\w+)(\s*)(\=)/, ['key', '', 'delimiter']],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// numbers
+			[/\d+/, 'number'],
+
+			// strings: recover on non-terminated strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/'([^'\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"/, 'string', '@string."'],
+			[/'/, 'string', "@string.'"]
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/^\s*[#;].*$/, 'comment']
+		],
+
+		string: [
+			[/[^\\"']+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[
+				/["']/,
+				{
+					cases: {
+						'$#==$S2': { token: 'string', next: '@pop' },
+						'@default': 'string'
+					}
+				}
+			]
+		]
+	}
+};

monaco-languages/src/java/java.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'java',
+	extensions: ['.java', '.jav'],
+	aliases: ['Java', 'java'],
+	mimetypes: ['text/x-java-source', 'text/x-java'],
+	loader: () => import('./java')
+});

monaco-languages/src/java/java.test.ts

@@ -0,0 +1,878 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('java', [
+	// Comments - single line
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.java' }
+			]
+		}
+	],
+
+	// Broken nested tokens due to invalid comment tokenization
+	[
+		{
+			line: '/* //*/ a',
+			tokens: [
+				{ startIndex: 0, type: 'comment.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		}
+	],
+
+	[
+		{
+			line: '/almost a comment',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.java' },
+				{ startIndex: 1, type: 'identifier.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.java' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1 / 2; /* comment',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.java' },
+				{ startIndex: 5, type: 'delimiter.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'comment.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'int x = 1; // my comment // is a nice one',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.java' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.java' },
+				{ startIndex: 9, type: 'delimiter.java' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'comment.java' }
+			]
+		}
+	],
+
+	// Comments - range comment, single line
+	[
+		{
+			line: '/* a simple comment */',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* a simple comment */ 1;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.java' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.java' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'number.java' },
+				{ startIndex: 32, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* comment */ 1; */',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.java' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.java' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'number.java' },
+				{ startIndex: 23, type: 'delimiter.java' },
+				{ startIndex: 24, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /**/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.java' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.java' },
+				{ startIndex: 8, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /*/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.java' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.java' }
+			]
+		}
+	],
+
+	// Comments - range comment, multiple lines
+	[
+		{
+			line: '/* start of multiline comment',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		},
+		{
+			line: 'a comment between without a star',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		},
+		{
+			line: 'end of multiline comment*/',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		}
+	],
+
+	[
+		{
+			line: 'int x = /* start a comment',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.java' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.java' }
+			]
+		},
+		{
+			line: ' a ',
+			tokens: [{ startIndex: 0, type: 'comment.java' }]
+		},
+		{
+			line: 'and end it */ 2;',
+			tokens: [
+				{ startIndex: 0, type: 'comment.java' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'number.java' },
+				{ startIndex: 15, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	// Java Doc, multiple lines
+	[
+		{
+			line: '/** start of Java Doc',
+			tokens: [{ startIndex: 0, type: 'comment.doc.java' }]
+		},
+		{
+			line: 'a comment between without a star',
+			tokens: [{ startIndex: 0, type: 'comment.doc.java' }]
+		},
+		{
+			line: 'end of multiline comment*/',
+			tokens: [{ startIndex: 0, type: 'comment.doc.java' }]
+		}
+	],
+
+	// Keywords
+	[
+		{
+			line: 'package test; class Program { static void main(String[] args) {} } }',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.package.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.java' },
+				{ startIndex: 12, type: 'delimiter.java' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'keyword.class.java' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'identifier.java' },
+				{ startIndex: 27, type: '' },
+				{ startIndex: 28, type: 'delimiter.curly.java' },
+				{ startIndex: 29, type: '' },
+				{ startIndex: 30, type: 'keyword.static.java' },
+				{ startIndex: 36, type: '' },
+				{ startIndex: 37, type: 'keyword.void.java' },
+				{ startIndex: 41, type: '' },
+				{ startIndex: 42, type: 'identifier.java' },
+				{ startIndex: 46, type: 'delimiter.parenthesis.java' },
+				{ startIndex: 47, type: 'identifier.java' },
+				{ startIndex: 53, type: 'delimiter.square.java' },
+				{ startIndex: 55, type: '' },
+				{ startIndex: 56, type: 'identifier.java' },
+				{ startIndex: 60, type: 'delimiter.parenthesis.java' },
+				{ startIndex: 61, type: '' },
+				{ startIndex: 62, type: 'delimiter.curly.java' },
+				{ startIndex: 64, type: '' },
+				{ startIndex: 65, type: 'delimiter.curly.java' },
+				{ startIndex: 66, type: '' },
+				{ startIndex: 67, type: 'delimiter.curly.java' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.java' }]
+		}
+	],
+
+	[
+		{
+			line: '0.10',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '0x',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.java' }]
+		}
+	],
+
+	[
+		{
+			line: '0x5_2',
+			tokens: [{ startIndex: 0, type: 'number.hex.java' }]
+		}
+	],
+
+	[
+		{
+			line: '023L',
+			tokens: [{ startIndex: 0, type: 'number.octal.java' }]
+		}
+	],
+
+	[
+		{
+			line: '0123l',
+			tokens: [{ startIndex: 0, type: 'number.octal.java' }]
+		}
+	],
+
+	[
+		{
+			line: '05_2',
+			tokens: [{ startIndex: 0, type: 'number.octal.java' }]
+		}
+	],
+
+	[
+		{
+			line: '0b1010_0101',
+			tokens: [{ startIndex: 0, type: 'number.binary.java' }]
+		}
+	],
+
+	[
+		{
+			line: '0B001',
+			tokens: [{ startIndex: 0, type: 'number.binary.java' }]
+		}
+	],
+
+	[
+		{
+			line: '10e3',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '10f',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e3',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5e-3',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E3',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5E-3',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5F',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5f',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5D',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23.5d',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3D',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E3d',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72E-3d',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3D',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e3d',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '1.72e-3d',
+			tokens: [{ startIndex: 0, type: 'number.float.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23L',
+			tokens: [{ startIndex: 0, type: 'number.java' }]
+		}
+	],
+
+	[
+		{
+			line: '23l',
+			tokens: [{ startIndex: 0, type: 'number.java' }]
+		}
+	],
+
+	[
+		{
+			line: '0_52',
+			tokens: [{ startIndex: 0, type: 'number.java' }]
+		}
+	],
+
+	[
+		{
+			line: '5_2',
+			tokens: [{ startIndex: 0, type: 'number.java' }]
+		}
+	],
+
+	[
+		{
+			line: '5_______2',
+			tokens: [{ startIndex: 0, type: 'number.java' }]
+		}
+	],
+
+	[
+		{
+			line: '3_.1415F',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: 'identifier.java' },
+				{ startIndex: 2, type: 'delimiter.java' },
+				{ startIndex: 3, type: 'number.float.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '3._1415F',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: 'delimiter.java' },
+				{ startIndex: 2, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '999_99_9999_L',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 11, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '52_',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 2, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0_x52',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x_52',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x52_',
+			tokens: [
+				{ startIndex: 0, type: 'number.hex.java' },
+				{ startIndex: 4, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '052_',
+			tokens: [
+				{ startIndex: 0, type: 'number.octal.java' },
+				{ startIndex: 3, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '23.5L',
+			tokens: [
+				{ startIndex: 0, type: 'number.float.java' },
+				{ startIndex: 4, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: 'delimiter.java' },
+				{ startIndex: 2, type: 'number.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 3, type: 'delimiter.java' },
+				{ startIndex: 4, type: 'number.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.java' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.java' }
+			]
+		}
+	],
+
+	// single line Strings
+	[
+		{
+			line: 'String s = "I\'m a Java String";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.java' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.java' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.java' },
+				{ startIndex: 30, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'String s = "concatenated" + " String" ;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.java' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.java' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.java' },
+				{ startIndex: 25, type: '' },
+				{ startIndex: 26, type: 'delimiter.java' },
+				{ startIndex: 27, type: '' },
+				{ startIndex: 28, type: 'string.java' },
+				{ startIndex: 37, type: '' },
+				{ startIndex: 38, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"quote in a string"',
+			tokens: [{ startIndex: 0, type: 'string.java' }]
+		}
+	],
+
+	[
+		{
+			line: '"escaping \\"quotes\\" is cool"',
+			tokens: [
+				{ startIndex: 0, type: 'string.java' },
+				{ startIndex: 10, type: 'string.escape.java' },
+				{ startIndex: 12, type: 'string.java' },
+				{ startIndex: 18, type: 'string.escape.java' },
+				{ startIndex: 20, type: 'string.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"\\"',
+			tokens: [{ startIndex: 0, type: 'string.invalid.java' }]
+		}
+	],
+
+	// Annotations
+	[
+		{
+			line: '@',
+			tokens: [{ startIndex: 0, type: '' }]
+		}
+	],
+
+	[
+		{
+			line: '@Override',
+			tokens: [{ startIndex: 0, type: 'annotation.java' }]
+		}
+	],
+
+	[
+		{
+			line: '@SuppressWarnings(value = "aString")',
+			tokens: [
+				{ startIndex: 0, type: 'annotation.java' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.java' },
+				{ startIndex: 18, type: 'identifier.java' },
+				{ startIndex: 23, type: '' },
+				{ startIndex: 24, type: 'delimiter.java' },
+				{ startIndex: 25, type: '' },
+				{ startIndex: 26, type: 'string.java' },
+				{ startIndex: 35, type: 'delimiter.parenthesis.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '@ AnnotationWithKeywordAfter private',
+			tokens: [
+				{ startIndex: 0, type: 'annotation.java' },
+				{ startIndex: 28, type: '' },
+				{ startIndex: 29, type: 'keyword.private.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'String s = """Multiline string""";',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.java' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.java' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.java' },
+				{ startIndex: 33, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'String s = """',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.java' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.java' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.java' }
+			]
+		},
+		{
+			line: '     <html></html>',
+			tokens: [{ startIndex: 0, type: 'string.java' }]
+		},
+		{
+			line: '     """;',
+			tokens: [
+				{ startIndex: 0, type: 'string.java' },
+				{ startIndex: 8, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'String s = """',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'identifier.java' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.java' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'string.java' }
+			]
+		},
+
+		{
+			line: '     \\"""Triple quoted string inside text block\\"""',
+			tokens: [
+				{ startIndex: 0, type: 'string.java' },
+				{ startIndex: 5, type: 'string.escape.java' },
+				{ startIndex: 7, type: 'string.java' },
+				{ startIndex: 47, type: 'string.escape.java' },
+				{ startIndex: 49, type: 'string.java' }
+			]
+		},
+
+		{
+			line: '     """;',
+			tokens: [
+				{ startIndex: 0, type: 'string.java' },
+				{ startIndex: 8, type: 'delimiter.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'yield "123"',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.yield.java' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'string.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'public sealed class Shape permits Circle, Square { }',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.public.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'keyword.sealed.java' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'keyword.class.java' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'identifier.java' },
+				{ startIndex: 25, type: '' },
+				{ startIndex: 26, type: 'keyword.permits.java' },
+				{ startIndex: 33, type: '' },
+				{ startIndex: 34, type: 'identifier.java' },
+				{ startIndex: 40, type: 'delimiter.java' },
+				{ startIndex: 41, type: '' },
+				{ startIndex: 42, type: 'identifier.java' },
+				{ startIndex: 48, type: '' },
+				{ startIndex: 49, type: 'delimiter.curly.java' },
+				{ startIndex: 50, type: '' },
+				{ startIndex: 51, type: 'delimiter.curly.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'public non-sealed class Shape',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.public.java' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'keyword.non-sealed.java' },
+				{ startIndex: 17, type: '' },
+				{ startIndex: 18, type: 'keyword.class.java' },
+				{ startIndex: 23, type: '' },
+				{ startIndex: 24, type: 'identifier.java' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'int x = y-z;', // Make sure "y-z" is not identifier
+			tokens: [
+				{ startIndex: 0, type: 'keyword.int.java' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.java' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.java' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.java' },
+				{ startIndex: 9, type: 'delimiter.java' },
+				{ startIndex: 10, type: 'identifier.java' },
+				{ startIndex: 11, type: 'delimiter.java' }
+			]
+		}
+	]
+]);

monaco-languages/src/java/java.ts

@@ -0,0 +1,253 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	// the default separators except `@$`
+	wordPattern:
+		/(-?\d*\.\d\w*)|([^\`\~\!\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g,
+	comments: {
+		lineComment: '//',
+		blockComment: ['/*', '*/']
+	},
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" },
+		{ open: '<', close: '>' }
+	],
+	folding: {
+		markers: {
+			start: new RegExp('^\\s*//\\s*(?:(?:#?region\\b)|(?:<editor-fold\\b))'),
+			end: new RegExp('^\\s*//\\s*(?:(?:#?endregion\\b)|(?:</editor-fold>))')
+		}
+	}
+};
+
+export const language = <languages.IMonarchLanguage>{
+	defaultToken: '',
+	tokenPostfix: '.java',
+
+	keywords: [
+		'abstract',
+		'continue',
+		'for',
+		'new',
+		'switch',
+		'assert',
+		'default',
+		'goto',
+		'package',
+		'synchronized',
+		'boolean',
+		'do',
+		'if',
+		'private',
+		'this',
+		'break',
+		'double',
+		'implements',
+		'protected',
+		'throw',
+		'byte',
+		'else',
+		'import',
+		'public',
+		'throws',
+		'case',
+		'enum',
+		'instanceof',
+		'return',
+		'transient',
+		'catch',
+		'extends',
+		'int',
+		'short',
+		'try',
+		'char',
+		'final',
+		'interface',
+		'static',
+		'void',
+		'class',
+		'finally',
+		'long',
+		'strictfp',
+		'volatile',
+		'const',
+		'float',
+		'native',
+		'super',
+		'while',
+		'true',
+		'false',
+		'yield',
+		'record',
+		'sealed',
+		'non-sealed',
+		'permits'
+	],
+
+	operators: [
+		'=',
+		'>',
+		'<',
+		'!',
+		'~',
+		'?',
+		':',
+		'==',
+		'<=',
+		'>=',
+		'!=',
+		'&&',
+		'||',
+		'++',
+		'--',
+		'+',
+		'-',
+		'*',
+		'/',
+		'&',
+		'|',
+		'^',
+		'%',
+		'<<',
+		'>>',
+		'>>>',
+		'+=',
+		'-=',
+		'*=',
+		'/=',
+		'&=',
+		'|=',
+		'^=',
+		'%=',
+		'<<=',
+		'>>=',
+		'>>>='
+	],
+
+	// we include these common regular expressions
+	symbols: /[=><!~?:&|+\-*\/\^%]+/,
+	escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+	digits: /\d+(_+\d+)*/,
+	octaldigits: /[0-7]+(_+[0-7]+)*/,
+	binarydigits: /[0-1]+(_+[0-1]+)*/,
+	hexdigits: /[[0-9a-fA-F]+(_+[0-9a-fA-F]+)*/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			// Special keyword with a dash
+			['non-sealed', 'keyword.non-sealed'],
+
+			// identifiers and keywords
+			[
+				/[a-zA-Z_$][\w$]*/,
+				{
+					cases: {
+						'@keywords': { token: 'keyword.$0' },
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// delimiters and operators
+			[/[{}()\[\]]/, '@brackets'],
+			[/[<>](?!@symbols)/, '@brackets'],
+			[
+				/@symbols/,
+				{
+					cases: {
+						'@operators': 'delimiter',
+						'@default': ''
+					}
+				}
+			],
+
+			// @ annotations.
+			[/@\s*[a-zA-Z_\$][\w\$]*/, 'annotation'],
+
+			// numbers
+			[/(@digits)[eE]([\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/(@digits)\.(@digits)([eE][\-+]?(@digits))?[fFdD]?/, 'number.float'],
+			[/0[xX](@hexdigits)[Ll]?/, 'number.hex'],
+			[/0(@octaldigits)[Ll]?/, 'number.octal'],
+			[/0[bB](@binarydigits)[Ll]?/, 'number.binary'],
+			[/(@digits)[fFdD]/, 'number.float'],
+			[/(@digits)[lL]?/, 'number'],
+
+			// delimiter: after number because of .\d floats
+			[/[;,.]/, 'delimiter'],
+
+			// strings
+			[/"([^"\\]|\\.)*$/, 'string.invalid'], // non-teminated string
+			[/"""/, 'string', '@multistring'],
+			[/"/, 'string', '@string'],
+
+			// characters
+			[/'[^\\']'/, 'string'],
+			[/(')(@escapes)(')/, ['string', 'string.escape', 'string']],
+			[/'/, 'string.invalid']
+		],
+
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/\/\*\*(?!\/)/, 'comment.doc', '@javadoc'],
+			[/\/\*/, 'comment', '@comment'],
+			[/\/\/.*$/, 'comment']
+		],
+
+		comment: [
+			[/[^\/*]+/, 'comment'],
+			// [/\/\*/, 'comment', '@push' ],    // nested comment not allowed :-(
+			// [/\/\*/,    'comment.invalid' ],    // this breaks block comments in the shape of /* //*/
+			[/\*\//, 'comment', '@pop'],
+			[/[\/*]/, 'comment']
+		],
+		//Identical copy of comment above, except for the addition of .doc
+		javadoc: [
+			[/[^\/*]+/, 'comment.doc'],
+			// [/\/\*/, 'comment.doc', '@push' ],    // nested comment not allowed :-(
+			[/\/\*/, 'comment.doc.invalid'],
+			[/\*\//, 'comment.doc', '@pop'],
+			[/[\/*]/, 'comment.doc']
+		],
+
+		string: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string', '@pop']
+		],
+
+		multistring: [
+			[/[^\\"]+/, 'string'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"""/, 'string', '@pop'],
+			[/./, 'string']
+		]
+	}
+};

monaco-languages/src/javascript/javascript.contribution.ts

@@ -0,0 +1,16 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'javascript',
+	extensions: ['.js', '.es6', '.jsx', '.mjs'],
+	firstLine: '^#!.*\\bnode',
+	filenames: ['jakefile'],
+	aliases: ['JavaScript', 'javascript', 'js'],
+	mimetypes: ['text/javascript'],
+	loader: () => import('./javascript')
+});

monaco-languages/src/javascript/javascript.test.ts

@@ -0,0 +1,865 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('javascript', [
+	// Keywords
+	[
+		{
+			line: 'var x = function() { };',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'keyword.js' },
+				{ startIndex: 16, type: 'delimiter.parenthesis.js' },
+				{ startIndex: 18, type: '' },
+				{ startIndex: 19, type: 'delimiter.bracket.js' },
+				{ startIndex: 20, type: '' },
+				{ startIndex: 21, type: 'delimiter.bracket.js' },
+				{ startIndex: 22, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '    var    ',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'keyword.js' },
+				{ startIndex: 7, type: '' }
+			]
+		}
+	],
+
+	// Comments - single line
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: '    // a comment',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 4, type: 'comment.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '// a comment',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: '// a comment /*',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: '// a comment /**',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: '//sticky comment',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: 'var x = 1; // my comment // is a nice one',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'number.js' },
+				{ startIndex: 9, type: 'delimiter.js' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'comment.js' }
+			]
+		}
+	],
+
+	// Comments - range comment, single line
+	[
+		{
+			line: '/* a simple comment */',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: 'var x = /* a simple comment */ 1;',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.js' },
+				{ startIndex: 30, type: '' },
+				{ startIndex: 31, type: 'number.js' },
+				{ startIndex: 32, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /**/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.js' },
+				{ startIndex: 8, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = /*/;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'comment.js' }
+			]
+		}
+	],
+
+	// Comments - range comment, multi lines
+	[
+		{
+			line: '/* a multiline comment',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		},
+		{
+			line: 'can actually span',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		},
+		{
+			line: 'multiple lines */',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: 'var x = /* start a comment',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'comment.js' }
+			]
+		},
+		{
+			line: ' a ',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		},
+		{
+			line: 'and end it */ var a = 2;',
+			tokens: [
+				{ startIndex: 0, type: 'comment.js' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'keyword.js' },
+				{ startIndex: 17, type: '' },
+				{ startIndex: 18, type: 'identifier.js' },
+				{ startIndex: 19, type: '' },
+				{ startIndex: 20, type: 'delimiter.js' },
+				{ startIndex: 21, type: '' },
+				{ startIndex: 22, type: 'number.js' },
+				{ startIndex: 23, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	// Strings
+	[
+		{
+			line: "var a = 'a';",
+			tokens: [
+				{ startIndex: 0, type: 'keyword.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'string.js' },
+				{ startIndex: 11, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"use strict";',
+			tokens: [
+				{ startIndex: 0, type: 'string.js' },
+				{ startIndex: 12, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'b = a + " \'cool\'  "',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'identifier.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'string.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '"escaping \\"quotes\\" is cool"',
+			tokens: [
+				{ startIndex: 0, type: 'string.js' },
+				{ startIndex: 10, type: 'string.escape.js' },
+				{ startIndex: 12, type: 'string.js' },
+				{ startIndex: 18, type: 'string.escape.js' },
+				{ startIndex: 20, type: 'string.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: "'''",
+			tokens: [
+				{ startIndex: 0, type: 'string.js' },
+				{ startIndex: 2, type: 'string.invalid.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: "'\\''",
+			tokens: [
+				{ startIndex: 0, type: 'string.js' },
+				{ startIndex: 1, type: 'string.escape.js' },
+				{ startIndex: 3, type: 'string.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: "'be careful \\not to escape'",
+			tokens: [
+				{ startIndex: 0, type: 'string.js' },
+				{ startIndex: 12, type: 'string.escape.js' },
+				{ startIndex: 14, type: 'string.js' }
+			]
+		}
+	],
+
+	// Numbers
+	[
+		{
+			line: '0',
+			tokens: [{ startIndex: 0, type: 'number.js' }]
+		}
+	],
+
+	[
+		{
+			line: ' 0',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: ' 0 ',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'number.js' },
+				{ startIndex: 2, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 ',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0+0',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: 'delimiter.js' },
+				{ startIndex: 2, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100+10',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 3, type: 'delimiter.js' },
+				{ startIndex: 4, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0 + 0',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0123',
+			tokens: [{ startIndex: 0, type: 'number.octal.js' }]
+		}
+	],
+
+	[
+		{
+			line: '01239',
+			tokens: [
+				{ startIndex: 0, type: 'number.octal.js' },
+				{ startIndex: 4, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0o123',
+			tokens: [{ startIndex: 0, type: 'number.octal.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0O123',
+			tokens: [{ startIndex: 0, type: 'number.octal.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0x',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: 'identifier.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0x123',
+			tokens: [{ startIndex: 0, type: 'number.hex.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0X123',
+			tokens: [{ startIndex: 0, type: 'number.hex.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0b101',
+			tokens: [{ startIndex: 0, type: 'number.binary.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0B101',
+			tokens: [{ startIndex: 0, type: 'number.binary.js' }]
+		}
+	],
+
+	// Bigint
+	[
+		{
+			line: '0n',
+			tokens: [{ startIndex: 0, type: 'number.js' }]
+		}
+	],
+
+	[
+		{
+			line: ' 0n',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: ' 0n ',
+			tokens: [
+				{ startIndex: 0, type: '' },
+				{ startIndex: 1, type: 'number.js' },
+				{ startIndex: 3, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0n ',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 2, type: '' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0n+0n',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '100n+10n',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 4, type: 'delimiter.js' },
+				{ startIndex: 5, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0n + 0n',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 2, type: '' },
+				{ startIndex: 3, type: 'delimiter.js' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '0b101n',
+			tokens: [{ startIndex: 0, type: 'number.binary.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0123n',
+			tokens: [{ startIndex: 0, type: 'number.octal.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0o123n',
+			tokens: [{ startIndex: 0, type: 'number.octal.js' }]
+		}
+	],
+
+	[
+		{
+			line: '0x123n',
+			tokens: [{ startIndex: 0, type: 'number.hex.js' }]
+		}
+	],
+
+	// Regular Expressions
+	[
+		{
+			line: '//',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: '/**/',
+			tokens: [{ startIndex: 0, type: 'comment.js' }]
+		}
+	],
+
+	[
+		{
+			line: '/***/',
+			tokens: [{ startIndex: 0, type: 'comment.doc.js' }]
+		}
+	],
+
+	[
+		{
+			line: '5 / 3;',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.js' },
+				{ startIndex: 5, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	// Advanced regular expressions
+	[
+		{
+			line: '1 / 2; /* comment',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.js' },
+				{ startIndex: 5, type: 'delimiter.js' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'comment.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1 / 2 / x / b;',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.js' },
+				{ startIndex: 9, type: '' },
+				{ startIndex: 10, type: 'delimiter.js' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'identifier.js' },
+				{ startIndex: 13, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: "x = /foo/.test('')",
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'regexp.js' },
+				{ startIndex: 9, type: 'delimiter.js' },
+				{ startIndex: 10, type: 'identifier.js' },
+				{ startIndex: 14, type: 'delimiter.parenthesis.js' },
+				{ startIndex: 15, type: 'string.js' },
+				{ startIndex: 17, type: 'delimiter.parenthesis.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '/foo/',
+			tokens: [{ startIndex: 0, type: 'regexp.js' }]
+		}
+	],
+
+	[
+		{
+			line: '/foo/g',
+			tokens: [
+				{ startIndex: 0, type: 'regexp.js' },
+				{ startIndex: 5, type: 'keyword.other.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '/foo/dgimsuy',
+			tokens: [
+				{ startIndex: 0, type: 'regexp.js' },
+				{ startIndex: 5, type: 'keyword.other.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '/foo/q', // invalid flag
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.js' },
+				{ startIndex: 1, type: 'identifier.js' },
+				{ startIndex: 4, type: 'delimiter.js' },
+				{ startIndex: 5, type: 'identifier.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = 1 + f(2 / 3, /foo/)',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.js' },
+				{ startIndex: 5, type: '' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.js' },
+				{ startIndex: 9, type: 'delimiter.parenthesis.js' },
+				{ startIndex: 10, type: 'number.js' },
+				{ startIndex: 11, type: '' },
+				{ startIndex: 12, type: 'delimiter.js' },
+				{ startIndex: 13, type: '' },
+				{ startIndex: 14, type: 'number.js' },
+				{ startIndex: 15, type: 'delimiter.js' },
+				{ startIndex: 16, type: '' },
+				{ startIndex: 17, type: 'regexp.js' },
+				{ startIndex: 22, type: 'delimiter.parenthesis.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'a /ads/ b;',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: 'identifier.js' },
+				{ startIndex: 6, type: 'delimiter.js' },
+				{ startIndex: 7, type: '' },
+				{ startIndex: 8, type: 'identifier.js' },
+				{ startIndex: 9, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '1/(2/3)/2/3;',
+			tokens: [
+				{ startIndex: 0, type: 'number.js' },
+				{ startIndex: 1, type: 'delimiter.js' },
+				{ startIndex: 2, type: 'delimiter.parenthesis.js' },
+				{ startIndex: 3, type: 'number.js' },
+				{ startIndex: 4, type: 'delimiter.js' },
+				{ startIndex: 5, type: 'number.js' },
+				{ startIndex: 6, type: 'delimiter.parenthesis.js' },
+				{ startIndex: 7, type: 'delimiter.js' },
+				{ startIndex: 8, type: 'number.js' },
+				{ startIndex: 9, type: 'delimiter.js' },
+				{ startIndex: 10, type: 'number.js' },
+				{ startIndex: 11, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '{ key: 123 }',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.bracket.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'identifier.js' },
+				{ startIndex: 5, type: 'delimiter.js' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'number.js' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'delimiter.bracket.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '[1,2,3]',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.square.js' },
+				{ startIndex: 1, type: 'number.js' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: 'number.js' },
+				{ startIndex: 4, type: 'delimiter.js' },
+				{ startIndex: 5, type: 'number.js' },
+				{ startIndex: 6, type: 'delimiter.square.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'foo(123);',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 3, type: 'delimiter.parenthesis.js' },
+				{ startIndex: 4, type: 'number.js' },
+				{ startIndex: 7, type: 'delimiter.parenthesis.js' },
+				{ startIndex: 8, type: 'delimiter.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: '{a:{b:[]}}',
+			tokens: [
+				{ startIndex: 0, type: 'delimiter.bracket.js' },
+				{ startIndex: 1, type: 'identifier.js' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: 'delimiter.bracket.js' },
+				{ startIndex: 4, type: 'identifier.js' },
+				{ startIndex: 5, type: 'delimiter.js' },
+				{ startIndex: 6, type: 'delimiter.square.js' },
+				{ startIndex: 8, type: 'delimiter.bracket.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'x = "[{()}]"',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'delimiter.js' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'string.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'test ? 1 : 2',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'delimiter.js' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'number.js' },
+				{ startIndex: 8, type: '' },
+				{ startIndex: 9, type: 'delimiter.js' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'couldBeNullish ?? 1',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.js' },
+				{ startIndex: 14, type: '' },
+				{ startIndex: 15, type: 'delimiter.js' },
+				{ startIndex: 17, type: '' },
+				{ startIndex: 18, type: 'number.js' }
+			]
+		}
+	],
+
+	[
+		{
+			line: "`${5 + 'x' + 3}a${4}`",
+			tokens: [
+				{ startIndex: 0, type: 'string.js' },
+				{ startIndex: 1, type: 'delimiter.bracket.js' },
+				{ startIndex: 3, type: 'number.js' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'delimiter.js' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'string.js' },
+				{ startIndex: 10, type: '' },
+				{ startIndex: 11, type: 'delimiter.js' },
+				{ startIndex: 12, type: '' },
+				{ startIndex: 13, type: 'number.js' },
+				{ startIndex: 14, type: 'delimiter.bracket.js' },
+				{ startIndex: 15, type: 'string.js' },
+				{ startIndex: 16, type: 'delimiter.bracket.js' },
+				{ startIndex: 18, type: 'number.js' },
+				{ startIndex: 19, type: 'delimiter.bracket.js' },
+				{ startIndex: 20, type: 'string.js' }
+			]
+		}
+	]
+]);

monaco-languages/src/javascript/javascript.ts

@@ -0,0 +1,76 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { conf as tsConf, language as tsLanguage } from '../typescript/typescript';
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = tsConf;
+
+export const language = <languages.IMonarchLanguage>{
+	// Set defaultToken to invalid to see what you do not tokenize yet
+	defaultToken: 'invalid',
+	tokenPostfix: '.js',
+
+	keywords: [
+		'break',
+		'case',
+		'catch',
+		'class',
+		'continue',
+		'const',
+		'constructor',
+		'debugger',
+		'default',
+		'delete',
+		'do',
+		'else',
+		'export',
+		'extends',
+		'false',
+		'finally',
+		'for',
+		'from',
+		'function',
+		'get',
+		'if',
+		'import',
+		'in',
+		'instanceof',
+		'let',
+		'new',
+		'null',
+		'return',
+		'set',
+		'super',
+		'switch',
+		'symbol',
+		'this',
+		'throw',
+		'true',
+		'try',
+		'typeof',
+		'undefined',
+		'var',
+		'void',
+		'while',
+		'with',
+		'yield',
+		'async',
+		'await',
+		'of'
+	],
+	typeKeywords: [],
+
+	operators: tsLanguage.operators,
+	symbols: tsLanguage.symbols,
+	escapes: tsLanguage.escapes,
+	digits: tsLanguage.digits,
+	octaldigits: tsLanguage.octaldigits,
+	binarydigits: tsLanguage.binarydigits,
+	hexdigits: tsLanguage.hexdigits,
+	regexpctl: tsLanguage.regexpctl,
+	regexpesc: tsLanguage.regexpesc,
+	tokenizer: tsLanguage.tokenizer
+};

monaco-languages/src/julia/julia.contribution.ts

@@ -0,0 +1,13 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'julia',
+	extensions: ['.jl'],
+	aliases: ['julia', 'Julia'],
+	loader: () => import('./julia')
+});

monaco-languages/src/julia/julia.test.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { testTokenization } from '../test/testRunner';
+
+testTokenization('julia', [
+	[
+		{
+			line: 'a = 1',
+			tokens: [
+				{ startIndex: 0, type: 'identifier.julia' },
+				{ startIndex: 1, type: '' },
+				{ startIndex: 2, type: 'source.julia' },
+				{ startIndex: 3, type: '' },
+				{ startIndex: 4, type: 'number.julia' }
+			]
+		}
+	],
+
+	[
+		{
+			line: 'b(c) = 2c',
+			tokens: [
+				{ startIndex: 0, type: 'keyword.flow.julia' },
+				{ startIndex: 1, type: 'delimiter.parenthesis.julia' },
+				{ startIndex: 2, type: 'identifier.julia' },
+				{ startIndex: 3, type: 'delimiter.parenthesis.julia' },
+				{ startIndex: 4, type: '' },
+				{ startIndex: 5, type: 'source.julia' },
+				{ startIndex: 6, type: '' },
+				{ startIndex: 7, type: 'number.julia' },
+				{ startIndex: 8, type: 'identifier.julia' }
+			]
+		}
+	]
+]);

monaco-languages/src/julia/julia.ts

@@ -0,0 +1,550 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import type { languages } from '../fillers/monaco-editor-core';
+
+export const conf: languages.LanguageConfiguration = {
+	brackets: [
+		['{', '}'],
+		['[', ']'],
+		['(', ')']
+	],
+	autoClosingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	],
+	surroundingPairs: [
+		{ open: '{', close: '}' },
+		{ open: '[', close: ']' },
+		{ open: '(', close: ')' },
+		{ open: '"', close: '"' },
+		{ open: "'", close: "'" }
+	]
+};
+
+export const language = <languages.IMonarchLanguage>{
+	tokenPostfix: '.julia',
+
+	keywords: [
+		'begin',
+		'while',
+		'if',
+		'for',
+		'try',
+		'return',
+		'break',
+		'continue',
+		'function',
+		'macro',
+		'quote',
+		'let',
+		'local',
+		'global',
+		'const',
+		'do',
+		'struct',
+		'module',
+		'baremodule',
+		'using',
+		'import',
+		'export',
+		'end',
+		'else',
+		'elseif',
+		'catch',
+		'finally',
+		'mutable',
+		'primitive',
+		'abstract',
+		'type',
+		'in',
+		'isa',
+		'where',
+		'new'
+	],
+
+	types: [
+		'LinRange',
+		'LineNumberNode',
+		'LinearIndices',
+		'LoadError',
+		'MIME',
+		'Matrix',
+		'Method',
+		'MethodError',
+		'Missing',
+		'MissingException',
+		'Module',
+		'NTuple',
+		'NamedTuple',
+		'Nothing',
+		'Number',
+		'OrdinalRange',
+		'OutOfMemoryError',
+		'OverflowError',
+		'Pair',
+		'PartialQuickSort',
+		'PermutedDimsArray',
+		'Pipe',
+		'Ptr',
+		'QuoteNode',
+		'Rational',
+		'RawFD',
+		'ReadOnlyMemoryError',
+		'Real',
+		'ReentrantLock',
+		'Ref',
+		'Regex',
+		'RegexMatch',
+		'RoundingMode',
+		'SegmentationFault',
+		'Set',
+		'Signed',
+		'Some',
+		'StackOverflowError',
+		'StepRange',
+		'StepRangeLen',
+		'StridedArray',
+		'StridedMatrix',
+		'StridedVecOrMat',
+		'StridedVector',
+		'String',
+		'StringIndexError',
+		'SubArray',
+		'SubString',
+		'SubstitutionString',
+		'Symbol',
+		'SystemError',
+		'Task',
+		'Text',
+		'TextDisplay',
+		'Timer',
+		'Tuple',
+		'Type',
+		'TypeError',
+		'TypeVar',
+		'UInt',
+		'UInt128',
+		'UInt16',
+		'UInt32',
+		'UInt64',
+		'UInt8',
+		'UndefInitializer',
+		'AbstractArray',
+		'UndefKeywordError',
+		'AbstractChannel',
+		'UndefRefError',
+		'AbstractChar',
+		'UndefVarError',
+		'AbstractDict',
+		'Union',
+		'AbstractDisplay',
+		'UnionAll',
+		'AbstractFloat',
+		'UnitRange',
+		'AbstractIrrational',
+		'Unsigned',
+		'AbstractMatrix',
+		'AbstractRange',
+		'Val',
+		'AbstractSet',
+		'Vararg',
+		'AbstractString',
+		'VecElement',
+		'AbstractUnitRange',
+		'VecOrMat',
+		'AbstractVecOrMat',
+		'Vector',
+		'AbstractVector',
+		'VersionNumber',
+		'Any',
+		'WeakKeyDict',
+		'ArgumentError',
+		'WeakRef',
+		'Array',
+		'AssertionError',
+		'BigFloat',
+		'BigInt',
+		'BitArray',
+		'BitMatrix',
+		'BitSet',
+		'BitVector',
+		'Bool',
+		'BoundsError',
+		'CapturedException',
+		'CartesianIndex',
+		'CartesianIndices',
+		'Cchar',
+		'Cdouble',
+		'Cfloat',
+		'Channel',
+		'Char',
+		'Cint',
+		'Cintmax_t',
+		'Clong',
+		'Clonglong',
+		'Cmd',
+		'Colon',
+		'Complex',
+		'ComplexF16',
+		'ComplexF32',
+		'ComplexF64',
+		'CompositeException',
+		'Condition',
+		'Cptrdiff_t',
+		'Cshort',
+		'Csize_t',
+		'Cssize_t',
+		'Cstring',
+		'Cuchar',
+		'Cuint',
+		'Cuintmax_t',
+		'Culong',
+		'Culonglong',
+		'Cushort',
+		'Cvoid',
+		'Cwchar_t',
+		'Cwstring',
+		'DataType',
+		'DenseArray',
+		'DenseMatrix',
+		'DenseVecOrMat',
+		'DenseVector',
+		'Dict',
+		'DimensionMismatch',
+		'Dims',
+		'DivideError',
+		'DomainError',
+		'EOFError',
+		'Enum',
+		'ErrorException',
+		'Exception',
+		'ExponentialBackOff',
+		'Expr',
+		'Float16',
+		'Float32',
+		'Float64',
+		'Function',
+		'GlobalRef',
+		'HTML',
+		'IO',
+		'IOBuffer',
+		'IOContext',
+		'IOStream',
+		'IdDict',
+		'IndexCartesian',
+		'IndexLinear',
+		'IndexStyle',
+		'InexactError',
+		'InitError',
+		'Int',
+		'Int128',
+		'Int16',
+		'Int32',
+		'Int64',
+		'Int8',
+		'Integer',
+		'InterruptException',
+		'InvalidStateException',
+		'Irrational',
+		'KeyError'
+	],
+
+	keywordops: ['<:', '>:', ':', '=>', '...', '.', '->', '?'],
+
+	allops: /[^\w\d\s()\[\]{}"'#]+/,
+
+	constants: [
+		'true',
+		'false',
+		'nothing',
+		'missing',
+		'undef',
+		'Inf',
+		'pi',
+		'NaN',
+		'π',
+		'ℯ',
+		'ans',
+		'PROGRAM_FILE',
+		'ARGS',
+		'C_NULL',
+		'VERSION',
+		'DEPOT_PATH',
+		'LOAD_PATH'
+	],
+
+	operators: [
+		'!',
+		'!=',
+		'!==',
+		'%',
+		'&',
+		'*',
+		'+',
+		'-',
+		'/',
+		'//',
+		'<',
+		'<<',
+		'<=',
+		'==',
+		'===',
+		'=>',
+		'>',
+		'>=',
+		'>>',
+		'>>>',
+		'\\',
+		'^',
+		'|',
+		'|>',
+		'~',
+		'÷',
+		'∈',
+		'∉',
+		'∋',
+		'∌',
+		'∘',
+		'√',
+		'∛',
+		'∩',
+		'∪',
+		'≈',
+		'≉',
+		'≠',
+		'≡',
+		'≢',
+		'≤',
+		'≥',
+		'⊆',
+		'⊇',
+		'⊈',
+		'⊉',
+		'⊊',
+		'⊋',
+		'⊻'
+	],
+
+	brackets: [
+		{ open: '(', close: ')', token: 'delimiter.parenthesis' },
+		{ open: '{', close: '}', token: 'delimiter.curly' },
+		{ open: '[', close: ']', token: 'delimiter.square' }
+	],
+
+	ident: /π|ℯ|\b(?!\d)\w+\b/,
+
+	// escape sequences
+	escape: /(?:[abefnrstv\\"'\n\r]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2}|u[0-9A-Fa-f]{4})/,
+	escapes: /\\(?:C\-(@escape|.)|c(@escape|.)|@escape)/,
+
+	// The main tokenizer for our languages
+	tokenizer: {
+		root: [
+			[/(::)\s*|\b(isa)\s+/, 'keyword', '@typeanno'],
+			[/\b(isa)(\s*\(@ident\s*,\s*)/, ['keyword', { token: '', next: '@typeanno' }]],
+			[/\b(type|struct)[ \t]+/, 'keyword', '@typeanno'],
+
+			// symbols
+			[/^\s*:@ident[!?]?/, 'metatag'],
+			[/(return)(\s*:@ident[!?]?)/, ['keyword', 'metatag']],
+			[/(\(|\[|\{|@allops)(\s*:@ident[!?]?)/, ['', 'metatag']],
+			[/:\(/, 'metatag', '@quote'],
+
+			// regular expressions
+			[/r"""/, 'regexp.delim', '@tregexp'],
+			[/r"/, 'regexp.delim', '@sregexp'],
+
+			// strings
+			[/raw"""/, 'string.delim', '@rtstring'],
+			[/[bv]?"""/, 'string.delim', '@dtstring'],
+
+			[/raw"/, 'string.delim', '@rsstring'],
+			[/[bv]?"/, 'string.delim', '@dsstring'],
+
+			[
+				/(@ident)\{/,
+				{
+					cases: {
+						'$1@types': { token: 'type', next: '@gen' },
+						'@default': { token: 'type', next: '@gen' }
+					}
+				}
+			],
+
+			[
+				/@ident[!?'']?(?=\.?\()/,
+				{
+					cases: {
+						'@types': 'type',
+						'@keywords': 'keyword',
+						'@constants': 'variable',
+						'@default': 'keyword.flow'
+					}
+				}
+			],
+
+			[
+				/@ident[!?']?/,
+				{
+					cases: {
+						'@types': 'type',
+						'@keywords': 'keyword',
+						'@constants': 'variable',
+						'@default': 'identifier'
+					}
+				}
+			],
+
+			[/\$\w+/, 'key'],
+			[/\$\(/, 'key', '@paste'],
+
+			[/@@@ident/, 'annotation'],
+
+			// whitespace
+			{ include: '@whitespace' },
+
+			// characters
+			[/'(?:@escapes|.)'/, 'string.character'],
+
+			// delimiters and operators
+			[/[()\[\]{}]/, '@brackets'],
+			[
+				/@allops/,
+				{
+					cases: {
+						'@keywordops': 'keyword',
+						'@operators': 'operator'
+					}
+				}
+			],
+			[/[;,]/, 'delimiter'],
+
+			// numbers
+			[/0[xX][0-9a-fA-F](_?[0-9a-fA-F])*/, 'number.hex'],
+			[/0[_oO][0-7](_?[0-7])*/, 'number.octal'],
+			[/0[bB][01](_?[01])*/, 'number.binary'],
+			[/[+\-]?\d+(\.\d+)?(im?|[eE][+\-]?\d+(\.\d+)?)?/, 'number']
+		],
+
+		// type
+		typeanno: [
+			[/[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*\{/, 'type', '@gen'],
+			[/([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)(\s*<:\s*)/, ['type', 'keyword']],
+			[/[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*/, 'type', '@pop'],
+			['', '', '@pop']
+		],
+
+		// generic type
+		gen: [
+			[/[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*\{/, 'type', '@push'],
+			[/[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*/, 'type'],
+			[/<:/, 'keyword'],
+			[/(\})(\s*<:\s*)/, ['type', { token: 'keyword', next: '@pop' }]],
+			[/\}/, 'type', '@pop'],
+			{ include: '@root' }
+		],
+
+		// $(...)
+		quote: [
+			[/\$\(/, 'key', '@paste'],
+			[/\(/, '@brackets', '@paren'],
+			[/\)/, 'metatag', '@pop'],
+			{ include: '@root' }
+		],
+
+		// :(...)
+		paste: [
+			[/:\(/, 'metatag', '@quote'],
+			[/\(/, '@brackets', '@paren'],
+			[/\)/, 'key', '@pop'],
+			{ include: '@root' }
+		],
+
+		// (...)
+		paren: [
+			[/\$\(/, 'key', '@paste'],
+			[/:\(/, 'metatag', '@quote'],
+			[/\(/, '@brackets', '@push'],
+			[/\)/, '@brackets', '@pop'],
+			{ include: '@root' }
+		],
+
+		// r"egex string"
+		sregexp: [
+			[/^.*/, 'invalid'],
+			[/[^\\"()\[\]{}]/, 'regexp'],
+			[/[()\[\]{}]/, '@brackets'],
+			[/\\./, 'operator.scss'],
+			[/"[imsx]*/, 'regexp.delim', '@pop']
+		],
+		tregexp: [
+			[/[^\\"()\[\]{}]/, 'regexp'],
+			[/[()\[\]{}]/, '@brackets'],
+			[/\\./, 'operator.scss'],
+			[/"(?!"")/, 'string'],
+			[/"""[imsx]*/, 'regexp.delim', '@pop']
+		],
+
+		// raw"string"
+		rsstring: [
+			[/^.*/, 'invalid'],
+			[/[^\\"]/, 'string'],
+			[/\\./, 'string.escape'],
+			[/"/, 'string.delim', '@pop']
+		],
+		rtstring: [
+			[/[^\\"]/, 'string'],
+			[/\\./, 'string.escape'],
+			[/"(?!"")/, 'string'],
+			[/"""/, 'string.delim', '@pop']
+		],
+
+		// "string".
+		dsstring: [
+			[/^.*/, 'invalid'],
+			[/[^\\"\$]/, 'string'],
+			[/\$/, '', '@interpolated'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"/, 'string.delim', '@pop']
+		],
+		dtstring: [
+			[/[^\\"\$]/, 'string'],
+			[/\$/, '', '@interpolated'],
+			[/@escapes/, 'string.escape'],
+			[/\\./, 'string.escape.invalid'],
+			[/"(?!"")/, 'string'],
+			[/"""/, 'string.delim', '@pop']
+		],
+
+		// interpolated sequence
+		interpolated: [
+			[/\(/, { token: '', switchTo: '@interpolated_compound' }],
+			[/[a-zA-Z_]\w*/, 'identifier'],
+			['', '', '@pop'] // just a $ is interpreted as a $
+		],
+
+		// any code
+		interpolated_compound: [[/\)/, '', '@pop'], { include: '@root' }],
+
+		// whitespace & comments
+		whitespace: [
+			[/[ \t\r\n]+/, ''],
+			[/#=/, 'comment', '@multi_comment'],
+			[/#.*$/, 'comment']
+		],
+
+		multi_comment: [
+			[/#=/, 'comment', '@push'],
+			[/=#/, 'comment', '@pop'],
+			[/=(?!#)|#(?!=)/, 'comment'],
+			[/[^#=]+/, 'comment']
+		]
+	}
+};

monaco-languages/src/kotlin/kotlin.contribution.ts

@@ -0,0 +1,14 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { registerLanguage } from '../_.contribution';
+
+registerLanguage({
+	id: 'kotlin',
+	extensions: ['.kt'],
+	aliases: ['Kotlin', 'kotlin'],
+	mimetypes: ['text/x-kotlin-source', 'text/x-kotlin'],
+	loader: () => import('./kotlin')
+});