Tokenization of html, razor and handlebars is in monaco-languages

package.json

@@ -25,7 +25,7 @@
     "gulp-tsb": "^1.10.4",
     "gulp-uglify": "^1.5.3",
     "merge-stream": "^1.0.0",
-    "monaco-editor-core": "0.6.0",
+    "monaco-editor-core": "0.7.1",
     "object-assign": "^4.1.0",
     "rimraf": "^2.5.2",
     "typescript": "1.8.10",

src/htmlMode.ts

@@ -8,7 +8,6 @@ import {WorkerManager} from './workerManager';
 import {HTMLWorker} from './htmlWorker';
 import {LanguageServiceDefaultsImpl} from './monaco.contribution';
 import * as languageFeatures from './languageFeatures';
-import {createTokenizationSupport} from './tokenization';
 
 import Promise = monaco.Promise;
 import Uri = monaco.Uri;
@@ -27,53 +26,15 @@ export function setupMode(defaults: LanguageServiceDefaultsImpl): void {
 
 	let languageId = defaults.languageId;
 
+	// all modes
 	disposables.push(monaco.languages.registerCompletionItemProvider(languageId, new languageFeatures.CompletionAdapter(worker)));
 	disposables.push(monaco.languages.registerDocumentHighlightProvider(languageId, new languageFeatures.DocumentHighlightAdapter(worker)));
+	disposables.push(monaco.languages.registerLinkProvider(languageId, new languageFeatures.DocumentLinkAdapter(worker)));
+
+	// only html
+	if (languageId === 'html') {
 		disposables.push(monaco.languages.registerDocumentFormattingEditProvider(languageId, new languageFeatures.DocumentFormattingEditProvider(worker)));
 		disposables.push(monaco.languages.registerDocumentRangeFormattingEditProvider(languageId, new languageFeatures.DocumentRangeFormattingEditProvider(worker)));
 		disposables.push(new languageFeatures.DiagnostcsAdapter(languageId, worker));
-	disposables.push(monaco.languages.registerLinkProvider(languageId, new languageFeatures.DocumentLinkAdapter(worker)));
-	disposables.push(monaco.languages.setTokensProvider(languageId, createTokenizationSupport(true)));
-	disposables.push(monaco.languages.setLanguageConfiguration(languageId, richEditConfiguration));
-}
-
-const EMPTY_ELEMENTS:string[] = ['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr'];
-
-const richEditConfiguration: monaco.languages.LanguageConfiguration = {
-	wordPattern: /(-?\d*\.\d\w*)|([^\[\{\]\}\:\"\,\s]+)/g,
-
-		comments: {
-			blockComment: ['<!--', '-->']
-		},
-
-		brackets: [
-			['<!--', '-->'],
-			['<', '>'],
-		],
-
-		autoClosingPairs: [
-			{ open: '{', close: '}' },
-			{ open: '[', close: ']' },
-			{ open: '(', close: ')' },
-			{ open: '"', close: '"' },
-			{ open: '\'', close: '\'' }
-		],
-
-		surroundingPairs: [
-			{ open: '"', close: '"' },
-			{ open: '\'', close: '\'' }
-		],
-
-		onEnterRules: [
-			{
-				beforeText: new RegExp(`<(?!(?:${EMPTY_ELEMENTS.join('|')}))([_:\\w][_:\\w-.\\d]*)([^/>]*(?!/)>)[^<]*$`, 'i'),
-				afterText: /^<\/([_:\w][_:\w-.\d]*)\s*>$/i,
-				action: { indentAction: monaco.languages.IndentAction.IndentOutdent }
-			},
-			{
-				beforeText: new RegExp(`<(?!(?:${EMPTY_ELEMENTS.join('|')}))(\\w[\\w\\d]*)([^/>]*(?!/)>)[^<]*$`, 'i'),
-				action: { indentAction: monaco.languages.IndentAction.Indent }
 	}
-		],
+}
-};
-

src/monaco.contribution.ts

@@ -95,24 +95,6 @@ function withMode(callback: (module: typeof mode) => void): void {
 	require<typeof mode>(['vs/language/html/htmlMode'], callback);
 }
 
-monaco.languages.register({
-	id: htmlLanguageId,
-	extensions: ['.html', '.htm', '.shtml', '.xhtml', '.mdoc', '.jsp', '.asp', '.aspx', '.jshtm'],
-	aliases: ['HTML', 'htm', 'html', 'xhtml'],
-	mimetypes: ['text/html', 'text/x-jshtm', 'text/template', 'text/ng-template']
-});
-monaco.languages.register({
-	id: handlebarsLanguageId,
-	extensions: ['.handlebars', '.hbs'],
-	aliases: ['Handlebars', 'handlebars'],
-	mimetypes: ['text/x-handlebars-template']
-});
-monaco.languages.register({
-	id: razorLanguageId,
-	extensions: ['.cshtml'],
-	aliases: ['Razor', 'razor'],
-	mimetypes: ['text/x-cshtml']
-});
 monaco.languages.onLanguage(htmlLanguageId, () => {
 	withMode(mode => mode.setupMode(htmlDefaults));
 });

src/tokenization.ts

@@ -1,136 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import {Scanner, ScannerState, TokenType, createScanner} from 'vscode-html-languageservice/lib/parser/htmlScanner';
-
-
-export function createTokenizationSupport(supportComments: boolean): monaco.languages.TokensProvider {
-    return {
-        getInitialState: () => new HTMLState(null, ScannerState.WithinContent),
-        tokenize: (line, state, offsetDelta?, stopAtOffset?) => tokenize(line, <HTMLState>state, offsetDelta, stopAtOffset)
-    };
-}
-
-const DELIM_END = 'punctuation.definition.meta.tag.end.html';
-const DELIM_START = 'punctuation.definition.meta.tag.begin.html';
-const DELIM_ASSIGN = 'meta.tag.assign.html';
-const ATTRIB_NAME = 'entity.other.attribute-name.html';
-const ATTRIB_VALUE = 'string.html';
-const COMMENT = 'comment.html.content';
-const DELIM_COMMENT = 'comment.html';
-const DOCTYPE = 'entity.other.attribute-name.html';
-const DELIM_DOCTYPE = 'entity.name.tag.html';
-
-function getTag(name: string) {
-	return TAG_PREFIX + name;
-}
-
-const TAG_PREFIX = 'entity.name.tag.tag-';
-
-class HTMLState implements monaco.languages.IState {
-
-    private _state: monaco.languages.IState;
-
-    public scannerState: ScannerState;
-
-    constructor(state: monaco.languages.IState, scannerState: ScannerState) {
-        this._state = state;
-        this.scannerState = scannerState;
-    }
-
-    public clone(): HTMLState {
-        return new HTMLState(this._state, this.scannerState);
-    }
-
-    public equals(other: monaco.languages.IState): boolean {
-        if (other === this) {
-            return true;
-        }
-        if (!other || !(other instanceof HTMLState)) {
-            return false;
-        }
-        return this.scannerState === (<HTMLState>other).scannerState;
-    }
-
-    public getStateData(): monaco.languages.IState {
-        return this._state;
-    }
-
-    public setStateData(state: monaco.languages.IState): void {
-        this._state = state;
-    }
-}
-
-
-function tokenize(line: string, state: HTMLState, offsetDelta: number = 0, stopAtOffset = line.length): monaco.languages.ILineTokens {
-
-    let scanner = createScanner(line, 0, state && state.scannerState);
-    let tokenType = scanner.scan();
-    let ret = {
-        tokens: <monaco.languages.IToken[]>[],
-        endState: state.clone()
-    };
-    let position = -1;
-    while (tokenType !== TokenType.EOS && scanner.getTokenOffset() < stopAtOffset) {
-        let scope;
-        switch (tokenType) {
-            case TokenType.AttributeName:
-                scope = ATTRIB_NAME;
-                break;
-            case TokenType.AttributeValue:
-                scope = ATTRIB_VALUE;
-                break;
-            case TokenType.StartTag:
-            case TokenType.EndTag:
-                scope = getTag(scanner.getTokenText());
-                break;
-            case TokenType.DelimiterAssign:
-                scope = DELIM_ASSIGN;
-                break;
-            case TokenType.StartTagOpen:
-            case TokenType.StartTagClose:
-            case TokenType.StartTagSelfClose:
-                scope = DELIM_START;
-                break;
-            case TokenType.EndTagOpen:
-            case TokenType.EndTagClose:
-                scope = DELIM_END;
-                break;
-            case TokenType.Doctype:
-                scope = DOCTYPE;
-                break;
-            case TokenType.StartDoctypeTag:
-            case TokenType.EndDoctypeTag:
-                scope = DELIM_DOCTYPE;
-                break;
-            case TokenType.Comment:
-                scope = COMMENT;
-                break;
-            case TokenType.StartCommentTag:
-            case TokenType.EndCommentTag:
-                scope = DELIM_COMMENT;
-                break;
-            default:
-                scope = '';
-                break;
-
-        }
-        if (position < scanner.getTokenOffset()) {
-            ret.tokens.push({
-                startIndex: scanner.getTokenOffset() + offsetDelta,
-                scopes: scope
-            });
-        } else {
-            throw new Error('Scanner did not advance, next 3 characters are: ' + line.substr(scanner.getTokenOffset(), 3));
-        }
-        position = scanner.getTokenOffset();
-
-        tokenType = scanner.scan();
-    }
-    ret.endState = new HTMLState(state.getStateData(), scanner.getScannerState());
-
-    return ret;
-}

src/workerManager.ts

@@ -68,7 +68,9 @@ export class WorkerManager {
 				createData: {
 					languageSettings: this._defaults.options,
 					languageId: this._defaults.languageId
-				}
+				},
+
+				label: this._defaults.languageId
 			});
 
 			this._client = this._worker.getProxy();