diff --git a/gulpfile.js b/gulpfile.js index 898e50a7..62ce2045 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -78,7 +78,9 @@ gulp.task('release', ['clean-release','compile'], function() { bundleOne('src/yaml'), bundleOne('src/solidity'), bundleOne('src/sb'), - bundleOne('src/mysql') + bundleOne('src/mysql'), + bundleOne('src/redshift'), + bundleOne('src/pgsql') ) .pipe(uglify({ output: { diff --git a/package.json b/package.json index 535b3176..07de5384 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@teamsqlio/monaco-languages", - "version": "0.9.0", + "version": "0.9.3", "description": "Bundle of many languages for the Monaco Editor.", "scripts": { "compile": "node_modules/.bin/gulp compile", diff --git a/src/monaco.contribution.ts b/src/monaco.contribution.ts index 3b8333ce..70a37a23 100644 --- a/src/monaco.contribution.ts +++ b/src/monaco.contribution.ts @@ -279,3 +279,17 @@ registerLanguage({ aliases: ['MySQL', 'mysql'], module: './mysql' }); + +registerLanguage({ + id: 'pgsql', + extensions: ['.sql'], + aliases: ['PostgreSQL', 'postgres', 'pg', 'postgre'], + module: './pgsql' +}); + +registerLanguage({ + id: 'redshift', + extensions: ['.sql'], + aliases: ['Redshift', 'redshift'], + module: './redshift' +}); diff --git a/src/mysql.ts b/src/mysql.ts index 94f87bad..c1485d24 100644 --- a/src/mysql.ts +++ b/src/mysql.ts @@ -62,9 +62,9 @@ export const language = { "FLOAT8", "FLUSH", "FOLLOWS", "FOR", "FORCE", "FOREIGN", "FORMAT", "FOUND", "FROM", "FULL", "FULLTEXT", "FUNCTION", "GENERAL", "GENERATED", "GEOMETRY", "GEOMETRYCOLLECTION", "GET", "GET_FORMAT", "GLOBAL", "GRANT", "GRANTS", "GROUP", "GROUP_REPLICATION", "HANDLER", "HASH", "HAVING", "HELP", "HIGH_PRIORITY", "HOST", "HOSTS", "HOUR", "HOUR_MICROSECOND", "HOUR_MINUTE", "HOUR_SECOND", "IDENTIFIED", "IF", "IGNORE", "IGNORE_SERVER_IDS", - "IMPORT", "IN", "INDEX", "INDEXES", "INFILE", "INITIAL_SIZE", "INNER", "INOUT", "INSENSITIVE", "INSERT", "INSERT_METHOD", "INSTALL", "INSTANCE", + "IMPORT", "INDEX", "INDEXES", "INFILE", "INITIAL_SIZE", "INNER", "INOUT", "INSENSITIVE", "INSERT", "INSERT_METHOD", "INSTALL", "INSTANCE", "INT", "INT1", "INT2", "INT3", "INT4", "INT8", "INTEGER", "INTERVAL", "INTO", "INVOKER", "IO", "IO_AFTER_GTIDS", "IO_BEFORE_GTIDS", "IO_THREAD", - "IPC", "IS", "ISOLATION", "ISSUER", "ITERATE", "JOIN", "JSON", "KEY", "KEYS", "KEY_BLOCK_SIZE", "KILL", "LANGUAGE", "LAST", "LEADING", "LEAVE", + "IPC", "ISOLATION", "ISSUER", "ITERATE", "JOIN", "JSON", "KEY", "KEYS", "KEY_BLOCK_SIZE", "KILL", "LANGUAGE", "LAST", "LEADING", "LEAVE", "LEAVES", "LEFT", "LESS", "LEVEL", "LIKE", "LIMIT", "LINEAR", "LINES", "LINESTRING", "LIST", "LOAD", "LOCAL", "LOCALTIME", "LOCALTIMESTAMP", "LOCK", "LOCKS", "LOGFILE", "LOGS", "LONG", "LONGBLOB", "LONGTEXT", "LOOP", "LOW_PRIORITY", "MASTER", "MASTER_AUTO_POSITION", "MASTER_BIND", "MASTER_CONNECT_RETRY", "MASTER_DELAY", "MASTER_HEARTBEAT_PERIOD", "MASTER_HOST", "MASTER_LOG_FILE", "MASTER_LOG_POS", "MASTER_PASSWORD", "MASTER_PORT", "MASTER_RETRY_COUNT", @@ -73,7 +73,7 @@ export const language = { "MAX_ROWS", "MAX_SIZE", "MAX_STATEMENT_TIME", "MAX_UPDATES_PER_HOUR", "MAX_USER_CONNECTIONS", "MEDIUM", "MEDIUMBLOB", "MEDIUMINT", "MEDIUMTEXT", "MEMORY", "MERGE", "MESSAGE_TEXT", "MICROSECOND", "MIDDLEINT", "MIGRATE", "MINUTE", "MINUTE_MICROSECOND", "MINUTE_SECOND", "MIN_ROWS", "MOD", "MODE", "MODIFIES", "MODIFY", "MONTH", "MULTILINESTRING", "MULTIPOINT", "MULTIPOLYGON", "MUTEX", "MYSQL_ERRNO", "NAME", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NDB", - "NDBCLUSTER", "NEVER", "NEW", "NEXT", "NO", "NODEGROUP", "NONBLOCKING", "NONE", "NOT", "NO_WAIT", "NO_WRITE_TO_BINLOG", "NULL", "NUMBER", "NUMERIC", + "NDBCLUSTER", "NEVER", "NEW", "NEXT", "NO", "NODEGROUP", "NONBLOCKING", "NONE", "NO_WAIT", "NO_WRITE_TO_BINLOG", "NUMBER", "NUMERIC", "NVARCHAR", "OFFSET", "OLD_PASSWORD", "ON", "ONE", "ONLY", "OPEN", "OPTIMIZE", "OPTIMIZER_COSTS", "OPTION", "OPTIONALLY", "OPTIONS", "OR", "ORDER", "OUT", "OUTER", "OUTFILE", "OWNER", "PACK_KEYS", "PAGE", "PARSER", "PARSE_GCOL_EXPR", "PARTIAL", "PARTITION", "PARTITIONING", "PARTITIONS", "PASSWORD", "PHASE", "PLUGIN", "PLUGINS", "PLUGIN_DIR", "POINT", "POLYGON", "PORT", "PRECEDES", "PRECISION", "PREPARE", "PRESERVE", "PREV", "PRIMARY", "PRIVILEGES", @@ -112,7 +112,7 @@ export const language = { "EXP", "EXPORT_SET", "ExteriorRing", "EXTRACT", "ExtractValue", "FIELD", "FIND_IN_SET", "FLOOR", "FORMAT", "FOUND_ROWS", "FROM_BASE64", "FROM_DAYS", "FROM_UNIXTIME", "GeomCollFromText", "GeometryCollectionFromText", "GeomCollFromWKB", "GeometryCollectionFromWKB", "GeometryCollection", "GeometryN", "GeometryType", "GeomFromText", "GeometryFromText", "GeomFromWKB", "GeometryFromWKB", "GET_FORMAT", - "GET_LOCK", "GLength", "GREATEST", "GROUP_CONCAT", "GTID_SUBSET", "GTID_SUBTRACT", "HEX", "HOUR", "IF", "IFNULL", "IN", "INET_ATON", + "GET_LOCK", "GLength", "GREATEST", "GROUP_CONCAT", "GTID_SUBSET", "GTID_SUBTRACT", "HEX", "HOUR", "IF", "IFNULL", "INET_ATON", "INET_NTOA", "INET6_ATON", "INET6_NTOA", "INSERT", "INSTR", "InteriorRingN", "Intersects", "INTERVAL", "IS_FREE_LOCK", "IS_IPV4", "IS_IPV4_COMPAT", "IS_IPV4_MAPPED", "IS_IPV6", "IS_USED_LOCK", "IsClosed", "IsEmpty", "ISNULL", "IsSimple", "JSON_APPEND", "JSON_ARRAY", "JSON_ARRAY_APPEND", "JSON_ARRAY_INSERT", "JSON_CONTAINS", "JSON_CONTAINS_PATH", "JSON_DEPTH", "JSON_EXTRACT", "JSON_INSERT", "JSON_KEYS", @@ -150,14 +150,10 @@ export const language = { builtinVariables: [ // NOT SUPPORTED ], - pseudoColumns: [ - // NOT SUPPORTED - ], tokenizer: { root: [ { include: '@comments' }, { include: '@whitespace' }, - { include: '@pseudoColumns' }, { include: '@numbers' }, { include: '@strings' }, { include: '@complexIdentifiers' }, @@ -191,14 +187,6 @@ export const language = { [/\*\//, { token: 'comment.quote', next: '@pop' }], [/./, 'comment'] ], - pseudoColumns: [ - [/[$][A-Za-z_][\w@#$]*/, { - cases: { - '@pseudoColumns': 'predefined', - '@default': 'identifier' - } - }], - ], numbers: [ [/0[xX][0-9a-fA-F]*/, 'number'], [/[$][+-]*\d*(\.\d*)?/, 'number'], diff --git a/src/pgsql.ts b/src/pgsql.ts index ed12c8fc..1367bbbd 100644 --- a/src/pgsql.ts +++ b/src/pgsql.ts @@ -211,7 +211,6 @@ export const language = { ], comments: [ [/--+.*/, 'comment'], - [/#+.*/, 'comment'], [/\/\*/, { token: 'comment.quote', next: '@comment' }] ], comment: [ @@ -237,24 +236,19 @@ export const language = { ], strings: [ [/'/, { token: 'string', next: '@string' }], - [/"/, { token: 'string', next: '@string' }] ], string: [ [/[^']+/, 'string'], - [/[^"]+/, 'string'], [/''/, 'string'], - [/""/, 'string'], - [/'/, { token: 'string', next: '@pop' }], - [/"/, { token: 'string', next: '@pop' }] + [/'/, { token: 'string', next: '@pop' }] ], complexIdentifiers: [ - - [/`/, { token: 'identifier.quote', next: '@quotedIdentifier' }] + [/"/, { token: 'identifier.quote', next: '@quotedIdentifier' }] ], quotedIdentifier: [ - [/[^`]+/, 'identifier'], - [/``/, 'identifier'], - [/`/, { token: 'identifier.quote', next: '@pop' }] + [/[^"]+/, 'identifier'], + [/""/, 'identifier'], + [/"/, { token: 'identifier.quote', next: '@pop' }] ], scopes: [ // NOT SUPPORTED diff --git a/src/redshift.ts b/src/redshift.ts index fb878f8d..9fb3dc98 100644 --- a/src/redshift.ts +++ b/src/redshift.ts @@ -43,7 +43,6 @@ export const language = { { open: '[', close: ']', token: 'delimiter.square' }, { open: '(', close: ')', token: 'delimiter.parenthesis' } ], - keywords: [ "AES128", "AES256", "ALL", "ALLOWOVERWRITE", "ANALYSE", "ANALYZE", "AND", "ANY", "ARRAY", "AS", "ASC", "AUTHORIZATION", "BACKUP", "BETWEEN", "BINARY", "BLANKSASNULL", "BOTH", "BYTEDICT", "BZIP2", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", @@ -169,7 +168,6 @@ export const language = { ], comments: [ [/--+.*/, 'comment'], - [/#+.*/, 'comment'], [/\/\*/, { token: 'comment.quote', next: '@comment' }] ], comment: [ @@ -195,24 +193,19 @@ export const language = { ], strings: [ [/'/, { token: 'string', next: '@string' }], - [/"/, { token: 'string', next: '@string' }] ], string: [ [/[^']+/, 'string'], - [/[^"]+/, 'string'], [/''/, 'string'], - [/""/, 'string'], - [/'/, { token: 'string', next: '@pop' }], - [/"/, { token: 'string', next: '@pop' }] + [/'/, { token: 'string', next: '@pop' }] ], complexIdentifiers: [ - - [/`/, { token: 'identifier.quote', next: '@quotedIdentifier' }] + [/"/, { token: 'identifier.quote', next: '@quotedIdentifier' }] ], quotedIdentifier: [ - [/[^`]+/, 'identifier'], - [/``/, 'identifier'], - [/`/, { token: 'identifier.quote', next: '@pop' }] + [/[^"]+/, 'identifier'], + [/""/, 'identifier'], + [/"/, { token: 'identifier.quote', next: '@pop' }] ], scopes: [ // NOT SUPPORTED diff --git a/test/all.js b/test/all.js index 123c4273..e53c5926 100644 --- a/test/all.js +++ b/test/all.js @@ -59,7 +59,9 @@ requirejs([ 'out/test/yaml.test', 'out/test/solidity.test', 'out/test/sb.test', - 'out/test/mysql.test' + 'out/test/mysql.test', + 'out/test/pgsql.test', + 'out/test/redshift.test' ], function() { run(); // We can launch the tests! }); diff --git a/test/mysql.test.ts b/test/mysql.test.ts index b832ff18..0faf6c30 100644 --- a/test/mysql.test.ts +++ b/test/mysql.test.ts @@ -513,22 +513,22 @@ testTokenization('mysql', [ }], [{ - line: 'WHERE x IS NOT NULL', + line: 'WHERE myfield IS NOT NULL', tokens: [ { startIndex: 0, type: 'keyword.sql' }, { startIndex: 5, type: 'white.sql' }, { startIndex: 6, type: 'identifier.sql' }, - { startIndex: 7, type: 'white.sql' }, - { startIndex: 8, type: 'operator.sql' }, - { startIndex: 10, type: 'white.sql' }, - { startIndex: 11, type: 'operator.sql' }, - { startIndex: 14, type: 'white.sql' }, - { startIndex: 15, type: 'operator.sql' } + { startIndex: 13, type: 'white.sql' }, + { startIndex: 14, type: 'operator.sql' }, + { startIndex: 16, type: 'white.sql' }, + { startIndex: 17, type: 'operator.sql' }, + { startIndex: 20, type: 'white.sql' }, + { startIndex: 21, type: 'operator.sql' } ] }], [{ - line: 'SELECT * FROM MyTable WHERE MyColumn IN (1,2)', + line: 'SELECT * FROM tbl WHERE MyColumn IN (1,2)', tokens: [ { startIndex: 0, type: 'keyword.sql' }, { startIndex: 6, type: 'white.sql' }, @@ -537,21 +537,18 @@ testTokenization('mysql', [ { startIndex: 9, type: 'keyword.sql' }, { startIndex: 13, type: 'white.sql' }, { startIndex: 14, type: 'identifier.sql' }, - { startIndex: 17, type: 'delimiter.sql' }, - { startIndex: 18, type: 'identifier.sql' }, - { startIndex: 25, type: 'white.sql' }, - { startIndex: 26, type: 'keyword.sql' }, - { startIndex: 31, type: 'white.sql' }, - { startIndex: 32, type: 'identifier.sql' }, - { startIndex: 40, type: 'white.sql' }, - { startIndex: 41, type: 'operator.sql' }, - { startIndex: 43, type: 'white.sql' }, - { startIndex: 44, type: 'delimiter.parenthesis.sql' }, - { startIndex: 45, type: 'number.sql' }, - { startIndex: 46, type: 'delimiter.sql' }, - { startIndex: 47, type: 'number.sql' }, - { startIndex: 48, type: 'delimiter.parenthesis.sql' } + { startIndex: 17, type: 'white.sql' }, + { startIndex: 18, type: 'keyword.sql' }, + { startIndex: 23, type: 'white.sql' }, + { startIndex: 24, type: 'identifier.sql' }, + { startIndex: 32, type: 'white.sql' }, + { startIndex: 33, type: 'operator.sql' }, + { startIndex: 35, type: 'white.sql' }, + { startIndex: 36, type: 'delimiter.parenthesis.sql' }, + { startIndex: 37, type: 'number.sql' }, + { startIndex: 38, type: 'delimiter.sql' }, + { startIndex: 39, type: 'number.sql' }, + { startIndex: 40, type: 'delimiter.parenthesis.sql' } ] }] - ]); diff --git a/test/pgsql.test.ts b/test/pgsql.test.ts new file mode 100644 index 00000000..d8958603 --- /dev/null +++ b/test/pgsql.test.ts @@ -0,0 +1,555 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { testTokenization } from './testRunner'; + +testTokenization('sql', [ + // Comments + [{ + line: '-- a comment', + tokens: [ + { startIndex: 0, type: 'comment.sql' } + ] + }], + + [{ + line: '---sticky -- comment', + tokens: [ + { startIndex: 0, type: 'comment.sql' } + ] + }], + + [{ + line: '-almost a comment', + tokens: [ + { startIndex: 0, type: 'operator.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.sql' }, + { startIndex: 9, type: 'white.sql' }, + { startIndex: 10, type: 'identifier.sql' } + ] + }], + + [{ + line: '/* a full line comment */', + tokens: [ + { startIndex: 0, type: 'comment.quote.sql' }, + { startIndex: 2, type: 'comment.sql' }, + { startIndex: 23, type: 'comment.quote.sql' } + ] + }], + + [{ + line: '/* /// *** /// */', + tokens: [ + { startIndex: 0, type: 'comment.quote.sql' }, + { startIndex: 2, type: 'comment.sql' }, + { startIndex: 15, type: 'comment.quote.sql' } + ] + }], + + [{ + line: 'declare _x int = /* a simple comment */ 1;', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.sql' }, + { startIndex: 10, type: 'white.sql' }, + { startIndex: 11, type: 'keyword.sql' }, + { startIndex: 14, type: 'white.sql' }, + { startIndex: 15, type: 'operator.sql' }, + { startIndex: 16, type: 'white.sql' }, + { startIndex: 17, type: 'comment.quote.sql' }, + { startIndex: 19, type: 'comment.sql' }, + { startIndex: 37, type: 'comment.quote.sql' }, + { startIndex: 39, type: 'white.sql' }, + { startIndex: 40, type: 'number.sql' }, + { startIndex: 41, type: 'delimiter.sql' } + ] + }], + + // Not supporting nested comments, as nested comments seem to not be standard? + // i.e. http://stackoverflow.com/questions/728172/are-there-multiline-comment-delimiters-in-sql-that-are-vendor-agnostic + [{ + line: '_x=/* a /* nested comment 1*/;', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 3, type: 'comment.quote.sql' }, + { startIndex: 5, type: 'comment.sql' }, + { startIndex: 28, type: 'comment.quote.sql' }, + { startIndex: 30, type: 'delimiter.sql' } + ] + }], + + [{ + line: '_x=/* another comment */ 1*/;', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 3, type: 'comment.quote.sql' }, + { startIndex: 5, type: 'comment.sql' }, + { startIndex: 22, type: 'comment.quote.sql' }, + { startIndex: 24, type: 'white.sql' }, + { startIndex: 25, type: 'number.sql' }, + { startIndex: 26, type: 'operator.sql' }, + { startIndex: 28, type: 'delimiter.sql' } + ] + }], + + [{ + line: '_x=/*/;', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 3, type: 'comment.quote.sql' }, + { startIndex: 5, type: 'comment.sql' } + ] + }], + + // Numbers + [{ + line: '123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '-123', + tokens: [ + { startIndex: 0, type: 'operator.sql' }, + { startIndex: 1, type: 'number.sql' } + ] + }], + + [{ + line: '0xaBc123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0XaBc123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0x', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0x0', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0xAB_CD', + tokens: [ + { startIndex: 0, type: 'number.sql' }, + { startIndex: 4, type: 'identifier.sql' } + ] + }], + + [{ + line: '$', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$-123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$-+-123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$123.5678', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$0.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$99.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$0.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$.0', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.', + tokens: [ + { startIndex: 0, type: 'delimiter.sql' } + ] + }], + + [{ + line: '123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '123.5678', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '99.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.0', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1E-2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1E+2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0.1E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1.E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.1E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + // Identifiers + [{ + line: '_abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '#abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '##abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '@abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '@@abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '$abc', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '$nonexistent', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '@@nonexistent', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: 'declare "abc 321";', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.quote.sql' }, + { startIndex: 9, type: 'identifier.sql' }, + { startIndex: 16, type: 'identifier.quote.sql' }, + { startIndex: 17, type: 'delimiter.sql' } + ] + }], + + [{ + line: '"abc"" 321 "" xyz"', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 17, type: 'identifier.quote.sql' } + ] + }], + + [{ + line: '"abc', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' } + ] + }], + + [{ + line: 'declare "abc 321";', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.quote.sql' }, + { startIndex: 9, type: 'identifier.sql' }, + { startIndex: 16, type: 'identifier.quote.sql' }, + { startIndex: 17, type: 'delimiter.sql' } + ] + }], + + [{ + line: '"abc"" 321 "" xyz"', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 17, type: 'identifier.quote.sql' } + ] + }], + + [{ + line: '"abc', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' } + ] + }], + + [{ + line: 'int', + tokens: [ + { startIndex: 0, type: 'keyword.sql' } + ] + }], + + [{ + line: '"int"', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 4, type: 'identifier.quote.sql' } + ] + }], + + // Strings + [{ + line: 'declare _x=\'a string\';', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.sql' }, + { startIndex: 10, type: 'operator.sql' }, + { startIndex: 11, type: 'string.sql' }, + { startIndex: 21, type: 'delimiter.sql' } + ] + }], + + [{ + line: '\'a \'\' string with quotes\'', + tokens: [ + { startIndex: 0, type: 'string.sql' }, + ] + }], + + [{ + line: '\'a -- string with comment\'', + tokens: [ + { startIndex: 0, type: 'string.sql' }, + ] + }], + + [{ + line: '\'a endless string', + tokens: [ + { startIndex: 0, type: 'string.sql' }, + ] + }], + + // Operators + [{ + line: 'x=x+1', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 1, type: 'operator.sql' }, + { startIndex: 2, type: 'identifier.sql' }, + { startIndex: 3, type: 'operator.sql' }, + { startIndex: 4, type: 'number.sql' } + ] + }], + + [{ + line: '_x^=_x', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 4, type: 'identifier.sql' } + ] + }], + + [{ + line: 'WHERE x IS NOT NULL', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 5, type: 'white.sql' }, + { startIndex: 6, type: 'identifier.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'operator.sql' }, + { startIndex: 10, type: 'white.sql' }, + { startIndex: 11, type: 'operator.sql' }, + { startIndex: 14, type: 'white.sql' }, + { startIndex: 15, type: 'operator.sql' } + ] + }], + + [{ + line: 'SELECT * FROM sch.MyTable WHERE MyColumn IN (1,2)', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 6, type: 'white.sql' }, + { startIndex: 7, type: 'operator.sql' }, + { startIndex: 8, type: 'white.sql' }, + { startIndex: 9, type: 'keyword.sql' }, + { startIndex: 13, type: 'white.sql' }, + { startIndex: 14, type: 'identifier.sql' }, + { startIndex: 17, type: 'delimiter.sql' }, + { startIndex: 18, type: 'identifier.sql' }, + { startIndex: 25, type: 'white.sql' }, + { startIndex: 26, type: 'keyword.sql' }, + { startIndex: 31, type: 'white.sql' }, + { startIndex: 32, type: 'identifier.sql' }, + { startIndex: 40, type: 'white.sql' }, + { startIndex: 41, type: 'operator.sql' }, + { startIndex: 43, type: 'white.sql' }, + { startIndex: 44, type: 'delimiter.parenthesis.sql' }, + { startIndex: 45, type: 'number.sql' }, + { startIndex: 46, type: 'delimiter.sql' }, + { startIndex: 47, type: 'number.sql' }, + { startIndex: 48, type: 'delimiter.parenthesis.sql' } + ] + }] +]); diff --git a/test/redshift.test.ts b/test/redshift.test.ts new file mode 100644 index 00000000..d8958603 --- /dev/null +++ b/test/redshift.test.ts @@ -0,0 +1,555 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { testTokenization } from './testRunner'; + +testTokenization('sql', [ + // Comments + [{ + line: '-- a comment', + tokens: [ + { startIndex: 0, type: 'comment.sql' } + ] + }], + + [{ + line: '---sticky -- comment', + tokens: [ + { startIndex: 0, type: 'comment.sql' } + ] + }], + + [{ + line: '-almost a comment', + tokens: [ + { startIndex: 0, type: 'operator.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.sql' }, + { startIndex: 9, type: 'white.sql' }, + { startIndex: 10, type: 'identifier.sql' } + ] + }], + + [{ + line: '/* a full line comment */', + tokens: [ + { startIndex: 0, type: 'comment.quote.sql' }, + { startIndex: 2, type: 'comment.sql' }, + { startIndex: 23, type: 'comment.quote.sql' } + ] + }], + + [{ + line: '/* /// *** /// */', + tokens: [ + { startIndex: 0, type: 'comment.quote.sql' }, + { startIndex: 2, type: 'comment.sql' }, + { startIndex: 15, type: 'comment.quote.sql' } + ] + }], + + [{ + line: 'declare _x int = /* a simple comment */ 1;', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.sql' }, + { startIndex: 10, type: 'white.sql' }, + { startIndex: 11, type: 'keyword.sql' }, + { startIndex: 14, type: 'white.sql' }, + { startIndex: 15, type: 'operator.sql' }, + { startIndex: 16, type: 'white.sql' }, + { startIndex: 17, type: 'comment.quote.sql' }, + { startIndex: 19, type: 'comment.sql' }, + { startIndex: 37, type: 'comment.quote.sql' }, + { startIndex: 39, type: 'white.sql' }, + { startIndex: 40, type: 'number.sql' }, + { startIndex: 41, type: 'delimiter.sql' } + ] + }], + + // Not supporting nested comments, as nested comments seem to not be standard? + // i.e. http://stackoverflow.com/questions/728172/are-there-multiline-comment-delimiters-in-sql-that-are-vendor-agnostic + [{ + line: '_x=/* a /* nested comment 1*/;', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 3, type: 'comment.quote.sql' }, + { startIndex: 5, type: 'comment.sql' }, + { startIndex: 28, type: 'comment.quote.sql' }, + { startIndex: 30, type: 'delimiter.sql' } + ] + }], + + [{ + line: '_x=/* another comment */ 1*/;', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 3, type: 'comment.quote.sql' }, + { startIndex: 5, type: 'comment.sql' }, + { startIndex: 22, type: 'comment.quote.sql' }, + { startIndex: 24, type: 'white.sql' }, + { startIndex: 25, type: 'number.sql' }, + { startIndex: 26, type: 'operator.sql' }, + { startIndex: 28, type: 'delimiter.sql' } + ] + }], + + [{ + line: '_x=/*/;', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 3, type: 'comment.quote.sql' }, + { startIndex: 5, type: 'comment.sql' } + ] + }], + + // Numbers + [{ + line: '123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '-123', + tokens: [ + { startIndex: 0, type: 'operator.sql' }, + { startIndex: 1, type: 'number.sql' } + ] + }], + + [{ + line: '0xaBc123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0XaBc123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0x', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0x0', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0xAB_CD', + tokens: [ + { startIndex: 0, type: 'number.sql' }, + { startIndex: 4, type: 'identifier.sql' } + ] + }], + + [{ + line: '$', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$-123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$-+-123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$123.5678', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$0.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$99.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$0.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '$.0', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.', + tokens: [ + { startIndex: 0, type: 'delimiter.sql' } + ] + }], + + [{ + line: '123', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '123.5678', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.99', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '99.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0.', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.0', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1E-2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1E+2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '0.1E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '1.E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + [{ + line: '.1E2', + tokens: [ + { startIndex: 0, type: 'number.sql' } + ] + }], + + // Identifiers + [{ + line: '_abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '#abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '##abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '@abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '@@abc$01', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '$abc', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '$nonexistent', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: '@@nonexistent', + tokens: [ + { startIndex: 0, type: 'identifier.sql' } + ] + }], + + [{ + line: 'declare "abc 321";', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.quote.sql' }, + { startIndex: 9, type: 'identifier.sql' }, + { startIndex: 16, type: 'identifier.quote.sql' }, + { startIndex: 17, type: 'delimiter.sql' } + ] + }], + + [{ + line: '"abc"" 321 "" xyz"', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 17, type: 'identifier.quote.sql' } + ] + }], + + [{ + line: '"abc', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' } + ] + }], + + [{ + line: 'declare "abc 321";', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.quote.sql' }, + { startIndex: 9, type: 'identifier.sql' }, + { startIndex: 16, type: 'identifier.quote.sql' }, + { startIndex: 17, type: 'delimiter.sql' } + ] + }], + + [{ + line: '"abc"" 321 "" xyz"', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 17, type: 'identifier.quote.sql' } + ] + }], + + [{ + line: '"abc', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' } + ] + }], + + [{ + line: 'int', + tokens: [ + { startIndex: 0, type: 'keyword.sql' } + ] + }], + + [{ + line: '"int"', + tokens: [ + { startIndex: 0, type: 'identifier.quote.sql' }, + { startIndex: 1, type: 'identifier.sql' }, + { startIndex: 4, type: 'identifier.quote.sql' } + ] + }], + + // Strings + [{ + line: 'declare _x=\'a string\';', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'identifier.sql' }, + { startIndex: 10, type: 'operator.sql' }, + { startIndex: 11, type: 'string.sql' }, + { startIndex: 21, type: 'delimiter.sql' } + ] + }], + + [{ + line: '\'a \'\' string with quotes\'', + tokens: [ + { startIndex: 0, type: 'string.sql' }, + ] + }], + + [{ + line: '\'a -- string with comment\'', + tokens: [ + { startIndex: 0, type: 'string.sql' }, + ] + }], + + [{ + line: '\'a endless string', + tokens: [ + { startIndex: 0, type: 'string.sql' }, + ] + }], + + // Operators + [{ + line: 'x=x+1', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 1, type: 'operator.sql' }, + { startIndex: 2, type: 'identifier.sql' }, + { startIndex: 3, type: 'operator.sql' }, + { startIndex: 4, type: 'number.sql' } + ] + }], + + [{ + line: '_x^=_x', + tokens: [ + { startIndex: 0, type: 'identifier.sql' }, + { startIndex: 2, type: 'operator.sql' }, + { startIndex: 4, type: 'identifier.sql' } + ] + }], + + [{ + line: 'WHERE x IS NOT NULL', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 5, type: 'white.sql' }, + { startIndex: 6, type: 'identifier.sql' }, + { startIndex: 7, type: 'white.sql' }, + { startIndex: 8, type: 'operator.sql' }, + { startIndex: 10, type: 'white.sql' }, + { startIndex: 11, type: 'operator.sql' }, + { startIndex: 14, type: 'white.sql' }, + { startIndex: 15, type: 'operator.sql' } + ] + }], + + [{ + line: 'SELECT * FROM sch.MyTable WHERE MyColumn IN (1,2)', + tokens: [ + { startIndex: 0, type: 'keyword.sql' }, + { startIndex: 6, type: 'white.sql' }, + { startIndex: 7, type: 'operator.sql' }, + { startIndex: 8, type: 'white.sql' }, + { startIndex: 9, type: 'keyword.sql' }, + { startIndex: 13, type: 'white.sql' }, + { startIndex: 14, type: 'identifier.sql' }, + { startIndex: 17, type: 'delimiter.sql' }, + { startIndex: 18, type: 'identifier.sql' }, + { startIndex: 25, type: 'white.sql' }, + { startIndex: 26, type: 'keyword.sql' }, + { startIndex: 31, type: 'white.sql' }, + { startIndex: 32, type: 'identifier.sql' }, + { startIndex: 40, type: 'white.sql' }, + { startIndex: 41, type: 'operator.sql' }, + { startIndex: 43, type: 'white.sql' }, + { startIndex: 44, type: 'delimiter.parenthesis.sql' }, + { startIndex: 45, type: 'number.sql' }, + { startIndex: 46, type: 'delimiter.sql' }, + { startIndex: 47, type: 'number.sql' }, + { startIndex: 48, type: 'delimiter.parenthesis.sql' } + ] + }] +]);