mirror of
https://github.com/microsoft/monaco-editor.git
synced 2025-12-22 10:25:42 +01:00
434 lines
17 KiB
TypeScript
434 lines
17 KiB
TypeScript
/*---------------------------------------------------------------------------------------------
|
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
*--------------------------------------------------------------------------------------------*/
|
|
|
|
import { testTokenization } from '../test/testRunner';
|
|
|
|
testTokenization('jinja', [
|
|
// Comments
|
|
[
|
|
{
|
|
line: '{# This is a comment #}',
|
|
tokens: [{ startIndex: 0, type: 'comment.block.jinja' }]
|
|
},
|
|
{
|
|
line: 'Some text {#- comment -#} More text',
|
|
tokens: [
|
|
{ startIndex: 0, type: '' }, // Some text
|
|
{ startIndex: 10, type: 'comment.block.jinja' }, // {#- comment -#}
|
|
{ startIndex: 25, type: '' } // More text (Adjusted expectation)
|
|
]
|
|
}
|
|
],
|
|
|
|
// Variables
|
|
[
|
|
{
|
|
line: '{{ variable_name }}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'variable.other.jinja' }, // variable_name
|
|
{ startIndex: 16, type: 'white.jinja' }, // Corrected index
|
|
{ startIndex: 17, type: 'delimiter.variable.jinja' } // Corrected index
|
|
]
|
|
},
|
|
{
|
|
line: '{{- variable | filter -}}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.variable.jinja' }, // {{-
|
|
{ startIndex: 3, type: 'white.jinja' },
|
|
{ startIndex: 4, type: 'variable.other.jinja' }, // variable
|
|
{ startIndex: 12, type: 'white.jinja' },
|
|
{ startIndex: 13, type: 'operators.filter.jinja' }, // |
|
|
{ startIndex: 14, type: 'white.jinja' },
|
|
{ startIndex: 15, type: 'variable.other.filter.jinja' }, // filter
|
|
{ startIndex: 21, type: 'white.jinja' },
|
|
{ startIndex: 22, type: 'delimiter.variable.jinja' } // -}}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Blocks
|
|
[
|
|
{
|
|
line: '{% if condition %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // if
|
|
{ startIndex: 5, type: 'white.jinja' },
|
|
{ startIndex: 6, type: 'variable.other.jinja' }, // condition
|
|
{ startIndex: 15, type: 'white.jinja' },
|
|
{ startIndex: 16, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
},
|
|
{
|
|
line: '{% set my_var = "value" %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // set
|
|
{ startIndex: 6, type: 'white.jinja' },
|
|
{ startIndex: 7, type: 'variable.other.jinja' }, // my_var
|
|
{ startIndex: 13, type: 'white.jinja' },
|
|
{ startIndex: 14, type: 'keyword.operator.jinja' }, // =
|
|
{ startIndex: 15, type: 'white.jinja' },
|
|
{ startIndex: 16, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 17, type: 'string.jinja' }, // value
|
|
{ startIndex: 22, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 23, type: 'white.jinja' },
|
|
{ startIndex: 24, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Raw Block
|
|
[
|
|
{
|
|
line: '{% raw %}This {{ is not processed }} {% endraw %}',
|
|
tokens: [
|
|
// Actual tokens produced by the simpler tokenizer rules:
|
|
// Adjusted to match actual output reported by test runner
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // raw
|
|
{ startIndex: 6, type: 'delimiter.tag.jinja' }, // %}
|
|
{ startIndex: 9, type: 'comment.block.raw.jinja' }, // This {{ is not processed }}
|
|
{ startIndex: 37, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 40, type: 'keyword.control.jinja' }, // endraw
|
|
{ startIndex: 46, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Strings and Numbers within expressions
|
|
[
|
|
{
|
|
line: "{{ 'string' + 123 }}",
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'string.quote.single.jinja' }, // '
|
|
{ startIndex: 4, type: 'string.jinja' }, // string
|
|
{ startIndex: 10, type: 'string.quote.single.jinja' }, // '
|
|
{ startIndex: 11, type: 'white.jinja' },
|
|
{ startIndex: 12, type: 'keyword.operator.jinja' }, // +
|
|
{ startIndex: 13, type: 'white.jinja' },
|
|
{ startIndex: 14, type: 'number.jinja' }, // 123
|
|
{ startIndex: 17, type: 'white.jinja' },
|
|
{ startIndex: 18, type: 'delimiter.variable.jinja' } // }}
|
|
]
|
|
}
|
|
],
|
|
|
|
// For loop with loop variable and else
|
|
[
|
|
{
|
|
line: '{% for item in items %}{{ loop.index }}: {{ item }}{% else %}No items.{% endfor %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // for
|
|
{ startIndex: 6, type: 'white.jinja' },
|
|
{ startIndex: 7, type: 'variable.other.jinja' }, // item
|
|
{ startIndex: 11, type: 'white.jinja' },
|
|
{ startIndex: 12, type: 'keyword.control.jinja' }, // in
|
|
{ startIndex: 14, type: 'white.jinja' },
|
|
{ startIndex: 15, type: 'variable.other.jinja' }, // items
|
|
{ startIndex: 20, type: 'white.jinja' },
|
|
{ startIndex: 21, type: 'delimiter.tag.jinja' }, // %}
|
|
{ startIndex: 23, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 25, type: 'white.jinja' },
|
|
{ startIndex: 26, type: 'variable.language.jinja' }, // loop
|
|
{ startIndex: 30, type: 'delimiter.accessor.jinja' }, // .
|
|
{ startIndex: 31, type: 'variable.other.jinja' }, // index
|
|
{ startIndex: 36, type: 'white.jinja' },
|
|
{ startIndex: 37, type: 'delimiter.variable.jinja' }, // }}
|
|
{ startIndex: 39, type: '' }, // ': ' (colon and space are plain text)
|
|
{ startIndex: 41, type: 'delimiter.variable.jinja' }, // {{ (starts at index 41 now)
|
|
{ startIndex: 43, type: 'white.jinja' },
|
|
{ startIndex: 44, type: 'variable.other.jinja' }, // item
|
|
{ startIndex: 48, type: 'white.jinja' },
|
|
{ startIndex: 49, type: 'delimiter.variable.jinja' }, // }}
|
|
{ startIndex: 51, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 53, type: 'white.jinja' },
|
|
{ startIndex: 54, type: 'keyword.control.jinja' }, // else
|
|
{ startIndex: 58, type: 'white.jinja' },
|
|
{ startIndex: 59, type: 'delimiter.tag.jinja' }, // %}
|
|
{ startIndex: 61, type: '' }, // No items.
|
|
{ startIndex: 70, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 72, type: 'white.jinja' },
|
|
{ startIndex: 73, type: 'keyword.control.jinja' }, // endfor
|
|
{ startIndex: 79, type: 'white.jinja' },
|
|
{ startIndex: 80, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Complex Expressions: attr access, subscript, func call, comparison, logic, test
|
|
[
|
|
{
|
|
line: "{{ obj.attr + my_dict['key'] | func(1 > 0 and not False) is defined }}",
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'variable.other.jinja' }, // obj
|
|
{ startIndex: 6, type: 'delimiter.accessor.jinja' }, // .
|
|
{ startIndex: 7, type: 'variable.other.jinja' }, // attr
|
|
{ startIndex: 11, type: 'white.jinja' },
|
|
{ startIndex: 12, type: 'keyword.operator.jinja' }, // +
|
|
{ startIndex: 13, type: 'white.jinja' },
|
|
{ startIndex: 14, type: 'variable.other.jinja' }, // my_dict
|
|
{ startIndex: 21, type: 'delimiter.jinja' }, // [
|
|
{ startIndex: 22, type: 'string.quote.single.jinja' }, // '
|
|
{ startIndex: 23, type: 'string.jinja' }, // key
|
|
{ startIndex: 26, type: 'string.quote.single.jinja' }, // '
|
|
{ startIndex: 27, type: 'delimiter.jinja' }, // ]
|
|
{ startIndex: 28, type: 'white.jinja' },
|
|
{ startIndex: 29, type: 'operators.filter.jinja' }, // |
|
|
{ startIndex: 30, type: 'white.jinja' },
|
|
{ startIndex: 31, type: 'variable.other.filter.jinja' }, // func
|
|
{ startIndex: 35, type: 'delimiter.jinja' }, // (
|
|
{ startIndex: 36, type: 'number.jinja' }, // 1
|
|
{ startIndex: 37, type: 'white.jinja' },
|
|
{ startIndex: 38, type: 'keyword.operator.jinja' }, // >
|
|
{ startIndex: 39, type: 'white.jinja' },
|
|
{ startIndex: 40, type: 'number.jinja' }, // 0
|
|
{ startIndex: 41, type: 'white.jinja' },
|
|
{ startIndex: 42, type: 'keyword.control.jinja' }, // and
|
|
{ startIndex: 45, type: 'white.jinja' },
|
|
{ startIndex: 46, type: 'keyword.control.jinja' }, // not
|
|
{ startIndex: 49, type: 'white.jinja' },
|
|
{ startIndex: 50, type: 'constant.language.jinja' }, // False
|
|
{ startIndex: 55, type: 'delimiter.jinja' }, // )
|
|
{ startIndex: 56, type: 'white.jinja' },
|
|
{ startIndex: 57, type: 'keyword.control.jinja' }, // is
|
|
{ startIndex: 59, type: 'white.jinja' },
|
|
{ startIndex: 60, type: 'variable.other.jinja' }, // defined (common test treated as variable here, which is acceptable)
|
|
{ startIndex: 67, type: 'white.jinja' },
|
|
{ startIndex: 68, type: 'delimiter.variable.jinja' } // }}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Block and Extends
|
|
[
|
|
{
|
|
line: '{% extends "base.html" %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // extends
|
|
{ startIndex: 10, type: 'white.jinja' },
|
|
{ startIndex: 11, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 12, type: 'string.jinja' }, // base.html
|
|
{ startIndex: 21, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 22, type: 'white.jinja' },
|
|
{ startIndex: 23, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
},
|
|
{
|
|
line: '{% block content %} Content {% endblock %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // block
|
|
{ startIndex: 8, type: 'white.jinja' },
|
|
{ startIndex: 9, type: 'variable.other.jinja' }, // content
|
|
{ startIndex: 16, type: 'white.jinja' },
|
|
{ startIndex: 17, type: 'delimiter.tag.jinja' }, // %}
|
|
{ startIndex: 19, type: '' }, // Content
|
|
{ startIndex: 28, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 30, type: 'white.jinja' },
|
|
{ startIndex: 31, type: 'keyword.control.jinja' }, // endblock
|
|
{ startIndex: 39, type: 'white.jinja' },
|
|
{ startIndex: 40, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Macro definition and call
|
|
[
|
|
{
|
|
line: '{% macro input(name, value) %}<input name="{{ name }}">{% endmacro %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // macro
|
|
{ startIndex: 8, type: 'white.jinja' },
|
|
{ startIndex: 9, type: 'variable.other.jinja' }, // input
|
|
{ startIndex: 14, type: 'delimiter.jinja' }, // (
|
|
{ startIndex: 15, type: 'variable.other.jinja' }, // name
|
|
{ startIndex: 19, type: 'delimiter.jinja' }, // ,
|
|
{ startIndex: 20, type: 'white.jinja' },
|
|
{ startIndex: 21, type: 'variable.other.jinja' }, // value
|
|
{ startIndex: 26, type: 'delimiter.jinja' }, // )
|
|
{ startIndex: 27, type: 'white.jinja' },
|
|
{ startIndex: 28, type: 'delimiter.tag.jinja' }, // %}
|
|
{ startIndex: 30, type: '' }, // <input name="
|
|
{ startIndex: 43, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 45, type: 'white.jinja' },
|
|
{ startIndex: 46, type: 'variable.other.jinja' }, // name
|
|
{ startIndex: 50, type: 'white.jinja' },
|
|
{ startIndex: 51, type: 'delimiter.variable.jinja' }, // }}
|
|
{ startIndex: 53, type: '' }, // ">
|
|
{ startIndex: 55, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 57, type: 'white.jinja' },
|
|
{ startIndex: 58, type: 'keyword.control.jinja' }, // endmacro
|
|
{ startIndex: 66, type: 'white.jinja' },
|
|
{ startIndex: 67, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
},
|
|
{
|
|
line: '{{ mymacros.input("user") }}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'variable.other.jinja' }, // mymacros
|
|
{ startIndex: 11, type: 'delimiter.accessor.jinja' }, // .
|
|
{ startIndex: 12, type: 'variable.other.jinja' }, // input
|
|
{ startIndex: 17, type: 'delimiter.jinja' }, // (
|
|
{ startIndex: 18, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 19, type: 'string.jinja' }, // user
|
|
{ startIndex: 23, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 24, type: 'delimiter.jinja' }, // )
|
|
{ startIndex: 25, type: 'white.jinja' },
|
|
{ startIndex: 26, type: 'delimiter.variable.jinja' } // }}
|
|
]
|
|
}
|
|
],
|
|
|
|
// String Escapes
|
|
[
|
|
{
|
|
// Test escapes on a single line
|
|
line: '{{ "World \\"Quote\\" \\\\ Backslash" }}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 4, type: 'string.jinja' }, // World
|
|
{ startIndex: 10, type: 'constant.character.escape.jinja' }, // \"
|
|
{ startIndex: 12, type: 'string.jinja' }, // Quote
|
|
{ startIndex: 17, type: 'constant.character.escape.jinja' }, // \"
|
|
{ startIndex: 19, type: 'string.jinja' }, //
|
|
{ startIndex: 20, type: 'constant.character.escape.jinja' }, // \\
|
|
{ startIndex: 22, type: 'string.jinja' }, // Backslash
|
|
{ startIndex: 32, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 33, type: 'white.jinja' },
|
|
{ startIndex: 34, type: 'delimiter.variable.jinja' } // }}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Constants
|
|
[
|
|
{
|
|
line: '{{ True and false or None }}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.variable.jinja' },
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'constant.language.jinja' },
|
|
{ startIndex: 7, type: 'white.jinja' },
|
|
{ startIndex: 8, type: 'keyword.control.jinja' },
|
|
{ startIndex: 11, type: 'white.jinja' },
|
|
{ startIndex: 12, type: 'constant.language.jinja' },
|
|
{ startIndex: 17, type: 'white.jinja' },
|
|
{ startIndex: 18, type: 'keyword.control.jinja' },
|
|
{ startIndex: 20, type: 'white.jinja' },
|
|
{ startIndex: 21, type: 'constant.language.jinja' },
|
|
{ startIndex: 25, type: 'white.jinja' },
|
|
{ startIndex: 26, type: 'delimiter.variable.jinja' }
|
|
]
|
|
}
|
|
],
|
|
|
|
// Filter block
|
|
[
|
|
{
|
|
line: '{% filter upper %}Text{% endfilter %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // filter
|
|
{ startIndex: 9, type: 'white.jinja' },
|
|
{ startIndex: 10, type: 'variable.other.jinja' }, // upper (filter name treated as variable here)
|
|
{ startIndex: 15, type: 'white.jinja' },
|
|
{ startIndex: 16, type: 'delimiter.tag.jinja' }, // %}
|
|
{ startIndex: 18, type: '' }, // Text
|
|
{ startIndex: 22, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 24, type: 'white.jinja' },
|
|
{ startIndex: 25, type: 'keyword.control.jinja' }, // endfilter
|
|
{ startIndex: 34, type: 'white.jinja' },
|
|
{ startIndex: 35, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
}
|
|
],
|
|
|
|
// Include and Import
|
|
[
|
|
{
|
|
line: '{% include "partial.html" %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // include
|
|
{ startIndex: 10, type: 'white.jinja' },
|
|
{ startIndex: 11, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 12, type: 'string.jinja' }, // partial.html
|
|
{ startIndex: 24, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 25, type: 'white.jinja' },
|
|
{ startIndex: 26, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
},
|
|
{
|
|
line: '{% import "macros.jinja" as forms %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // import
|
|
{ startIndex: 9, type: 'white.jinja' },
|
|
{ startIndex: 10, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 11, type: 'string.jinja' }, // macros.jinja
|
|
{ startIndex: 23, type: 'string.quote.double.jinja' }, // "
|
|
{ startIndex: 24, type: 'white.jinja' },
|
|
{ startIndex: 25, type: 'keyword.control.jinja' }, // as
|
|
{ startIndex: 27, type: 'white.jinja' },
|
|
{ startIndex: 28, type: 'variable.other.jinja' }, // forms
|
|
{ startIndex: 33, type: 'white.jinja' },
|
|
{ startIndex: 34, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
}
|
|
],
|
|
|
|
// With block
|
|
[
|
|
{
|
|
line: '{% with var = 42 %}{{ var }}{% endwith %}',
|
|
tokens: [
|
|
{ startIndex: 0, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 2, type: 'white.jinja' },
|
|
{ startIndex: 3, type: 'keyword.control.jinja' }, // with
|
|
{ startIndex: 7, type: 'white.jinja' },
|
|
{ startIndex: 8, type: 'variable.other.jinja' }, // var
|
|
{ startIndex: 11, type: 'white.jinja' },
|
|
{ startIndex: 12, type: 'keyword.operator.jinja' }, // =
|
|
{ startIndex: 13, type: 'white.jinja' },
|
|
{ startIndex: 14, type: 'number.jinja' }, // 42
|
|
{ startIndex: 16, type: 'white.jinja' },
|
|
{ startIndex: 17, type: 'delimiter.tag.jinja' }, // %}
|
|
{ startIndex: 19, type: 'delimiter.variable.jinja' }, // {{
|
|
{ startIndex: 21, type: 'white.jinja' },
|
|
{ startIndex: 22, type: 'variable.other.jinja' }, // var
|
|
{ startIndex: 25, type: 'white.jinja' },
|
|
{ startIndex: 26, type: 'delimiter.variable.jinja' }, // }}
|
|
{ startIndex: 28, type: 'delimiter.tag.jinja' }, // {%
|
|
{ startIndex: 30, type: 'white.jinja' },
|
|
{ startIndex: 31, type: 'keyword.control.jinja' }, // endwith
|
|
{ startIndex: 38, type: 'white.jinja' },
|
|
{ startIndex: 39, type: 'delimiter.tag.jinja' } // %}
|
|
]
|
|
}
|
|
]
|
|
]);
|