From fa5444129ab4f0edcb4a9d1d1ca26951c4c585cb Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Thu, 6 Aug 2026 14:34:54 +0200 Subject: [PATCH 1/6] Add semantic tokens support --- internal/grammar/semantic_tokens.go | 55 ++++++ internal/grammar/semantic_tokens_test.go | 46 +++++ internal/grammar/services.go | 5 + server/semantic_tokens_builder.go | 129 ++++++++++++ server/semantic_tokens_builder_test.go | 114 +++++++++++ server/semantic_tokens_legend.go | 237 +++++++++++++++++++++++ server/semantic_tokens_provider.go | 109 +++++++++++ server/server.go | 43 +++- test/doc_fixture_lsp.go | 71 +++++++ 9 files changed, 801 insertions(+), 8 deletions(-) create mode 100644 internal/grammar/semantic_tokens.go create mode 100644 internal/grammar/semantic_tokens_test.go create mode 100644 server/semantic_tokens_builder.go create mode 100644 server/semantic_tokens_builder_test.go create mode 100644 server/semantic_tokens_legend.go create mode 100644 server/semantic_tokens_provider.go diff --git a/internal/grammar/semantic_tokens.go b/internal/grammar/semantic_tokens.go new file mode 100644 index 00000000..a3e6f4c1 --- /dev/null +++ b/internal/grammar/semantic_tokens.go @@ -0,0 +1,55 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package grammar + +import ( + "context" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/server" +) + +var legendProvider = server.NewExtendableSemanticTokensLegendProvider() + +type GrammarTokenHighlightingStrategy struct{} + +func NewGrammarTokenHighlightingStrategy() server.TokenHighlightingStrategy { + return &GrammarTokenHighlightingStrategy{} +} + +func (s *GrammarTokenHighlightingStrategy) Highlight(ctx context.Context, token core.Token, accept server.TokenHighlightingStrategyAcceptor) { + switch token.Kind { + case Grammar_Name_ID: + accept(legendProvider.Namespace(), 0) + case Interface_Name_ID, + Interface_Extends_ID_0, + Interface_Extends_ID_1: + accept(legendProvider.Interface(), 0) + case ParserRule_Name_ID, + Token_Name_ID, + CompositeRule_Name_ID, + RuleCall_Rule_ID, + TokenGroup_Name_ID, + TokenGroup_TokenRefs_ID: + accept(legendProvider.Function(), 0) + case Field_Name_ID, + Assignment_Property_ID, + Action_Property_ID: + accept(legendProvider.Property(), 0) + case PrimitiveType_Type_bool, + PrimitiveType_Type_composite, + PrimitiveType_Type_string, + SimpleType_Type_ID, + ReferenceType_Type_ID, + CrossRef_Type_ID, + ParserRule_ReturnType_ID, + Action_Type_ID, + Action_current: + accept(legendProvider.Type(), 0) + case Token_Type_comment, + Token_Type_hidden: + accept(legendProvider.Modifier(), 0) + } +} diff --git a/internal/grammar/semantic_tokens_test.go b/internal/grammar/semantic_tokens_test.go new file mode 100644 index 00000000..afb4f732 --- /dev/null +++ b/internal/grammar/semantic_tokens_test.go @@ -0,0 +1,46 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package grammar + +import ( + "testing" + + "typefox.dev/fastbelt/test" +) + +func TestSemanticTokensIntegration(t *testing.T) { + fixture := test.New(t, CreateServices()) + + grammarText := `<|comment:// Grammar for semantic token testing|> +grammar <|namespace:Test|>; + +interface <|interface:Expression|> {} +interface <|interface:BinaryExpression|> extends <|interface:Expression|> { + <|property:Left|> <|type:Expression|> + <|property:Operator|> <|type:string|> + <|property:Right|> <|type:Expression|> +} + +<|function:Addition|> returns <|type:Expression|>: + <|function:Primary|> + ({<|type:BinaryExpression|>.<|property:Left|>=<|type:current|>} + <|property:Operator|>=("+" | "-") <|property:Right|>=<|function:Primary|>)* +<|function:Primary|> returns <|type:Expression|>: + <|property:Operator|>=<|function:ID|> + +token <|function:ID|>: /[a-zA-Z_][a-zA-Z0-9_]*/; +<|modifier:hidden|> token <|function:WS|>: /[ \n\r\t]+/; +` + + doc := fixture.ParseURI(grammarText, "file:///semantic.fb") + doc.AssertNoParseErrors(). + AssertSemanticTokens("namespace", legendProvider.Namespace(), 0). + AssertSemanticTokens("interface", legendProvider.Interface(), 0). + AssertSemanticTokens("function", legendProvider.Function(), 0). + AssertSemanticTokens("property", legendProvider.Property(), 0). + AssertSemanticTokens("type", legendProvider.Type(), 0). + AssertSemanticTokens("modifier", legendProvider.Modifier(), 0). + AssertSemanticTokens("comment", legendProvider.Comment(), 0) +} diff --git a/internal/grammar/services.go b/internal/grammar/services.go index 1bf43378..bbf56926 100644 --- a/internal/grammar/services.go +++ b/internal/grammar/services.go @@ -8,6 +8,7 @@ package grammar import ( "typefox.dev/fastbelt/linking" + "typefox.dev/fastbelt/server" "typefox.dev/fastbelt/textdoc" "typefox.dev/fastbelt/util/service" "typefox.dev/fastbelt/workspace" @@ -29,6 +30,10 @@ func SetupServices(sc *service.Container) { // Override the default scope provider service.Override[FastbeltScopeProvider](sc, newScopeProviderImpl(sc)) service.Override(sc, newImportedSymbolsProviderImpl(sc)) + + // Set a semantic token highlighting strategy + service.Put[server.SemanticTokensLegendProvider](sc, legendProvider) + service.Put(sc, server.NewTokenBasedSemanticTokensProvider(sc, NewGrammarTokenHighlightingStrategy())) } // CreateServices creates a service container for the grammar language to be used in the CLI and tests. diff --git a/server/semantic_tokens_builder.go b/server/semantic_tokens_builder.go new file mode 100644 index 00000000..b6cb4f79 --- /dev/null +++ b/server/semantic_tokens_builder.go @@ -0,0 +1,129 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "unicode/utf16" + "unicode/utf8" + + core "typefox.dev/fastbelt" +) + +type SemanticTokensBuilder interface { + Data() []uint32 + Push(textRange core.TextRange, tokenType, tokenModifiers uint32) +} + +func NewSemanticTokensBuilder(text string, tokenCount int) SemanticTokensBuilder { + return &semanticTokensBuilder{ + // Preallocate the data with the maximum possible length + // Each token potentially contributes 5 uint32 values + data: make([]uint32, 0, tokenCount*5), + text: text, + } +} + +type semanticTokensBuilder struct { + // data is a slice of uint32 values representing the semantic tokens data in the LSP format. + // Each token is represented by five consecutive values: + // - deltaLine, token line number, relative to the previous token, + // - deltaStart, token start character, relative to the previous token, + // - length, the length of the token, + // - tokenType, the token type index, + // - tokenModifiers, the token modifiers bitset. + data []uint32 + text string + cursor int + prevLine int + prevChar int + currentLine int + currentChar int + // lineBreaks is reused across push calls to avoid per-token allocations + lineBreaks []int +} + +func (tokenData *semanticTokensBuilder) Data() []uint32 { + return tokenData.data +} + +func (tokenData *semanticTokensBuilder) Push(textRange core.TextRange, typeIndex, modifierIndex uint32) { + textLen := len(tokenData.text) + tokenStart := int(textRange.Start) + tokenEnd := int(textRange.End) + cursor := tokenData.cursor + currentLine := tokenData.currentLine + currentChar := tokenData.currentChar + startLine, startChar := 0, 0 + // Count line breaks within the token range as necessary + // We need to emit multiple tokens if the token spans multiple lines + lineBreaks := tokenData.lineBreaks[:0] + // Advance the cursor up to the end of the token + for tokenEnd > cursor { + if cursor >= textLen { + break + } else if cursor == tokenStart { + startLine = currentLine + startChar = currentChar + } + if c := tokenData.text[cursor]; c < utf8.RuneSelf { + // ASCII fast path: one byte, one UTF-16 code unit + if c == '\n' { + // Record the line break character position + if cursor >= tokenStart { + lineBreaks = append(lineBreaks, currentChar) + } + // New line, reset currentChar and increment currentLine + currentLine++ + currentChar = 0 + } else { + currentChar++ + } + cursor++ + continue + } + rune, size := utf8.DecodeRuneInString(tokenData.text[cursor:]) + // Advance column by the number of UTF-16 code units for the rune + // (newlines are ASCII, so this rune can never be one) + currentChar += utf16.RuneLen(rune) + // Advance cursor by the byte size of the rune + cursor += size + } + tokenData.lineBreaks = lineBreaks + lineDelta := uint32(startLine - tokenData.prevLine) + charDelta := uint32(startChar) + if lineDelta == 0 { + // If the token is on the same line as the previous token, calculate the character delta + charDelta -= uint32(tokenData.prevChar) + } + if len(lineBreaks) == 0 { + // Token is on a single line, emit it directly + length := uint32(currentChar - startChar) + tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex) + // Update the previous character position for the next token + tokenData.prevChar = startChar + } else { + // Token spans multiple lines, emit a token for each line segment + // First segment: from startChar to the first line break + length := uint32(lineBreaks[0] - startChar) + tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex) + // Subsequent segments: from each line break to the next line break + for i := 1; i < len(lineBreaks); i++ { + // always use the full length of the line + length = uint32(lineBreaks[i]) + // Note: lineDelta is always 1, since each segment is on a new line + // charDelta is always 0, since we are starting at the beginning of the line + tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex) + } + // Last segment: from the start of the last line to the end of the token + length = uint32(currentChar) + tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex) + tokenData.prevChar = 0 + } + // Update the data for the next token + tokenData.cursor = cursor + tokenData.prevLine = currentLine + tokenData.currentLine = currentLine + tokenData.currentChar = currentChar +} diff --git a/server/semantic_tokens_builder_test.go b/server/semantic_tokens_builder_test.go new file mode 100644 index 00000000..df14fe66 --- /dev/null +++ b/server/semantic_tokens_builder_test.go @@ -0,0 +1,114 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "slices" + "testing" + + core "typefox.dev/fastbelt" +) + +func TestLspTokenDataPush(t *testing.T) { + tests := []struct { + name string + text string + ranges []core.TextRange + expected []uint32 + }{ + { + name: "Single token at start", + text: "hello world", + ranges: []core.TextRange{core.NewTextRange(0, 5)}, + expected: []uint32{0, 0, 5, 1, 2}, + }, + { + name: "Two tokens on same line use char delta", + text: "hello world", + ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 11)}, + expected: []uint32{ + 0, 0, 5, 1, 2, + 0, 6, 5, 1, 2, + }, + }, + { + name: "Token on next line resets char delta", + text: "hello\nworld", + ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 11)}, + expected: []uint32{ + 0, 0, 5, 1, 2, + 1, 0, 5, 1, 2, + }, + }, + { + name: "Multi-line token emits one token per line", + text: "ab\ncdef\ngh", + ranges: []core.TextRange{core.NewTextRange(1, 9)}, + expected: []uint32{ + 0, 1, 1, 1, 2, // "b" on line 0 + 1, 0, 4, 1, 2, // "cdef" on line 1 + 1, 0, 1, 1, 2, // "g" on line 2 + }, + }, + { + name: "Token after multi-line token", + text: "ab\ncd ef", + ranges: []core.TextRange{core.NewTextRange(0, 5), core.NewTextRange(6, 8)}, + expected: []uint32{ + 0, 0, 2, 1, 2, + 1, 0, 2, 1, 2, + 0, 3, 2, 1, 2, + }, + }, + { + name: "Non-ASCII counts UTF-16 code units", + // "😀" is 4 bytes but 2 UTF-16 code units + text: "😀ab", + ranges: []core.TextRange{core.NewTextRange(4, 6)}, + expected: []uint32{ + 0, 2, 2, 1, 2, + }, + }, + { + name: "Range past end of text is clamped", + text: "ab", + ranges: []core.TextRange{core.NewTextRange(0, 10)}, + expected: []uint32{0, 0, 2, 1, 2}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + builder := NewSemanticTokensBuilder(tt.text, len(tt.ranges)) + for _, rng := range tt.ranges { + builder.Push(rng, 1, 2) + } + if !slices.Equal(builder.Data(), tt.expected) { + t.Errorf("expected %v, got %v", tt.expected, builder.Data()) + } + }) + } +} + +func BenchmarkLspTokenDataPush(b *testing.B) { + // Build a document of 1000 lines with 4 tokens each + line := "foo bar baz qux\n" + text := "" + ranges := []core.TextRange{} + for range 1000 { + offset := len(text) + for start := 0; start < 15; start += 4 { + ranges = append(ranges, core.NewTextRange(offset+start, offset+start+3)) + } + text += line + } + + for b.Loop() { + builder := NewSemanticTokensBuilder(text, len(ranges)) + for _, rng := range ranges { + builder.Push(rng, 1, 2) + } + } +} diff --git a/server/semantic_tokens_legend.go b/server/semantic_tokens_legend.go new file mode 100644 index 00000000..65ae79a8 --- /dev/null +++ b/server/semantic_tokens_legend.go @@ -0,0 +1,237 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "slices" + + "typefox.dev/lsp" +) + +// SemanticTokensLegendProvider provides the legend for semantic tokens LSP requests. +// Must be registered together with a [SemanticTokensProvider] in the service container +// to enable semantic tokens support for the language server. +type SemanticTokensLegendProvider interface { + Legend() lsp.SemanticTokensLegend +} + +// ExtendableSemanticTokensLegendProvider is a type of [SemanticTokensLegendProvider] that allows +// adding new token types and modifiers at runtime, extending the default legend. +// +// Note that this interface is useful beyond the ability to add new token types and modifiers. +// It also provides a convenient way to retrieve the index of the default token types and the bit +// values of default token modifiers, which can be used when implementing custom semantic token provider. +// +// // Declaring a new legend provider with custom token types and modifiers +// var legendProvider = server.NewExtendableSemanticTokensLegendProvider() +// // Returns the index of the new token type within the legend +// var extraType = legendProvider.AddType("extraTokenType") +// // Returns the bit value of the new token modifier within the legend +// var extraModifier = legendProvider.AddModifier("extraTokenModifier") +// // Register within the service container +// service.Put[server.SemanticTokensLegendProvider](sc, legendProvider) +type ExtendableSemanticTokensLegendProvider interface { + SemanticTokensLegendProvider + + // Type returns the index of the "type" token type within the legend. + Type() uint32 + // Class returns the index of the "class" token type within the legend. + Class() uint32 + // Enum returns the index of the "enum" token type within the legend. + Enum() uint32 + // Interface returns the index of the "interface" token type within the legend. + Interface() uint32 + // Struct returns the index of the "struct" token type within the legend. + Struct() uint32 + // TypeParameter returns the index of the "typeParameter" token type within the legend. + TypeParameter() uint32 + // Parameter returns the index of the "parameter" token type within the legend. + Parameter() uint32 + // Variable returns the index of the "variable" token type within the legend. + Variable() uint32 + // Property returns the index of the "property" token type within the legend. + Property() uint32 + // EnumMember returns the index of the "enumMember" token type within the legend. + EnumMember() uint32 + // Event returns the index of the "event" token type within the legend. + Event() uint32 + // Function returns the index of the "function" token type within the legend. + Function() uint32 + // Method returns the index of the "method" token type within the legend. + Method() uint32 + // Macro returns the index of the "macro" token type within the legend. + Macro() uint32 + // Keyword returns the index of the "keyword" token type within the legend. + Keyword() uint32 + // Modifier returns the index of the "modifier" token type within the legend. + Modifier() uint32 + // Comment returns the index of the "comment" token type within the legend. + Comment() uint32 + // String returns the index of the "string" token type within the legend. + String() uint32 + // Number returns the index of the "number" token type within the legend. + Number() uint32 + // Regexp returns the index of the "regexp" token type within the legend. + Regexp() uint32 + // Operator returns the index of the "operator" token type within the legend. + Operator() uint32 + // Decorator returns the index of the "decorator" token type within the legend. + Decorator() uint32 + // Label returns the index of the "label" token type within the legend. + Label() uint32 + // Namespace returns the index of the "namespace" token type within the legend. + Namespace() uint32 + + // Declaration returns the bit value of the "declaration" token modifier within the legend. + Declaration() uint32 + // Definition returns the bit value of the "definition" token modifier within the legend. + Definition() uint32 + // Readonly returns the bit value of the "readonly" token modifier within the legend. + Readonly() uint32 + // Static returns the bit value of the "static" token modifier within the legend. + Static() uint32 + // Deprecated returns the bit value of the "deprecated" token modifier within the legend. + Deprecated() uint32 + // Abstract returns the bit value of the "abstract" token modifier within the legend. + Abstract() uint32 + // Async returns the bit value of the "async" token modifier within the legend. + Async() uint32 + // Modification returns the bit value of the "modification" token modifier within the legend. + Modification() uint32 + // Documentation returns the bit value of the "documentation" token modifier within the legend. + Documentation() uint32 + // DefaultLibrary returns the bit value of the "defaultLibrary" token modifier within the legend. + DefaultLibrary() uint32 + + // AddType adds a new token type to the legend and returns its index. + AddType(name string) uint32 + // AddModifier adds a new token modifier to the legend and returns its bit value. + AddModifier(name string) uint32 +} + +// NewExtendableSemanticTokensLegendProvider creates a new instance of [ExtendableSemanticTokensLegendProvider]. +func NewExtendableSemanticTokensLegendProvider() ExtendableSemanticTokensLegendProvider { + return &extendableSemanticTokensLegendProvider{} +} + +func add(name string, existing *[]string) uint32 { + index := uint32(len(*existing)) + *existing = append(*existing, name) + return index +} + +var defaultTokenTypes []string + +var _type = add("type", &defaultTokenTypes) +var _class = add("class", &defaultTokenTypes) +var _enum = add("enum", &defaultTokenTypes) +var _interface = add("interface", &defaultTokenTypes) +var _struct = add("struct", &defaultTokenTypes) +var _typeParameter = add("typeParameter", &defaultTokenTypes) +var _parameter = add("parameter", &defaultTokenTypes) +var _variable = add("variable", &defaultTokenTypes) +var _property = add("property", &defaultTokenTypes) +var _enumMember = add("enumMember", &defaultTokenTypes) +var _event = add("event", &defaultTokenTypes) +var _function = add("function", &defaultTokenTypes) +var _method = add("method", &defaultTokenTypes) +var _macro = add("macro", &defaultTokenTypes) +var _keyword = add("keyword", &defaultTokenTypes) +var _modifier = add("modifier", &defaultTokenTypes) +var _comment = add("comment", &defaultTokenTypes) +var _string = add("string", &defaultTokenTypes) +var _number = add("number", &defaultTokenTypes) +var _regexp = add("regexp", &defaultTokenTypes) +var _operator = add("operator", &defaultTokenTypes) +var _decorator = add("decorator", &defaultTokenTypes) +var _label = add("label", &defaultTokenTypes) +var _namespace = add("namespace", &defaultTokenTypes) + +var defaultTokenModifiers []string + +var _declaration uint32 = 1 << add("declaration", &defaultTokenModifiers) +var _definition uint32 = 1 << add("definition", &defaultTokenModifiers) +var _readonly uint32 = 1 << add("readonly", &defaultTokenModifiers) +var _static uint32 = 1 << add("static", &defaultTokenModifiers) +var _deprecated uint32 = 1 << add("deprecated", &defaultTokenModifiers) +var _abstract uint32 = 1 << add("abstract", &defaultTokenModifiers) +var _async uint32 = 1 << add("async", &defaultTokenModifiers) +var _modification uint32 = 1 << add("modification", &defaultTokenModifiers) +var _documentation uint32 = 1 << add("documentation", &defaultTokenModifiers) +var _defaultLibrary uint32 = 1 << add("defaultLibrary", &defaultTokenModifiers) + +type extendableSemanticTokensLegendProvider struct { + types []string + modifiers []string +} + +func (d *extendableSemanticTokensLegendProvider) Type() uint32 { return _type } +func (d *extendableSemanticTokensLegendProvider) Class() uint32 { return _class } +func (d *extendableSemanticTokensLegendProvider) Enum() uint32 { return _enum } +func (d *extendableSemanticTokensLegendProvider) Interface() uint32 { return _interface } +func (d *extendableSemanticTokensLegendProvider) Struct() uint32 { return _struct } +func (d *extendableSemanticTokensLegendProvider) TypeParameter() uint32 { return _typeParameter } +func (d *extendableSemanticTokensLegendProvider) Parameter() uint32 { return _parameter } +func (d *extendableSemanticTokensLegendProvider) Variable() uint32 { return _variable } +func (d *extendableSemanticTokensLegendProvider) Property() uint32 { return _property } +func (d *extendableSemanticTokensLegendProvider) EnumMember() uint32 { return _enumMember } +func (d *extendableSemanticTokensLegendProvider) Event() uint32 { return _event } +func (d *extendableSemanticTokensLegendProvider) Function() uint32 { return _function } +func (d *extendableSemanticTokensLegendProvider) Method() uint32 { return _method } +func (d *extendableSemanticTokensLegendProvider) Macro() uint32 { return _macro } +func (d *extendableSemanticTokensLegendProvider) Keyword() uint32 { return _keyword } +func (d *extendableSemanticTokensLegendProvider) Modifier() uint32 { return _modifier } +func (d *extendableSemanticTokensLegendProvider) Comment() uint32 { return _comment } +func (d *extendableSemanticTokensLegendProvider) String() uint32 { return _string } +func (d *extendableSemanticTokensLegendProvider) Number() uint32 { return _number } +func (d *extendableSemanticTokensLegendProvider) Regexp() uint32 { return _regexp } +func (d *extendableSemanticTokensLegendProvider) Operator() uint32 { return _operator } +func (d *extendableSemanticTokensLegendProvider) Decorator() uint32 { return _decorator } +func (d *extendableSemanticTokensLegendProvider) Label() uint32 { return _label } +func (d *extendableSemanticTokensLegendProvider) Namespace() uint32 { return _namespace } + +func (d *extendableSemanticTokensLegendProvider) Declaration() uint32 { return _declaration } +func (d *extendableSemanticTokensLegendProvider) Definition() uint32 { return _definition } +func (d *extendableSemanticTokensLegendProvider) Readonly() uint32 { return _readonly } +func (d *extendableSemanticTokensLegendProvider) Static() uint32 { return _static } +func (d *extendableSemanticTokensLegendProvider) Deprecated() uint32 { return _deprecated } +func (d *extendableSemanticTokensLegendProvider) Abstract() uint32 { return _abstract } +func (d *extendableSemanticTokensLegendProvider) Async() uint32 { return _async } +func (d *extendableSemanticTokensLegendProvider) Modification() uint32 { return _modification } +func (d *extendableSemanticTokensLegendProvider) Documentation() uint32 { return _documentation } +func (d *extendableSemanticTokensLegendProvider) DefaultLibrary() uint32 { return _defaultLibrary } + +func (d *extendableSemanticTokensLegendProvider) AddType(name string) uint32 { + if slices.Contains(defaultTokenTypes, name) { + panic("Cannot add a token type that already exists in the default legend: " + name) + } else if slices.Contains(d.types, name) { + panic("Cannot add a token type that already exists in the legend: " + name) + } + return add(name, &d.types) + uint32(len(defaultTokenTypes)) +} + +func (d *extendableSemanticTokensLegendProvider) AddModifier(name string) uint32 { + if slices.Contains(defaultTokenModifiers, name) { + panic("Cannot add a token modifier that already exists in the default legend: " + name) + } else if slices.Contains(d.modifiers, name) { + panic("Cannot add a token modifier that already exists in the legend: " + name) + } else if (len(d.modifiers) + len(defaultTokenModifiers)) >= 32 { + panic("Cannot add a token modifier because the legend already has 32 modifiers") + } + return 1 << (add(name, &d.modifiers) + uint32(len(defaultTokenModifiers))) +} + +func (d *extendableSemanticTokensLegendProvider) Legend() lsp.SemanticTokensLegend { + tokenTypes := make([]string, len(defaultTokenTypes)+len(d.types)) + copy(tokenTypes, defaultTokenTypes) + copy(tokenTypes[len(defaultTokenTypes):], d.types) + tokenModifiers := make([]string, len(defaultTokenModifiers)+len(d.modifiers)) + copy(tokenModifiers, defaultTokenModifiers) + copy(tokenModifiers[len(defaultTokenModifiers):], d.modifiers) + return lsp.SemanticTokensLegend{ + TokenTypes: tokenTypes, + TokenModifiers: tokenModifiers, + } +} diff --git a/server/semantic_tokens_provider.go b/server/semantic_tokens_provider.go new file mode 100644 index 00000000..9775929d --- /dev/null +++ b/server/semantic_tokens_provider.go @@ -0,0 +1,109 @@ +// Copyright 2026 TypeFox GmbH +// This program and the accompanying materials are made available under the +// terms of the MIT License, which is available in the project root. + +package server + +import ( + "context" + "errors" + "slices" + "strconv" + "strings" + + core "typefox.dev/fastbelt" + "typefox.dev/fastbelt/util/service" + "typefox.dev/fastbelt/workspace" + "typefox.dev/lsp" +) + +// SemanticTokensProvider defines the interface for handling semantic tokens requests in the LSP. +// Must be registered together with a [SemanticTokensLegendProvider] in the service container +// to enable semantic tokens support for the language server. +type SemanticTokensProvider interface { + HandleSemanticTokensFullRequest(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) +} + +// TokenHighlightingStrategyAcceptor is a function type used in the [TokenHighlightingStrategy]. +type TokenHighlightingStrategyAcceptor func(tokenType uint32, tokenModifier uint32) + +// TokenHighlightingStrategy defines the interface for strategies that determine how individual tokens +// are highlighted by the [TokenBasedSemanticTokensProvider]. +type TokenHighlightingStrategy interface { + Highlight(ctx context.Context, token core.Token, accept TokenHighlightingStrategyAcceptor) +} + +// TokenBasedSemanticTokensProvider is an implementation of [SemanticTokensProvider] that generates semantic tokens +// for each individual token in the document, using a provided [TokenHighlightingStrategy] to determine the highlighting for each token. +// It also generates semantic tokens for comments in the document, if the "comment" token type is present in the legend. +type TokenBasedSemanticTokensProvider struct { + sc *service.Container + strategy TokenHighlightingStrategy +} + +// NewTokenBasedSemanticTokensProvider creates a new instance of [TokenBasedSemanticTokensProvider] with the given [TokenHighlightingStrategy]. +func NewTokenBasedSemanticTokensProvider(sc *service.Container, strategy TokenHighlightingStrategy) SemanticTokensProvider { + return &TokenBasedSemanticTokensProvider{sc: sc, strategy: strategy} +} + +func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) { + documentManager := service.MustGet[workspace.DocumentManager](p.sc) + tokenTypes := service.MustGet[SemanticTokensLegendProvider](p.sc).Legend().TokenTypes + uri := core.ParseURI(string(params.TextDocument.URI)) + doc := documentManager.Get(uri) + if doc == nil { + return nil, nil // Document not found + } + tokens := doc.Tokens + comments := doc.Comments + totalLen := len(tokens) + len(comments) + if totalLen == 0 { + return nil, nil // Document is empty, no tokens found + } + commentTypeIndex := slices.Index(tokenTypes, string(lsp.CommentType)) + tokenBuilder := NewSemanticTokensBuilder(doc.TextDoc.Text(nil), totalLen) + var errorRanges []core.TextRange + commentIndex := 0 + for _, token := range tokens { + for commentTypeIndex != -1 && + commentIndex < len(comments) && + comments[commentIndex].Range.Start < token.Range.Start { + // Add all comments that precede the current token + tokenBuilder.Push(comments[commentIndex].Range, uint32(commentTypeIndex), 0) + commentIndex++ + } + added := false + p.strategy.Highlight(ctx, token, func(tokenType uint32, tokenModifier uint32) { + if !added { + tokenBuilder.Push(token.Range, tokenType, tokenModifier) + added = true + } else { + errorRanges = append(errorRanges, token.Range) + } + }) + } + // Report any tokens that were highlighted multiple times for the same range + if len(errorRanges) > 0 { + sb := strings.Builder{} + sb.WriteString("Multiple semantic tokens returned for the same token ranges: ") + for i, rng := range errorRanges { + if i > 0 { + sb.WriteString(", ") + } + sb.WriteString(strconv.Itoa(int(rng.Start))) + sb.WriteString("-") + sb.WriteString(strconv.Itoa(int(rng.End))) + } + return nil, errors.New(sb.String()) + } + if commentTypeIndex != -1 { + for commentIndex < len(comments) { + // Add remaining comments after the last token + tokenBuilder.Push(comments[commentIndex].Range, uint32(commentTypeIndex), 0) + commentIndex++ + } + } + return &lsp.SemanticTokens{ + Data: tokenBuilder.Data(), + }, nil +} diff --git a/server/server.go b/server/server.go index f34f1995..db66f660 100644 --- a/server/server.go +++ b/server/server.go @@ -53,9 +53,23 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para return nil, err } workspaceFolders.Value = params.WorkspaceFolders - var triggerChars []string - if triggers, err := service.Get[CompletionTriggers](s.sc); err == nil && triggers != nil { - triggerChars = triggers.TriggerCharacters() + var completionOptions *lsp.CompletionOptions + if completionProvider, err := service.Get[CompletionProvider](s.sc); err == nil && completionProvider != nil { + completionOptions = &lsp.CompletionOptions{ + ResolveProvider: false, + } + if triggers, err := service.Get[CompletionTriggers](s.sc); err == nil && triggers != nil { + completionOptions.TriggerCharacters = triggers.TriggerCharacters() + } + } + var semanticTokensOptions *lsp.SemanticTokensOptions + if legendProvider, err := service.Get[SemanticTokensLegendProvider](s.sc); err == nil && legendProvider != nil { + semanticTokensOptions = &lsp.SemanticTokensOptions{ + Legend: legendProvider.Legend(), + Full: &lsp.Or_SemanticTokensOptions_full{ + Value: true, + }, + } } var renameProvider *lsp.RenameOptions if service.Has[RenameProvider](s.sc) { @@ -72,10 +86,7 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para Save: &lsp.SaveOptions{IncludeText: true}, Change: lsp.Incremental, }, - CompletionProvider: &lsp.CompletionOptions{ - ResolveProvider: false, - TriggerCharacters: triggerChars, - }, + CompletionProvider: completionOptions, DefinitionProvider: optionsIf[DefinitionProvider, lsp.DefinitionOptions](s.sc), DocumentSymbolProvider: optionsIf[DocumentSymbolProvider, lsp.DocumentSymbolOptions](s.sc), FoldingRangeProvider: optionsIf[FoldingRangeProvider, lsp.FoldingRangeRegistrationOptions](s.sc), @@ -84,6 +95,7 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para HoverProvider: optionsIf[HoverProvider, lsp.HoverOptions](s.sc), ReferencesProvider: optionsIf[ReferencesProvider, lsp.ReferenceOptions](s.sc), RenameProvider: renameProvider, + SemanticTokensProvider: semanticTokensOptions, }, }, nil } @@ -476,7 +488,22 @@ func (s *DefaultLanguageServer) SelectionRange(ctx context.Context, params *lsp. return nil, nil } func (s *DefaultLanguageServer) SemanticTokensFull(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) { - return nil, nil + lock, err := service.Get[workspace.Lock](s.sc) + if err != nil { + return nil, err + } + tokensProvider, err := service.Get[SemanticTokensProvider](s.sc) + if err != nil { + return nil, err + } + var result *lsp.SemanticTokens + var providerErr error + if err := lock.Read(ctx, func(ctx context.Context) { + result, providerErr = tokensProvider.HandleSemanticTokensFullRequest(ctx, params) + }); err != nil { + return nil, err + } + return result, providerErr } func (s *DefaultLanguageServer) SemanticTokensFullDelta(ctx context.Context, params *lsp.SemanticTokensDeltaParams) (any, error) { return nil, nil diff --git a/test/doc_fixture_lsp.go b/test/doc_fixture_lsp.go index a4de5511..4b157a84 100644 --- a/test/doc_fixture_lsp.go +++ b/test/doc_fixture_lsp.go @@ -420,3 +420,74 @@ func findSymbolAtRange(symbols []lsp.DocumentSymbol, targetRange lsp.Range) *lsp } return nil } + +// AssertSemanticTokens verifies that every marker range with the given label +// has a semantic token with the expected type and modifiers. +// Returns the [Doc] for chaining. +func (d *Doc) AssertSemanticTokens(label string, expectedType uint32, expectedModifiers uint32) *Doc { + d.fixture.t.Helper() + + ranges := d.markerRanges(label) + if len(ranges) == 0 { + d.fixture.t.Fatalf("fbtest: no marker with label %q", label) + } + + semanticTokensProvider := service.MustGet[server.SemanticTokensProvider](d.fixture.sc) + params := &lsp.SemanticTokensParams{ + TextDocument: lsp.TextDocumentIdentifier{ + URI: lsp.DocumentURI(d.Document.URI.DocumentURI()), + }, + } + result, err := semanticTokensProvider.HandleSemanticTokensFullRequest(d.fixture.ctx, params) + if err != nil { + d.fixture.t.Fatalf("fbtest: HandleSemanticTokensFullRequest returned error: %v", err) + } + + // Decode the semantic tokens data, which comes in chunks of 5 uint32 values: + // [lineDelta, startCharDelta, length, tokenType, tokenModifiers] + type semanticToken struct { + line, column, length, tokenType, tokenModifiers uint32 + } + var tokens []semanticToken + var line, column uint32 + for i := 0; i+4 < len(result.Data); i += 5 { + line += result.Data[i] + if result.Data[i] == 0 { + // Same line, column is relative to previous token + column += result.Data[i+1] + } else { + // New line, column is absolute + column = result.Data[i+1] + } + tokens = append(tokens, semanticToken{line, column, result.Data[i+2], result.Data[i+3], result.Data[i+4]}) + } + + for _, rng := range ranges { + startPosition := d.Document.TextDoc.PositionAt(int(rng.Start)) + endPosition := d.Document.TextDoc.PositionAt(int(rng.End)) + if startPosition.Line != endPosition.Line { + d.fixture.t.Fatalf("fbtest: AssertSemanticToken: marker %q spans multiple lines, which is not supported", label) + } + found := false + for _, token := range tokens { + if token.line == startPosition.Line && token.column == startPosition.Character { + found = true + expectedLength := uint32(endPosition.Character - startPosition.Character) + if token.length != expectedLength { + d.fixture.t.Errorf("fbtest: semantic token at %q (%d:%d) has length %d, expected %d", label, token.line, token.column, token.length, expectedLength) + } + if token.tokenType != expectedType { + d.fixture.t.Errorf("fbtest: semantic token at %q (%d:%d) has type %d, expected %d", label, token.line, token.column, token.tokenType, expectedType) + } + if token.tokenModifiers != expectedModifiers { + d.fixture.t.Errorf("fbtest: semantic token at %q (%d:%d) has modifiers %d, expected %d", label, token.line, token.column, token.tokenModifiers, expectedModifiers) + } + break + } + } + if !found { + d.fixture.t.Errorf("fbtest: no semantic token found at %q (%d:%d)", label, startPosition.Line, startPosition.Character) + } + } + return d +} From 2b0c8ed4e336404220828d16abd8870798f67621 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Wed, 19 Aug 2026 15:44:22 +0200 Subject: [PATCH 2/6] Refactor testing --- internal/grammar/semantic_tokens_test.go | 18 +++++---- test/doc_fixture_lsp.go | 50 +++++++++++++++--------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/internal/grammar/semantic_tokens_test.go b/internal/grammar/semantic_tokens_test.go index afb4f732..a3dbbebb 100644 --- a/internal/grammar/semantic_tokens_test.go +++ b/internal/grammar/semantic_tokens_test.go @@ -35,12 +35,14 @@ token <|function:ID|>: /[a-zA-Z_][a-zA-Z0-9_]*/; ` doc := fixture.ParseURI(grammarText, "file:///semantic.fb") - doc.AssertNoParseErrors(). - AssertSemanticTokens("namespace", legendProvider.Namespace(), 0). - AssertSemanticTokens("interface", legendProvider.Interface(), 0). - AssertSemanticTokens("function", legendProvider.Function(), 0). - AssertSemanticTokens("property", legendProvider.Property(), 0). - AssertSemanticTokens("type", legendProvider.Type(), 0). - AssertSemanticTokens("modifier", legendProvider.Modifier(), 0). - AssertSemanticTokens("comment", legendProvider.Comment(), 0) + doc.AssertNoParseErrors() + semanticTokens := doc.ExpectSemanticTokens() + semanticTokens. + Assert("namespace", legendProvider.Namespace(), 0). + Assert("interface", legendProvider.Interface(), 0). + Assert("function", legendProvider.Function(), 0). + Assert("property", legendProvider.Property(), 0). + Assert("type", legendProvider.Type(), 0). + Assert("modifier", legendProvider.Modifier(), 0). + Assert("comment", legendProvider.Comment(), 0) } diff --git a/test/doc_fixture_lsp.go b/test/doc_fixture_lsp.go index 4b157a84..586f96b3 100644 --- a/test/doc_fixture_lsp.go +++ b/test/doc_fixture_lsp.go @@ -421,17 +421,10 @@ func findSymbolAtRange(symbols []lsp.DocumentSymbol, targetRange lsp.Range) *lsp return nil } -// AssertSemanticTokens verifies that every marker range with the given label -// has a semantic token with the expected type and modifiers. -// Returns the [Doc] for chaining. -func (d *Doc) AssertSemanticTokens(label string, expectedType uint32, expectedModifiers uint32) *Doc { +// ExpectSemanticTokens retrieves the semantic tokens for the document +// and returns a SemanticTokenExpectation for asserting on them. +func (d *Doc) ExpectSemanticTokens() *SemanticTokenExpectation { d.fixture.t.Helper() - - ranges := d.markerRanges(label) - if len(ranges) == 0 { - d.fixture.t.Fatalf("fbtest: no marker with label %q", label) - } - semanticTokensProvider := service.MustGet[server.SemanticTokensProvider](d.fixture.sc) params := &lsp.SemanticTokensParams{ TextDocument: lsp.TextDocumentIdentifier{ @@ -442,12 +435,6 @@ func (d *Doc) AssertSemanticTokens(label string, expectedType uint32, expectedMo if err != nil { d.fixture.t.Fatalf("fbtest: HandleSemanticTokensFullRequest returned error: %v", err) } - - // Decode the semantic tokens data, which comes in chunks of 5 uint32 values: - // [lineDelta, startCharDelta, length, tokenType, tokenModifiers] - type semanticToken struct { - line, column, length, tokenType, tokenModifiers uint32 - } var tokens []semanticToken var line, column uint32 for i := 0; i+4 < len(result.Data); i += 5 { @@ -461,7 +448,34 @@ func (d *Doc) AssertSemanticTokens(label string, expectedType uint32, expectedMo } tokens = append(tokens, semanticToken{line, column, result.Data[i+2], result.Data[i+3], result.Data[i+4]}) } + return &SemanticTokenExpectation{ + doc: d, + semanticTokens: tokens, + } +} +// Decode the semantic tokens data, which comes in chunks of 5 uint32 values: +// [lineDelta, startCharDelta, length, tokenType, tokenModifiers] +type semanticToken struct { + line, column, length, tokenType, tokenModifiers uint32 +} + +// SemanticTokenExpectation is a helper struct for asserting semantic tokens in tests. +type SemanticTokenExpectation struct { + doc *Doc + semanticTokens []semanticToken +} + +// Assert verifies that every marker range with the given label +// has a semantic token with the expected type and modifiers. +// Returns the [SemanticTokenExpectation] for chaining. +func (e *SemanticTokenExpectation) Assert(label string, expectedType uint32, expectedModifiers uint32) *SemanticTokenExpectation { + d := e.doc + d.fixture.t.Helper() + ranges := d.markerRanges(label) + if len(ranges) == 0 { + d.fixture.t.Fatalf("fbtest: no marker with label %q", label) + } for _, rng := range ranges { startPosition := d.Document.TextDoc.PositionAt(int(rng.Start)) endPosition := d.Document.TextDoc.PositionAt(int(rng.End)) @@ -469,7 +483,7 @@ func (d *Doc) AssertSemanticTokens(label string, expectedType uint32, expectedMo d.fixture.t.Fatalf("fbtest: AssertSemanticToken: marker %q spans multiple lines, which is not supported", label) } found := false - for _, token := range tokens { + for _, token := range e.semanticTokens { if token.line == startPosition.Line && token.column == startPosition.Character { found = true expectedLength := uint32(endPosition.Character - startPosition.Character) @@ -489,5 +503,5 @@ func (d *Doc) AssertSemanticTokens(label string, expectedType uint32, expectedMo d.fixture.t.Errorf("fbtest: no semantic token found at %q (%d:%d)", label, startPosition.Line, startPosition.Character) } } - return d + return e } From 2c31d540a8a915dc899996ed72eb649fafe204ca Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Mon, 31 Aug 2026 14:44:05 +0200 Subject: [PATCH 3/6] Review comments --- server/semantic_tokens_builder.go | 20 ++++- server/semantic_tokens_legend.go | 140 ++++++++++++++++------------- server/semantic_tokens_provider.go | 56 +++++++++--- server/server.go | 4 + test/doc_fixture_lsp.go | 2 + 5 files changed, 143 insertions(+), 79 deletions(-) diff --git a/server/semantic_tokens_builder.go b/server/semantic_tokens_builder.go index b6cb4f79..3c3e6182 100644 --- a/server/semantic_tokens_builder.go +++ b/server/semantic_tokens_builder.go @@ -11,15 +11,33 @@ import ( core "typefox.dev/fastbelt" ) +// SemanticTokensBuilder defines the interface for building semantic tokens data in the LSP format. +// It provides methods to push individual tokens and retrieve the final data slice. +// +// It is recommended to build the semantic tokens data by using the [TokenBasedSemanticTokensProvider] and its +// associated [TokenHighlightingStrategy] implementations. type SemanticTokensBuilder interface { + // Data returns the final semantic tokens data slice in the LSP format. + // The LSP data slice is a flat array of uint32 values, where each token is represented by five consecutive values: + // (1) deltaLine: token line number, relative to the previous token, + // (2) deltaStart: token start character, relative to the previous token, + // (3) length: the length of the token, + // (4) tokenType: the token type index in the legend, and + // (5) tokenModifiers: the token modifiers bitset in the legend. Data() []uint32 + // Push adds a new token to the semantic tokens data. + // Tokens should be pushed in the order they appear in the document, as the LSP + // data is based on the offset of the previous token. + // Due to this, this method is not thread-safe. Push(textRange core.TextRange, tokenType, tokenModifiers uint32) } +// NewSemanticTokensBuilder creates a new instance of [SemanticTokensBuilder]. func NewSemanticTokensBuilder(text string, tokenCount int) SemanticTokensBuilder { return &semanticTokensBuilder{ - // Preallocate the data with the maximum possible length + // Preallocate the data with a reasonable length // Each token potentially contributes 5 uint32 values + // Multiline tokens can contribute more, but we can resize the slice if needed data: make([]uint32, 0, tokenCount*5), text: text, } diff --git a/server/semantic_tokens_legend.go b/server/semantic_tokens_legend.go index 65ae79a8..ca9c4445 100644 --- a/server/semantic_tokens_legend.go +++ b/server/semantic_tokens_legend.go @@ -6,6 +6,7 @@ package server import ( "slices" + "sync" "typefox.dev/lsp" ) @@ -84,30 +85,32 @@ type ExtendableSemanticTokensLegendProvider interface { // Namespace returns the index of the "namespace" token type within the legend. Namespace() uint32 - // Declaration returns the bit value of the "declaration" token modifier within the legend. - Declaration() uint32 - // Definition returns the bit value of the "definition" token modifier within the legend. - Definition() uint32 - // Readonly returns the bit value of the "readonly" token modifier within the legend. - Readonly() uint32 - // Static returns the bit value of the "static" token modifier within the legend. - Static() uint32 - // Deprecated returns the bit value of the "deprecated" token modifier within the legend. - Deprecated() uint32 - // Abstract returns the bit value of the "abstract" token modifier within the legend. - Abstract() uint32 - // Async returns the bit value of the "async" token modifier within the legend. - Async() uint32 - // Modification returns the bit value of the "modification" token modifier within the legend. - Modification() uint32 - // Documentation returns the bit value of the "documentation" token modifier within the legend. - Documentation() uint32 - // DefaultLibrary returns the bit value of the "defaultLibrary" token modifier within the legend. - DefaultLibrary() uint32 + // ModDeclaration returns the bit value of the "declaration" token modifier within the legend. + ModDeclaration() uint32 + // ModDefinition returns the bit value of the "definition" token modifier within the legend. + ModDefinition() uint32 + // ModReadonly returns the bit value of the "readonly" token modifier within the legend. + ModReadonly() uint32 + // ModStatic returns the bit value of the "static" token modifier within the legend. + ModStatic() uint32 + // ModDeprecated returns the bit value of the "deprecated" token modifier within the legend. + ModDeprecated() uint32 + // ModAbstract returns the bit value of the "abstract" token modifier within the legend. + ModAbstract() uint32 + // ModAsync returns the bit value of the "async" token modifier within the legend. + ModAsync() uint32 + // ModModification returns the bit value of the "modification" token modifier within the legend. + ModModification() uint32 + // ModDocumentation returns the bit value of the "documentation" token modifier within the legend. + ModDocumentation() uint32 + // ModDefaultLibrary returns the bit value of the "defaultLibrary" token modifier within the legend. + ModDefaultLibrary() uint32 // AddType adds a new token type to the legend and returns its index. AddType(name string) uint32 // AddModifier adds a new token modifier to the legend and returns its bit value. + // Note that the legend can only have a maximum of 32 token modifiers, so adding a new modifier + // when the legend already has 32 modifiers will result in a panic. AddModifier(name string) uint32 } @@ -124,47 +127,48 @@ func add(name string, existing *[]string) uint32 { var defaultTokenTypes []string -var _type = add("type", &defaultTokenTypes) -var _class = add("class", &defaultTokenTypes) -var _enum = add("enum", &defaultTokenTypes) -var _interface = add("interface", &defaultTokenTypes) -var _struct = add("struct", &defaultTokenTypes) -var _typeParameter = add("typeParameter", &defaultTokenTypes) -var _parameter = add("parameter", &defaultTokenTypes) -var _variable = add("variable", &defaultTokenTypes) -var _property = add("property", &defaultTokenTypes) -var _enumMember = add("enumMember", &defaultTokenTypes) -var _event = add("event", &defaultTokenTypes) -var _function = add("function", &defaultTokenTypes) -var _method = add("method", &defaultTokenTypes) -var _macro = add("macro", &defaultTokenTypes) -var _keyword = add("keyword", &defaultTokenTypes) -var _modifier = add("modifier", &defaultTokenTypes) -var _comment = add("comment", &defaultTokenTypes) -var _string = add("string", &defaultTokenTypes) -var _number = add("number", &defaultTokenTypes) -var _regexp = add("regexp", &defaultTokenTypes) -var _operator = add("operator", &defaultTokenTypes) -var _decorator = add("decorator", &defaultTokenTypes) -var _label = add("label", &defaultTokenTypes) -var _namespace = add("namespace", &defaultTokenTypes) +var _type = add(string(lsp.TypeType), &defaultTokenTypes) +var _class = add(string(lsp.ClassType), &defaultTokenTypes) +var _enum = add(string(lsp.EnumType), &defaultTokenTypes) +var _interface = add(string(lsp.InterfaceType), &defaultTokenTypes) +var _struct = add(string(lsp.StructType), &defaultTokenTypes) +var _typeParameter = add(string(lsp.TypeParameterType), &defaultTokenTypes) +var _parameter = add(string(lsp.ParameterType), &defaultTokenTypes) +var _variable = add(string(lsp.VariableType), &defaultTokenTypes) +var _property = add(string(lsp.PropertyType), &defaultTokenTypes) +var _enumMember = add(string(lsp.EnumMemberType), &defaultTokenTypes) +var _event = add(string(lsp.EventType), &defaultTokenTypes) +var _function = add(string(lsp.FunctionType), &defaultTokenTypes) +var _method = add(string(lsp.MethodType), &defaultTokenTypes) +var _macro = add(string(lsp.MacroType), &defaultTokenTypes) +var _keyword = add(string(lsp.KeywordType), &defaultTokenTypes) +var _modifier = add(string(lsp.ModifierType), &defaultTokenTypes) +var _comment = add(string(lsp.CommentType), &defaultTokenTypes) +var _string = add(string(lsp.StringType), &defaultTokenTypes) +var _number = add(string(lsp.NumberType), &defaultTokenTypes) +var _regexp = add(string(lsp.RegexpType), &defaultTokenTypes) +var _operator = add(string(lsp.OperatorType), &defaultTokenTypes) +var _decorator = add(string(lsp.DecoratorType), &defaultTokenTypes) +var _label = add(string(lsp.LabelType), &defaultTokenTypes) +var _namespace = add(string(lsp.NamespaceType), &defaultTokenTypes) var defaultTokenModifiers []string -var _declaration uint32 = 1 << add("declaration", &defaultTokenModifiers) -var _definition uint32 = 1 << add("definition", &defaultTokenModifiers) -var _readonly uint32 = 1 << add("readonly", &defaultTokenModifiers) -var _static uint32 = 1 << add("static", &defaultTokenModifiers) -var _deprecated uint32 = 1 << add("deprecated", &defaultTokenModifiers) -var _abstract uint32 = 1 << add("abstract", &defaultTokenModifiers) -var _async uint32 = 1 << add("async", &defaultTokenModifiers) -var _modification uint32 = 1 << add("modification", &defaultTokenModifiers) -var _documentation uint32 = 1 << add("documentation", &defaultTokenModifiers) -var _defaultLibrary uint32 = 1 << add("defaultLibrary", &defaultTokenModifiers) +var _modDeclaration uint32 = 1 << add(string(lsp.ModDeclaration), &defaultTokenModifiers) +var _modDefinition uint32 = 1 << add(string(lsp.ModDefinition), &defaultTokenModifiers) +var _modReadonly uint32 = 1 << add(string(lsp.ModReadonly), &defaultTokenModifiers) +var _modStatic uint32 = 1 << add(string(lsp.ModStatic), &defaultTokenModifiers) +var _modDeprecated uint32 = 1 << add(string(lsp.ModDeprecated), &defaultTokenModifiers) +var _modAbstract uint32 = 1 << add(string(lsp.ModAbstract), &defaultTokenModifiers) +var _modAsync uint32 = 1 << add(string(lsp.ModAsync), &defaultTokenModifiers) +var _modModification uint32 = 1 << add(string(lsp.ModModification), &defaultTokenModifiers) +var _modDocumentation uint32 = 1 << add(string(lsp.ModDocumentation), &defaultTokenModifiers) +var _modDefaultLibrary uint32 = 1 << add(string(lsp.ModDefaultLibrary), &defaultTokenModifiers) type extendableSemanticTokensLegendProvider struct { types []string modifiers []string + mutex sync.Mutex } func (d *extendableSemanticTokensLegendProvider) Type() uint32 { return _type } @@ -192,18 +196,22 @@ func (d *extendableSemanticTokensLegendProvider) Decorator() uint32 { return func (d *extendableSemanticTokensLegendProvider) Label() uint32 { return _label } func (d *extendableSemanticTokensLegendProvider) Namespace() uint32 { return _namespace } -func (d *extendableSemanticTokensLegendProvider) Declaration() uint32 { return _declaration } -func (d *extendableSemanticTokensLegendProvider) Definition() uint32 { return _definition } -func (d *extendableSemanticTokensLegendProvider) Readonly() uint32 { return _readonly } -func (d *extendableSemanticTokensLegendProvider) Static() uint32 { return _static } -func (d *extendableSemanticTokensLegendProvider) Deprecated() uint32 { return _deprecated } -func (d *extendableSemanticTokensLegendProvider) Abstract() uint32 { return _abstract } -func (d *extendableSemanticTokensLegendProvider) Async() uint32 { return _async } -func (d *extendableSemanticTokensLegendProvider) Modification() uint32 { return _modification } -func (d *extendableSemanticTokensLegendProvider) Documentation() uint32 { return _documentation } -func (d *extendableSemanticTokensLegendProvider) DefaultLibrary() uint32 { return _defaultLibrary } +func (d *extendableSemanticTokensLegendProvider) ModDeclaration() uint32 { return _modDeclaration } +func (d *extendableSemanticTokensLegendProvider) ModDefinition() uint32 { return _modDefinition } +func (d *extendableSemanticTokensLegendProvider) ModReadonly() uint32 { return _modReadonly } +func (d *extendableSemanticTokensLegendProvider) ModStatic() uint32 { return _modStatic } +func (d *extendableSemanticTokensLegendProvider) ModDeprecated() uint32 { return _modDeprecated } +func (d *extendableSemanticTokensLegendProvider) ModAbstract() uint32 { return _modAbstract } +func (d *extendableSemanticTokensLegendProvider) ModAsync() uint32 { return _modAsync } +func (d *extendableSemanticTokensLegendProvider) ModModification() uint32 { return _modModification } +func (d *extendableSemanticTokensLegendProvider) ModDocumentation() uint32 { return _modDocumentation } +func (d *extendableSemanticTokensLegendProvider) ModDefaultLibrary() uint32 { + return _modDefaultLibrary +} func (d *extendableSemanticTokensLegendProvider) AddType(name string) uint32 { + d.mutex.Lock() + defer d.mutex.Unlock() if slices.Contains(defaultTokenTypes, name) { panic("Cannot add a token type that already exists in the default legend: " + name) } else if slices.Contains(d.types, name) { @@ -213,6 +221,8 @@ func (d *extendableSemanticTokensLegendProvider) AddType(name string) uint32 { } func (d *extendableSemanticTokensLegendProvider) AddModifier(name string) uint32 { + d.mutex.Lock() + defer d.mutex.Unlock() if slices.Contains(defaultTokenModifiers, name) { panic("Cannot add a token modifier that already exists in the default legend: " + name) } else if slices.Contains(d.modifiers, name) { @@ -224,6 +234,8 @@ func (d *extendableSemanticTokensLegendProvider) AddModifier(name string) uint32 } func (d *extendableSemanticTokensLegendProvider) Legend() lsp.SemanticTokensLegend { + d.mutex.Lock() + defer d.mutex.Unlock() tokenTypes := make([]string, len(defaultTokenTypes)+len(d.types)) copy(tokenTypes, defaultTokenTypes) copy(tokenTypes[len(defaultTokenTypes):], d.types) diff --git a/server/semantic_tokens_provider.go b/server/semantic_tokens_provider.go index 9775929d..63c1d357 100644 --- a/server/semantic_tokens_provider.go +++ b/server/semantic_tokens_provider.go @@ -10,6 +10,7 @@ import ( "slices" "strconv" "strings" + "sync" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/util/service" @@ -29,26 +30,42 @@ type TokenHighlightingStrategyAcceptor func(tokenType uint32, tokenModifier uint // TokenHighlightingStrategy defines the interface for strategies that determine how individual tokens // are highlighted by the [TokenBasedSemanticTokensProvider]. +// +// Note that the "accept" function should only be called once per token. +// Calling it multiple times for the same token will result in an error being returned to the language client. type TokenHighlightingStrategy interface { Highlight(ctx context.Context, token core.Token, accept TokenHighlightingStrategyAcceptor) } +// CommentTokenHighlightingStrategy extends the [TokenHighlightingStrategy] interface to include a method for highlighting comment tokens. +// If a [TokenHighlightingStrategy] also implements this interface, the [TokenBasedSemanticTokensProvider] will use it to highlight +// comment tokens in addition to regular tokens. +// Otherwise, comment tokens will be highlighted using the comment token type from the legend with no modifiers. +type CommentTokenHighlightingStrategy interface { + TokenHighlightingStrategy + HighlightComment(ctx context.Context, commentToken core.Token, accept TokenHighlightingStrategyAcceptor) +} + // TokenBasedSemanticTokensProvider is an implementation of [SemanticTokensProvider] that generates semantic tokens // for each individual token in the document, using a provided [TokenHighlightingStrategy] to determine the highlighting for each token. // It also generates semantic tokens for comments in the document, if the "comment" token type is present in the legend. type TokenBasedSemanticTokensProvider struct { - sc *service.Container - strategy TokenHighlightingStrategy + sc *service.Container + strategy TokenHighlightingStrategy + commentTypeIndexFunc func() int // Lazily initialized index of the comment token type in the legend } // NewTokenBasedSemanticTokensProvider creates a new instance of [TokenBasedSemanticTokensProvider] with the given [TokenHighlightingStrategy]. func NewTokenBasedSemanticTokensProvider(sc *service.Container, strategy TokenHighlightingStrategy) SemanticTokensProvider { - return &TokenBasedSemanticTokensProvider{sc: sc, strategy: strategy} + return &TokenBasedSemanticTokensProvider{sc: sc, strategy: strategy, commentTypeIndexFunc: sync.OnceValue(func() int { + tokenTypes := service.MustGet[SemanticTokensLegendProvider](sc).Legend().TokenTypes + commentTypeIndex := slices.Index(tokenTypes, string(lsp.CommentType)) + return commentTypeIndex + })} } func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) { documentManager := service.MustGet[workspace.DocumentManager](p.sc) - tokenTypes := service.MustGet[SemanticTokensLegendProvider](p.sc).Legend().TokenTypes uri := core.ParseURI(string(params.TextDocument.URI)) doc := documentManager.Get(uri) if doc == nil { @@ -60,16 +77,29 @@ func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx c if totalLen == 0 { return nil, nil // Document is empty, no tokens found } - commentTypeIndex := slices.Index(tokenTypes, string(lsp.CommentType)) + commentTypeIndex := p.commentTypeIndexFunc() tokenBuilder := NewSemanticTokensBuilder(doc.TextDoc.Text(nil), totalLen) + highlightComment := func(commentToken core.Token) {} + if commentStrategy, ok := p.strategy.(CommentTokenHighlightingStrategy); ok { + // Adopter has supplied a comment highlighting strategy, use that one + highlightComment = func(commentToken core.Token) { + commentStrategy.HighlightComment(ctx, commentToken, func(tokenType uint32, tokenModifier uint32) { + tokenBuilder.Push(commentToken.Range, tokenType, tokenModifier) + }) + } + } else if commentTypeIndex >= 0 { + // Highlight comments using the comment token type from the legend with no modifiers + highlightComment = func(commentToken core.Token) { + tokenBuilder.Push(commentToken.Range, uint32(commentTypeIndex), 0) + } + } var errorRanges []core.TextRange commentIndex := 0 for _, token := range tokens { - for commentTypeIndex != -1 && - commentIndex < len(comments) && + for commentIndex < len(comments) && comments[commentIndex].Range.Start < token.Range.Start { // Add all comments that precede the current token - tokenBuilder.Push(comments[commentIndex].Range, uint32(commentTypeIndex), 0) + highlightComment(comments[commentIndex]) commentIndex++ } added := false @@ -96,12 +126,10 @@ func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx c } return nil, errors.New(sb.String()) } - if commentTypeIndex != -1 { - for commentIndex < len(comments) { - // Add remaining comments after the last token - tokenBuilder.Push(comments[commentIndex].Range, uint32(commentTypeIndex), 0) - commentIndex++ - } + for commentIndex < len(comments) { + // Add remaining comments after the last token + highlightComment(comments[commentIndex]) + commentIndex++ } return &lsp.SemanticTokens{ Data: tokenBuilder.Data(), diff --git a/server/server.go b/server/server.go index db66f660..6cd5870d 100644 --- a/server/server.go +++ b/server/server.go @@ -6,6 +6,7 @@ package server import ( "context" + "errors" "log" "golang.org/x/exp/jsonrpc2" @@ -70,6 +71,9 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para Value: true, }, } + if !service.Has[SemanticTokensProvider](s.sc) { + return nil, errors.New("SemanticTokensLegendProvider is registered without a SemanticTokensProvider") + } } var renameProvider *lsp.RenameOptions if service.Has[RenameProvider](s.sc) { diff --git a/test/doc_fixture_lsp.go b/test/doc_fixture_lsp.go index 586f96b3..dd3ab922 100644 --- a/test/doc_fixture_lsp.go +++ b/test/doc_fixture_lsp.go @@ -434,6 +434,8 @@ func (d *Doc) ExpectSemanticTokens() *SemanticTokenExpectation { result, err := semanticTokensProvider.HandleSemanticTokensFullRequest(d.fixture.ctx, params) if err != nil { d.fixture.t.Fatalf("fbtest: HandleSemanticTokensFullRequest returned error: %v", err) + } else if result == nil { + d.fixture.t.Fatalf("fbtest: HandleSemanticTokensFullRequest returned nil result") } var tokens []semanticToken var line, column uint32 From 6dc95ce3df7b34838b6c569089822f1add319a95 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Tue, 1 Sep 2026 13:03:56 +0200 Subject: [PATCH 4/6] Move legend provider into token provider --- .devcontainer/devcontainer.json | 2 +- internal/grammar/services.go | 7 ++++-- server/semantic_tokens_legend.go | 2 -- server/semantic_tokens_provider.go | 35 ++++++++++++++++++------------ server/server.go | 8 ++----- 5 files changed, 29 insertions(+), 25 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 02e96fa4..f469a6b8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ { "name": "Go", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/go:1-1.26-bookworm", + "image": "mcr.microsoft.com/devcontainers/go:1.26-bookworm", // Features to add to the dev container. More info: https://containers.dev/features. "features": { diff --git a/internal/grammar/services.go b/internal/grammar/services.go index bbf56926..80fc2368 100644 --- a/internal/grammar/services.go +++ b/internal/grammar/services.go @@ -32,8 +32,11 @@ func SetupServices(sc *service.Container) { service.Override(sc, newImportedSymbolsProviderImpl(sc)) // Set a semantic token highlighting strategy - service.Put[server.SemanticTokensLegendProvider](sc, legendProvider) - service.Put(sc, server.NewTokenBasedSemanticTokensProvider(sc, NewGrammarTokenHighlightingStrategy())) + service.Put(sc, server.NewTokenBasedSemanticTokensProvider( + sc, + legendProvider, + NewGrammarTokenHighlightingStrategy(), + )) } // CreateServices creates a service container for the grammar language to be used in the CLI and tests. diff --git a/server/semantic_tokens_legend.go b/server/semantic_tokens_legend.go index ca9c4445..06344d2f 100644 --- a/server/semantic_tokens_legend.go +++ b/server/semantic_tokens_legend.go @@ -12,8 +12,6 @@ import ( ) // SemanticTokensLegendProvider provides the legend for semantic tokens LSP requests. -// Must be registered together with a [SemanticTokensProvider] in the service container -// to enable semantic tokens support for the language server. type SemanticTokensLegendProvider interface { Legend() lsp.SemanticTokensLegend } diff --git a/server/semantic_tokens_provider.go b/server/semantic_tokens_provider.go index 63c1d357..fe275a5a 100644 --- a/server/semantic_tokens_provider.go +++ b/server/semantic_tokens_provider.go @@ -10,7 +10,6 @@ import ( "slices" "strconv" "strings" - "sync" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/util/service" @@ -19,9 +18,10 @@ import ( ) // SemanticTokensProvider defines the interface for handling semantic tokens requests in the LSP. -// Must be registered together with a [SemanticTokensLegendProvider] in the service container -// to enable semantic tokens support for the language server. +// Also provides the token legend that is sent to the language client. type SemanticTokensProvider interface { + SemanticTokensLegendProvider + HandleSemanticTokensFullRequest(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) } @@ -50,18 +50,25 @@ type CommentTokenHighlightingStrategy interface { // for each individual token in the document, using a provided [TokenHighlightingStrategy] to determine the highlighting for each token. // It also generates semantic tokens for comments in the document, if the "comment" token type is present in the legend. type TokenBasedSemanticTokensProvider struct { - sc *service.Container - strategy TokenHighlightingStrategy - commentTypeIndexFunc func() int // Lazily initialized index of the comment token type in the legend + sc *service.Container + legendProvider SemanticTokensLegendProvider + strategy TokenHighlightingStrategy + commentTypeIndex int // Index of the "comment" token type +} + +// NewTokenBasedSemanticTokensProvider creates a new instance of [TokenBasedSemanticTokensProvider] with the given [SemanticTokensLegendProvider] and [TokenHighlightingStrategy]. +func NewTokenBasedSemanticTokensProvider(sc *service.Container, legendProvider SemanticTokensLegendProvider, strategy TokenHighlightingStrategy) SemanticTokensProvider { + commentTypeIndex := slices.Index(legendProvider.Legend().TokenTypes, string(lsp.CommentType)) + return &TokenBasedSemanticTokensProvider{ + sc: sc, + legendProvider: legendProvider, + strategy: strategy, + commentTypeIndex: commentTypeIndex, + } } -// NewTokenBasedSemanticTokensProvider creates a new instance of [TokenBasedSemanticTokensProvider] with the given [TokenHighlightingStrategy]. -func NewTokenBasedSemanticTokensProvider(sc *service.Container, strategy TokenHighlightingStrategy) SemanticTokensProvider { - return &TokenBasedSemanticTokensProvider{sc: sc, strategy: strategy, commentTypeIndexFunc: sync.OnceValue(func() int { - tokenTypes := service.MustGet[SemanticTokensLegendProvider](sc).Legend().TokenTypes - commentTypeIndex := slices.Index(tokenTypes, string(lsp.CommentType)) - return commentTypeIndex - })} +func (p *TokenBasedSemanticTokensProvider) Legend() lsp.SemanticTokensLegend { + return p.legendProvider.Legend() } func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx context.Context, params *lsp.SemanticTokensParams) (*lsp.SemanticTokens, error) { @@ -77,7 +84,7 @@ func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx c if totalLen == 0 { return nil, nil // Document is empty, no tokens found } - commentTypeIndex := p.commentTypeIndexFunc() + commentTypeIndex := p.commentTypeIndex tokenBuilder := NewSemanticTokensBuilder(doc.TextDoc.Text(nil), totalLen) highlightComment := func(commentToken core.Token) {} if commentStrategy, ok := p.strategy.(CommentTokenHighlightingStrategy); ok { diff --git a/server/server.go b/server/server.go index 6cd5870d..ce220d91 100644 --- a/server/server.go +++ b/server/server.go @@ -6,7 +6,6 @@ package server import ( "context" - "errors" "log" "golang.org/x/exp/jsonrpc2" @@ -64,16 +63,13 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para } } var semanticTokensOptions *lsp.SemanticTokensOptions - if legendProvider, err := service.Get[SemanticTokensLegendProvider](s.sc); err == nil && legendProvider != nil { + if tokenProvider, err := service.Get[SemanticTokensProvider](s.sc); err == nil && tokenProvider != nil { semanticTokensOptions = &lsp.SemanticTokensOptions{ - Legend: legendProvider.Legend(), + Legend: tokenProvider.Legend(), Full: &lsp.Or_SemanticTokensOptions_full{ Value: true, }, } - if !service.Has[SemanticTokensProvider](s.sc) { - return nil, errors.New("SemanticTokensLegendProvider is registered without a SemanticTokensProvider") - } } var renameProvider *lsp.RenameOptions if service.Has[RenameProvider](s.sc) { From b2f2426007ba923b0c18b62ffaf7e98acf1df049 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Tue, 8 Sep 2026 16:30:58 +0200 Subject: [PATCH 5/6] Rebase --- internal/grammar/semantic_tokens.go | 3 +++ server/server.go | 16 ++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/grammar/semantic_tokens.go b/internal/grammar/semantic_tokens.go index a3e6f4c1..3d9722ff 100644 --- a/internal/grammar/semantic_tokens.go +++ b/internal/grammar/semantic_tokens.go @@ -23,6 +23,8 @@ func (s *GrammarTokenHighlightingStrategy) Highlight(ctx context.Context, token switch token.Kind { case Grammar_Name_ID: accept(legendProvider.Namespace(), 0) + case Keyword_Value_StringLiteral: + accept(legendProvider.String(), 0) case Interface_Name_ID, Interface_Extends_ID_0, Interface_Extends_ID_1: @@ -31,6 +33,7 @@ func (s *GrammarTokenHighlightingStrategy) Highlight(ctx context.Context, token Token_Name_ID, CompositeRule_Name_ID, RuleCall_Rule_ID, + InfixRule_Name_ID, TokenGroup_Name_ID, TokenGroup_TokenRefs_ID: accept(legendProvider.Function(), 0) diff --git a/server/server.go b/server/server.go index ce220d91..9f829dbe 100644 --- a/server/server.go +++ b/server/server.go @@ -62,12 +62,16 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para completionOptions.TriggerCharacters = triggers.TriggerCharacters() } } - var semanticTokensOptions *lsp.SemanticTokensOptions + var semanticTokensRegistrationOptions *lsp.SemanticTokensRegistrationOptions if tokenProvider, err := service.Get[SemanticTokensProvider](s.sc); err == nil && tokenProvider != nil { - semanticTokensOptions = &lsp.SemanticTokensOptions{ - Legend: tokenProvider.Legend(), - Full: &lsp.Or_SemanticTokensOptions_full{ - Value: true, + semanticTokensRegistrationOptions = &lsp.SemanticTokensRegistrationOptions{ + SemanticTokensOptions: lsp.SemanticTokensOptions{ + Legend: tokenProvider.Legend(), + Full: &lsp.SemanticTokensOptionsFull{ + SemanticTokensFullDelta: &lsp.SemanticTokensFullDelta{ + Delta: false, + }, + }, }, } } @@ -95,7 +99,7 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para HoverProvider: optionsIf[HoverProvider, lsp.HoverOptions](s.sc), ReferencesProvider: optionsIf[ReferencesProvider, lsp.ReferenceOptions](s.sc), RenameProvider: renameProvider, - SemanticTokensProvider: semanticTokensOptions, + SemanticTokensProvider: semanticTokensRegistrationOptions, }, }, nil } From 549db58b03b6087a3b5805953ac54dd2b7c50b42 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Wed, 9 Sep 2026 19:18:21 +0200 Subject: [PATCH 6/6] Further refinements --- internal/grammar/semantic_tokens.go | 5 +- internal/grammar/semantic_tokens_test.go | 3 + server/semantic_tokens_builder.go | 145 ++++++++++------------- server/semantic_tokens_builder_test.go | 45 ++++++- server/semantic_tokens_legend.go | 4 +- server/semantic_tokens_provider.go | 67 +++++------ server/server.go | 14 +-- test/doc_fixture_lsp.go | 4 +- 8 files changed, 145 insertions(+), 142 deletions(-) diff --git a/internal/grammar/semantic_tokens.go b/internal/grammar/semantic_tokens.go index 3d9722ff..b6735f00 100644 --- a/internal/grammar/semantic_tokens.go +++ b/internal/grammar/semantic_tokens.go @@ -48,11 +48,14 @@ func (s *GrammarTokenHighlightingStrategy) Highlight(ctx context.Context, token ReferenceType_Type_ID, CrossRef_Type_ID, ParserRule_ReturnType_ID, + InfixRule_ReturnType_ID, Action_Type_ID, Action_current: accept(legendProvider.Type(), 0) case Token_Type_comment, - Token_Type_hidden: + Token_Type_hidden, + PrecedenceGroup_Associativity_left, + PrecedenceGroup_Associativity_right: accept(legendProvider.Modifier(), 0) } } diff --git a/internal/grammar/semantic_tokens_test.go b/internal/grammar/semantic_tokens_test.go index a3dbbebb..efc230f6 100644 --- a/internal/grammar/semantic_tokens_test.go +++ b/internal/grammar/semantic_tokens_test.go @@ -29,6 +29,9 @@ interface <|interface:BinaryExpression|> extends <|interface:Expression|> { <|property:Operator|>=("+" | "-") <|property:Right|>=<|function:Primary|>)* <|function:Primary|> returns <|type:Expression|>: <|property:Operator|>=<|function:ID|> +infix <|function:Binary|> on <|function:Primary|> returns <|type:Expression|>: + <|modifier:left|> "*" | "/" + > <|modifier:right|> "+" | "-"; token <|function:ID|>: /[a-zA-Z_][a-zA-Z0-9_]*/; <|modifier:hidden|> token <|function:WS|>: /[ \n\r\t]+/; diff --git a/server/semantic_tokens_builder.go b/server/semantic_tokens_builder.go index 3c3e6182..ecd0474a 100644 --- a/server/semantic_tokens_builder.go +++ b/server/semantic_tokens_builder.go @@ -51,97 +51,74 @@ type semanticTokensBuilder struct { // - length, the length of the token, // - tokenType, the token type index, // - tokenModifiers, the token modifiers bitset. - data []uint32 - text string - cursor int - prevLine int - prevChar int - currentLine int - currentChar int - // lineBreaks is reused across push calls to avoid per-token allocations - lineBreaks []int + data []uint32 + text string + // cursor is the byte offset in text, line/char the corresponding LSP (line, UTF-16 column) position + cursor, line, char int + // prevLine/prevChar is the start position of the last emitted segment + prevLine, prevChar int } -func (tokenData *semanticTokensBuilder) Data() []uint32 { - return tokenData.data +func (b *semanticTokensBuilder) Data() []uint32 { + return b.data } -func (tokenData *semanticTokensBuilder) Push(textRange core.TextRange, typeIndex, modifierIndex uint32) { - textLen := len(tokenData.text) - tokenStart := int(textRange.Start) - tokenEnd := int(textRange.End) - cursor := tokenData.cursor - currentLine := tokenData.currentLine - currentChar := tokenData.currentChar - startLine, startChar := 0, 0 - // Count line breaks within the token range as necessary - // We need to emit multiple tokens if the token spans multiple lines - lineBreaks := tokenData.lineBreaks[:0] - // Advance the cursor up to the end of the token - for tokenEnd > cursor { - if cursor >= textLen { - break - } else if cursor == tokenStart { - startLine = currentLine - startChar = currentChar - } - if c := tokenData.text[cursor]; c < utf8.RuneSelf { - // ASCII fast path: one byte, one UTF-16 code unit - if c == '\n' { - // Record the line break character position - if cursor >= tokenStart { - lineBreaks = append(lineBreaks, currentChar) - } - // New line, reset currentChar and increment currentLine - currentLine++ - currentChar = 0 - } else { - currentChar++ - } - cursor++ - continue +func (b *semanticTokensBuilder) Push(textRange core.TextRange, typeIndex, modifierIndex uint32) { + // Out-of-order or out-of-range pushes violate the contract; clamp them instead of wrapping deltas + tokenStart := min(max(int(textRange.Start), b.cursor), len(b.text)) + tokenEnd := min(max(int(textRange.End), tokenStart), len(b.text)) + for b.cursor < tokenStart { + b.step() + } + segLine, segStart := b.line, b.char + for b.cursor < tokenEnd { + lineEnd := b.char + if b.step() { + // Token spans multiple lines, emit one segment per line + b.emit(segLine, segStart, lineEnd, typeIndex, modifierIndex) + segLine, segStart = b.line, 0 } - rune, size := utf8.DecodeRuneInString(tokenData.text[cursor:]) - // Advance column by the number of UTF-16 code units for the rune - // (newlines are ASCII, so this rune can never be one) - currentChar += utf16.RuneLen(rune) - // Advance cursor by the byte size of the rune - cursor += size } - tokenData.lineBreaks = lineBreaks - lineDelta := uint32(startLine - tokenData.prevLine) - charDelta := uint32(startChar) - if lineDelta == 0 { - // If the token is on the same line as the previous token, calculate the character delta - charDelta -= uint32(tokenData.prevChar) + b.emit(segLine, segStart, b.char, typeIndex, modifierIndex) +} + +// emit appends a single-line segment [start, end) on the given line, skipping empty segments. +func (b *semanticTokensBuilder) emit(line, start, end int, typeIndex, modifierIndex uint32) { + if end <= start { + return } - if len(lineBreaks) == 0 { - // Token is on a single line, emit it directly - length := uint32(currentChar - startChar) - tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex) - // Update the previous character position for the next token - tokenData.prevChar = startChar - } else { - // Token spans multiple lines, emit a token for each line segment - // First segment: from startChar to the first line break - length := uint32(lineBreaks[0] - startChar) - tokenData.data = append(tokenData.data, lineDelta, charDelta, length, typeIndex, modifierIndex) - // Subsequent segments: from each line break to the next line break - for i := 1; i < len(lineBreaks); i++ { - // always use the full length of the line - length = uint32(lineBreaks[i]) - // Note: lineDelta is always 1, since each segment is on a new line - // charDelta is always 0, since we are starting at the beginning of the line - tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex) + charDelta := start + if line == b.prevLine { + charDelta -= b.prevChar + } + b.data = append(b.data, uint32(line-b.prevLine), uint32(charDelta), uint32(end-start), typeIndex, modifierIndex) + b.prevLine, b.prevChar = line, start +} + +// step advances the cursor by one character and reports whether it crossed a line break. +// Line breaks follow the same rules as [textdoc]: "\r\n", "\r" and "\n". +func (b *semanticTokensBuilder) step() bool { + c := b.text[b.cursor] + switch { + case c == '\r': + b.cursor++ + if b.cursor < len(b.text) && b.text[b.cursor] == '\n' { + b.cursor++ } - // Last segment: from the start of the last line to the end of the token - length = uint32(currentChar) - tokenData.data = append(tokenData.data, 1, 0, length, typeIndex, modifierIndex) - tokenData.prevChar = 0 + case c == '\n': + b.cursor++ + case c < utf8.RuneSelf: + // ASCII fast path: one byte, one UTF-16 code unit + b.cursor++ + b.char++ + return false + default: + r, size := utf8.DecodeRuneInString(b.text[b.cursor:]) + b.cursor += size + b.char += utf16.RuneLen(r) + return false } - // Update the data for the next token - tokenData.cursor = cursor - tokenData.prevLine = currentLine - tokenData.currentLine = currentLine - tokenData.currentChar = currentChar + b.line++ + b.char = 0 + return true } diff --git a/server/semantic_tokens_builder_test.go b/server/semantic_tokens_builder_test.go index df14fe66..842a4b08 100644 --- a/server/semantic_tokens_builder_test.go +++ b/server/semantic_tokens_builder_test.go @@ -6,6 +6,7 @@ package server import ( "slices" + "strings" "testing" core "typefox.dev/fastbelt" @@ -77,6 +78,43 @@ func TestLspTokenDataPush(t *testing.T) { ranges: []core.TextRange{core.NewTextRange(0, 10)}, expected: []uint32{0, 0, 2, 1, 2}, }, + { + name: "CRLF line breaks are not counted as columns", + text: "/* x\r\n y */\r\nab", + ranges: []core.TextRange{core.NewTextRange(0, 11), core.NewTextRange(13, 15)}, + expected: []uint32{ + 0, 0, 4, 1, 2, + 1, 0, 5, 1, 2, + 1, 0, 2, 1, 2, + }, + }, + { + name: "Lone CR is a line break", + text: "ab\rcd", + ranges: []core.TextRange{core.NewTextRange(0, 2), core.NewTextRange(3, 5)}, + expected: []uint32{ + 0, 0, 2, 1, 2, + 1, 0, 2, 1, 2, + }, + }, + { + name: "Empty lines inside a token emit no segments", + text: "/*\n\n*/\nx", + ranges: []core.TextRange{core.NewTextRange(0, 7), core.NewTextRange(7, 8)}, + expected: []uint32{ + 0, 0, 2, 1, 2, + 2, 0, 2, 1, 2, + 1, 0, 1, 1, 2, + }, + }, + { + name: "Out-of-order push degrades to current position", + text: "a\n\nfoo", + ranges: []core.TextRange{core.NewTextRange(3, 6), core.NewTextRange(3, 6)}, + expected: []uint32{ + 2, 0, 3, 1, 2, + }, + }, } for _, tt := range tests { @@ -95,14 +133,13 @@ func TestLspTokenDataPush(t *testing.T) { func BenchmarkLspTokenDataPush(b *testing.B) { // Build a document of 1000 lines with 4 tokens each line := "foo bar baz qux\n" - text := "" + text := strings.Repeat(line, 1000) ranges := []core.TextRange{} - for range 1000 { - offset := len(text) + for i := range 1000 { + offset := i * len(line) for start := 0; start < 15; start += 4 { ranges = append(ranges, core.NewTextRange(offset+start, offset+start+3)) } - text += line } for b.Loop() { diff --git a/server/semantic_tokens_legend.go b/server/semantic_tokens_legend.go index 06344d2f..a25acd03 100644 --- a/server/semantic_tokens_legend.go +++ b/server/semantic_tokens_legend.go @@ -29,8 +29,8 @@ type SemanticTokensLegendProvider interface { // var extraType = legendProvider.AddType("extraTokenType") // // Returns the bit value of the new token modifier within the legend // var extraModifier = legendProvider.AddModifier("extraTokenModifier") -// // Register within the service container -// service.Put[server.SemanticTokensLegendProvider](sc, legendProvider) +// // Pass it to the semantic tokens provider, which sends the legend to the client +// service.Put(sc, server.NewTokenBasedSemanticTokensProvider(sc, legendProvider, strategy)) type ExtendableSemanticTokensLegendProvider interface { SemanticTokensLegendProvider diff --git a/server/semantic_tokens_provider.go b/server/semantic_tokens_provider.go index fe275a5a..34a82119 100644 --- a/server/semantic_tokens_provider.go +++ b/server/semantic_tokens_provider.go @@ -6,10 +6,8 @@ package server import ( "context" - "errors" + "fmt" "slices" - "strconv" - "strings" core "typefox.dev/fastbelt" "typefox.dev/fastbelt/util/service" @@ -84,23 +82,31 @@ func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx c if totalLen == 0 { return nil, nil // Document is empty, no tokens found } - commentTypeIndex := p.commentTypeIndex tokenBuilder := NewSemanticTokensBuilder(doc.TextDoc.Text(nil), totalLen) - highlightComment := func(commentToken core.Token) {} - if commentStrategy, ok := p.strategy.(CommentTokenHighlightingStrategy); ok { - // Adopter has supplied a comment highlighting strategy, use that one - highlightComment = func(commentToken core.Token) { - commentStrategy.HighlightComment(ctx, commentToken, func(tokenType uint32, tokenModifier uint32) { - tokenBuilder.Push(commentToken.Range, tokenType, tokenModifier) - }) + // A single acceptor is shared by all tokens and comments to avoid a closure allocation per token. + // It accepts only the first call per token; further calls are collected as errors. + var current core.Token + var errorRanges []core.TextRange + added := false + accept := func(tokenType uint32, tokenModifier uint32) { + if added { + errorRanges = append(errorRanges, current.Range) + return } - } else if commentTypeIndex >= 0 { - // Highlight comments using the comment token type from the legend with no modifiers - highlightComment = func(commentToken core.Token) { - tokenBuilder.Push(commentToken.Range, uint32(commentTypeIndex), 0) + tokenBuilder.Push(current.Range, tokenType, tokenModifier) + added = true + } + commentStrategy, _ := p.strategy.(CommentTokenHighlightingStrategy) + highlightComment := func(commentToken core.Token) { + if commentStrategy != nil { + // Adopter has supplied a comment highlighting strategy, use that one + current, added = commentToken, false + commentStrategy.HighlightComment(ctx, commentToken, accept) + } else if p.commentTypeIndex >= 0 { + // Highlight comments using the comment token type from the legend with no modifiers + tokenBuilder.Push(commentToken.Range, uint32(p.commentTypeIndex), 0) } } - var errorRanges []core.TextRange commentIndex := 0 for _, token := range tokens { for commentIndex < len(comments) && @@ -109,35 +115,18 @@ func (p *TokenBasedSemanticTokensProvider) HandleSemanticTokensFullRequest(ctx c highlightComment(comments[commentIndex]) commentIndex++ } - added := false - p.strategy.Highlight(ctx, token, func(tokenType uint32, tokenModifier uint32) { - if !added { - tokenBuilder.Push(token.Range, tokenType, tokenModifier) - added = true - } else { - errorRanges = append(errorRanges, token.Range) - } - }) - } - // Report any tokens that were highlighted multiple times for the same range - if len(errorRanges) > 0 { - sb := strings.Builder{} - sb.WriteString("Multiple semantic tokens returned for the same token ranges: ") - for i, rng := range errorRanges { - if i > 0 { - sb.WriteString(", ") - } - sb.WriteString(strconv.Itoa(int(rng.Start))) - sb.WriteString("-") - sb.WriteString(strconv.Itoa(int(rng.End))) - } - return nil, errors.New(sb.String()) + current, added = token, false + p.strategy.Highlight(ctx, token, accept) } for commentIndex < len(comments) { // Add remaining comments after the last token highlightComment(comments[commentIndex]) commentIndex++ } + // Report any tokens that were highlighted multiple times for the same range + if len(errorRanges) > 0 { + return nil, fmt.Errorf("multiple semantic tokens returned for the same token ranges: %v", errorRanges) + } return &lsp.SemanticTokens{ Data: tokenBuilder.Data(), }, nil diff --git a/server/server.go b/server/server.go index 9f829dbe..bb626205 100644 --- a/server/server.go +++ b/server/server.go @@ -53,25 +53,19 @@ func (s *DefaultLanguageServer) Initialize(ctx context.Context, params *lsp.Para return nil, err } workspaceFolders.Value = params.WorkspaceFolders - var completionOptions *lsp.CompletionOptions - if completionProvider, err := service.Get[CompletionProvider](s.sc); err == nil && completionProvider != nil { - completionOptions = &lsp.CompletionOptions{ - ResolveProvider: false, - } + completionOptions := optionsIf[CompletionProvider, lsp.CompletionOptions](s.sc) + if completionOptions != nil { if triggers, err := service.Get[CompletionTriggers](s.sc); err == nil && triggers != nil { completionOptions.TriggerCharacters = triggers.TriggerCharacters() } } var semanticTokensRegistrationOptions *lsp.SemanticTokensRegistrationOptions if tokenProvider, err := service.Get[SemanticTokensProvider](s.sc); err == nil && tokenProvider != nil { + full := lsp.SemanticTokensOptionsFullFromBool(true) semanticTokensRegistrationOptions = &lsp.SemanticTokensRegistrationOptions{ SemanticTokensOptions: lsp.SemanticTokensOptions{ Legend: tokenProvider.Legend(), - Full: &lsp.SemanticTokensOptionsFull{ - SemanticTokensFullDelta: &lsp.SemanticTokensFullDelta{ - Delta: false, - }, - }, + Full: &full, }, } } diff --git a/test/doc_fixture_lsp.go b/test/doc_fixture_lsp.go index dd3ab922..eaed19c3 100644 --- a/test/doc_fixture_lsp.go +++ b/test/doc_fixture_lsp.go @@ -479,8 +479,8 @@ func (e *SemanticTokenExpectation) Assert(label string, expectedType uint32, exp d.fixture.t.Fatalf("fbtest: no marker with label %q", label) } for _, rng := range ranges { - startPosition := d.Document.TextDoc.PositionAt(int(rng.Start)) - endPosition := d.Document.TextDoc.PositionAt(int(rng.End)) + lspRange := rng.LspRange(d.Document.TextDoc) + startPosition, endPosition := lspRange.Start, lspRange.End if startPosition.Line != endPosition.Line { d.fixture.t.Fatalf("fbtest: AssertSemanticToken: marker %q spans multiple lines, which is not supported", label) }