Add semantic tokens support

.devcontainer/devcontainer.json

@@ -3,7 +3,7 @@
 {
 	"name": "Go",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/go:1-1.26-bookworm",
+	"image": "mcr.microsoft.com/devcontainers/go:1.26-bookworm",
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	"features": {

internal/grammar/semantic_tokens.go

@@ -0,0 +1,61 @@
+// Copyright 2026 TypeFox GmbH
+// This program and the accompanying materials are made available under the
+// terms of the MIT License, which is available in the project root.
+
+package grammar
+
+import (
+	"context"
+
+	core "typefox.dev/fastbelt"
+	"typefox.dev/fastbelt/server"
+)
+
+var legendProvider = server.NewExtendableSemanticTokensLegendProvider()
+
+type GrammarTokenHighlightingStrategy struct{}
+
+func NewGrammarTokenHighlightingStrategy() server.TokenHighlightingStrategy {
+	return &GrammarTokenHighlightingStrategy{}
+}
+
+func (s *GrammarTokenHighlightingStrategy) Highlight(ctx context.Context, token core.Token, accept server.TokenHighlightingStrategyAcceptor) {
+	switch token.Kind {
+	case Grammar_Name_ID:
+		accept(legendProvider.Namespace(), 0)
+	case Keyword_Value_StringLiteral:
+		accept(legendProvider.String(), 0)
+	case Interface_Name_ID,
+		Interface_Extends_ID_0,
+		Interface_Extends_ID_1:
+		accept(legendProvider.Interface(), 0)
+	case ParserRule_Name_ID,
+		Token_Name_ID,
+		CompositeRule_Name_ID,
+		RuleCall_Rule_ID,
+		InfixRule_Name_ID,
+		TokenGroup_Name_ID,
+		TokenGroup_TokenRefs_ID:
+		accept(legendProvider.Function(), 0)
+	case Field_Name_ID,
+		Assignment_Property_ID,
+		Action_Property_ID:
+		accept(legendProvider.Property(), 0)
+	case PrimitiveType_Type_bool,
+		PrimitiveType_Type_composite,
+		PrimitiveType_Type_string,
+		SimpleType_Type_ID,
+		ReferenceType_Type_ID,
+		CrossRef_Type_ID,
+		ParserRule_ReturnType_ID,
+		InfixRule_ReturnType_ID,
+		Action_Type_ID,
+		Action_current:
+		accept(legendProvider.Type(), 0)
+	case Token_Type_comment,
+		Token_Type_hidden,
+		PrecedenceGroup_Associativity_left,
+		PrecedenceGroup_Associativity_right:
+		accept(legendProvider.Modifier(), 0)
+	}
+}

internal/grammar/semantic_tokens_test.go

@@ -0,0 +1,51 @@
+// Copyright 2026 TypeFox GmbH
+// This program and the accompanying materials are made available under the
+// terms of the MIT License, which is available in the project root.
+
+package grammar
+
+import (
+	"testing"
+
+	"typefox.dev/fastbelt/test"
+)
+
+func TestSemanticTokensIntegration(t *testing.T) {
+	fixture := test.New(t, CreateServices())
+
+	grammarText := `<|comment:// Grammar for semantic token testing|>
+grammar <|namespace:Test|>;
+
+interface <|interface:Expression|> {}
+interface <|interface:BinaryExpression|> extends <|interface:Expression|> {
+	<|property:Left|> <|type:Expression|>
+	<|property:Operator|> <|type:string|>
+	<|property:Right|> <|type:Expression|>
+}
+
+<|function:Addition|> returns <|type:Expression|>:
+	<|function:Primary|>
+	({<|type:BinaryExpression|>.<|property:Left|>=<|type:current|>}
+		<|property:Operator|>=("+" | "-") <|property:Right|>=<|function:Primary|>)*
+<|function:Primary|> returns <|type:Expression|>:
+	<|property:Operator|>=<|function:ID|>
+infix <|function:Binary|> on <|function:Primary|> returns <|type:Expression|>:
+	<|modifier:left|> "*" | "/"
+	> <|modifier:right|> "+" | "-";
+
+token <|function:ID|>: /[a-zA-Z_][a-zA-Z0-9_]*/;
+<|modifier:hidden|> token <|function:WS|>: /[ \n\r\t]+/;
+`
+
+	doc := fixture.ParseURI(grammarText, "file:///semantic.fb")
+	doc.AssertNoParseErrors()
+	semanticTokens := doc.ExpectSemanticTokens()
+	semanticTokens.
+		Assert("namespace", legendProvider.Namespace(), 0).
+		Assert("interface", legendProvider.Interface(), 0).
+		Assert("function", legendProvider.Function(), 0).
+		Assert("property", legendProvider.Property(), 0).
+		Assert("type", legendProvider.Type(), 0).
+		Assert("modifier", legendProvider.Modifier(), 0).
+		Assert("comment", legendProvider.Comment(), 0)
+}

internal/grammar/services.go

@@ -8,6 +8,7 @@ package grammar
 
 import (
 	"typefox.dev/fastbelt/linking"
+	"typefox.dev/fastbelt/server"
 	"typefox.dev/fastbelt/textdoc"
 	"typefox.dev/fastbelt/util/service"
 	"typefox.dev/fastbelt/workspace"
@@ -29,6 +30,13 @@ func SetupServices(sc *service.Container) {
 	// Override the default scope provider
 	service.Override[FastbeltScopeProvider](sc, newScopeProviderImpl(sc))
 	service.Override(sc, newImportedSymbolsProviderImpl(sc))
+
+	// Set a semantic token highlighting strategy
+	service.Put(sc, server.NewTokenBasedSemanticTokensProvider(
+		sc,
+		legendProvider,
+		NewGrammarTokenHighlightingStrategy(),
+	))
 }
 
 // CreateServices creates a service container for the grammar language to be used in the CLI and tests.

server/semantic_tokens_builder.go

@@ -0,0 +1,124 @@
+// Copyright 2026 TypeFox GmbH
+// This program and the accompanying materials are made available under the
+// terms of the MIT License, which is available in the project root.
+
+package server
+
+import (
+	"unicode/utf16"
+	"unicode/utf8"
+
+	core "typefox.dev/fastbelt"
+)
+
+// SemanticTokensBuilder defines the interface for building semantic tokens data in the LSP format.
+// It provides methods to push individual tokens and retrieve the final data slice.
+//
+// It is recommended to build the semantic tokens data by using the [TokenBasedSemanticTokensProvider] and its
+// associated [TokenHighlightingStrategy] implementations.
+type SemanticTokensBuilder interface {
+	// Data returns the final semantic tokens data slice in the LSP format.
+	// The LSP data slice is a flat array of uint32 values, where each token is represented by five consecutive values:
+	// (1) deltaLine: token line number, relative to the previous token,
+	// (2) deltaStart: token start character, relative to the previous token,
+	// (3) length: the length of the token,
+	// (4) tokenType: the token type index in the legend, and
+	// (5) tokenModifiers: the token modifiers bitset in the legend.
+	Data() []uint32
+	// Push adds a new token to the semantic tokens data.
+	// Tokens should be pushed in the order they appear in the document, as the LSP
+	// data is based on the offset of the previous token.
+	// Due to this, this method is not thread-safe.
+	Push(textRange core.TextRange, tokenType, tokenModifiers uint32)
+}
+
+// NewSemanticTokensBuilder creates a new instance of [SemanticTokensBuilder].
+func NewSemanticTokensBuilder(text string, tokenCount int) SemanticTokensBuilder {
+	return &semanticTokensBuilder{
+		// Preallocate the data with a reasonable length
+		// Each token potentially contributes 5 uint32 values
+		// Multiline tokens can contribute more, but we can resize the slice if needed
+		data: make([]uint32, 0, tokenCount*5),
+		text: text,
+	}
+}
+
+type semanticTokensBuilder struct {
+	// data is a slice of uint32 values representing the semantic tokens data in the LSP format.
+	// Each token is represented by five consecutive values:
+	// - deltaLine, token line number, relative to the previous token,
+	// - deltaStart, token start character, relative to the previous token,
+	// - length, the length of the token,
+	// - tokenType, the token type index,
+	// - tokenModifiers, the token modifiers bitset.
+	data []uint32
+	text string
+	// cursor is the byte offset in text, line/char the corresponding LSP (line, UTF-16 column) position
+	cursor, line, char int
+	// prevLine/prevChar is the start position of the last emitted segment
+	prevLine, prevChar int
+}
+
+func (b *semanticTokensBuilder) Data() []uint32 {
+	return b.data
+}
+
+func (b *semanticTokensBuilder) Push(textRange core.TextRange, typeIndex, modifierIndex uint32) {
+	// Out-of-order or out-of-range pushes violate the contract; clamp them instead of wrapping deltas
+	tokenStart := min(max(int(textRange.Start), b.cursor), len(b.text))
+	tokenEnd := min(max(int(textRange.End), tokenStart), len(b.text))
+	for b.cursor < tokenStart {
+		b.step()
+	}
+	segLine, segStart := b.line, b.char
+	for b.cursor < tokenEnd {
+		lineEnd := b.char
+		if b.step() {
+			// Token spans multiple lines, emit one segment per line
+			b.emit(segLine, segStart, lineEnd, typeIndex, modifierIndex)
+			segLine, segStart = b.line, 0
+		}
+	}
+	b.emit(segLine, segStart, b.char, typeIndex, modifierIndex)
+}
+
+// emit appends a single-line segment [start, end) on the given line, skipping empty segments.
+func (b *semanticTokensBuilder) emit(line, start, end int, typeIndex, modifierIndex uint32) {
+	if end <= start {
+		return
+	}
+	charDelta := start
+	if line == b.prevLine {
+		charDelta -= b.prevChar
+	}
+	b.data = append(b.data, uint32(line-b.prevLine), uint32(charDelta), uint32(end-start), typeIndex, modifierIndex)
+	b.prevLine, b.prevChar = line, start
+}
+
+// step advances the cursor by one character and reports whether it crossed a line break.
+// Line breaks follow the same rules as [textdoc]: "\r\n", "\r" and "\n".
+func (b *semanticTokensBuilder) step() bool {
+	c := b.text[b.cursor]
+	switch {
+	case c == '\r':
+		b.cursor++
+		if b.cursor < len(b.text) && b.text[b.cursor] == '\n' {
+			b.cursor++
+		}
+	case c == '\n':
+		b.cursor++
+	case c < utf8.RuneSelf:
+		// ASCII fast path: one byte, one UTF-16 code unit
+		b.cursor++
+		b.char++
+		return false
+	default:
+		r, size := utf8.DecodeRuneInString(b.text[b.cursor:])
+		b.cursor += size
+		b.char += utf16.RuneLen(r)
+		return false
+	}
+	b.line++
+	b.char = 0
+	return true
+}