From c608a0dfe5a69a7d0da695d37b7e22c9432913be Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:13:40 +0000 Subject: [PATCH 1/2] feat(highlight): diff, Kotlin, Dart and SQL code blocks Fenced diff, Kotlin, Dart and SQL blocks rendered as plain text: no grammar covered them, and Swift only by opt-in. Diffs are line-oriented, so the shared seam classifies them line by line instead of with a grammar: added and removed lines get two new token types, inserted and deleted, hunk ranges the function color and file headers the constant color. Inside a hunk whose header gives line counts, "--- x" and "+++ x" are changed lines, not headers. The existing size caps apply. Kotlin (tree-sitter-kotlin 0.3.8), Dart (tree-sitter-dart 1.0.0) and SQL (@derekstride/tree-sitter-sql 0.3.11) join the opt-in grammar tier, with kt/kts and mysql/postgres/postgresql/pgsql/sqlite aliases and diff/patch fences. Host doctest coverage for the seam lands in packages/core/cpp/tests. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 5 + .yarnrc.yml | 17 +- .../android/gradle.properties | 2 +- apps/react-native-example/ios/Podfile | 2 +- docs-md/CODE_HIGHLIGHT.md | 12 +- docs-md/STYLES.md | 2 + docs/docs/android/guides/code-highlighting.md | 6 +- .../ios/api-reference/style-properties.md | 4 +- docs/docs/ios/guides/syntax-highlighting.md | 1 + .../api-reference/style-properties.md | 2 + .../rich-text-formatting/code-highlighting.md | 11 +- package.json | 9 + .../cpp/highlight/CodeBlockHighlighter.cpp | 168 ++++++++++- .../cpp/highlight/CodeBlockHighlighter.hpp | 42 +-- .../core/cpp/highlight/CodeBlockLanguages.cpp | 15 +- packages/core/cpp/tests/CMakeLists.txt | 57 ++++ .../cpp/tests/highlight/HighlightTests.cpp | 275 ++++++++++++++++++ packages/core/cpp/tests/highlight/lsan.supp | 2 + packages/enriched-markdown-android/README.md | 2 +- .../src/main/cpp/jni/HighlightJni.cpp | 5 +- .../codehighlight/CodeHighlightStyle.kt | 8 + .../markdown/codehighlight/SyntaxTokenType.kt | 2 + .../compose/CodeHighlightStyleDsl.kt | 2 + .../compose/CodeHighlightStyleDslTest.kt | 2 + packages/enriched-markdown-ios/README.md | 2 +- .../SyntaxToken.swift | 2 + .../SyntaxTokenType.swift | 2 + .../SyntaxHighlighterTests.swift | 8 + .../SyntaxHighlightingRenderingTests.swift | 5 +- .../highlight/SwiftHighlightShim.cpp | 2 +- .../markdown/styles/CodeBlockStyle.kt | 2 + .../utils/common/CodeBlockHighlighter.kt | 2 + .../ios/styles/StyleConfig.mm | 2 +- .../ios/utils/StylePropsUtils.h | 2 + .../package.json | 3 + .../src/EnrichedMarkdownNativeComponent.ts | 2 + .../EnrichedMarkdownTextNativeComponent.ts | 2 + .../src/normalizeMarkdownStyle.ts | 2 + .../src/normalizeMarkdownStyle.web.ts | 2 + .../src/types/MarkdownStyle.ts | 4 + .../src/types/MarkdownStyleInternal.ts | 2 + vendor/grammar-versions.json | 22 ++ yarn.lock | 69 +++++ 43 files changed, 741 insertions(+), 49 deletions(-) create mode 100644 packages/core/cpp/tests/highlight/HighlightTests.cpp create mode 100644 packages/core/cpp/tests/highlight/lsan.supp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dac45f461..f1bc73da1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: filters: | core: - 'packages/core/**' + - 'vendor/grammar-versions.json' - 'package.json' - '.github/**' react-native: @@ -133,6 +134,10 @@ jobs: if: matrix.build_type == 'RelWithDebInfo' run: yarn workspace @enriched-markdown/core lint-clang + # The highlight tests need the gitignored tree-sitter runtime and grammars. + - name: Vendor tree-sitter grammars + run: yarn workspace react-native-enriched-markdown vendor-grammars + - name: Build and run C++ core tests (${{ matrix.build_type }}${{ matrix.sanitize && ', sanitized' || '' }}) env: CORE_CMAKE_ARGS: -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCORE_SANITIZE=${{ matrix.sanitize }} diff --git a/.yarnrc.yml b/.yarnrc.yml index b0863d344..8709bb2f3 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -4,15 +4,24 @@ nmHoistingLimits: workspaces yarnPath: .yarn/releases/yarn-4.11.0.cjs # The tree-sitter grammars are vendored from source (see vendor/vendor-grammars.mjs); -# we never load their compiled Node bindings. @tree-sitter-grammars/tree-sitter-markdown -# and tree-sitter-swift ship a typo'd peerDependenciesMeta key ("tree_sitter" instead of -# "tree-sitter"), so Yarn treats their optional "tree-sitter" peer as required and warns. -# Re-declare it optional. (Their native builds are disabled via dependenciesMeta in package.json.) +# we never load their compiled Node bindings. @tree-sitter-grammars/tree-sitter-markdown, +# tree-sitter-swift, tree-sitter-kotlin and @derekstride/tree-sitter-sql ship a typo'd +# peerDependenciesMeta key ("tree_sitter" instead of "tree-sitter"), so Yarn treats their +# optional "tree-sitter" peer as required and warns. Re-declare it optional. (Their +# native builds are disabled via dependenciesMeta in package.json.) packageExtensions: '@tree-sitter-grammars/tree-sitter-markdown@*': peerDependenciesMeta: tree-sitter: optional: true + '@derekstride/tree-sitter-sql@*': + peerDependenciesMeta: + tree-sitter: + optional: true + 'tree-sitter-kotlin@*': + peerDependenciesMeta: + tree-sitter: + optional: true 'tree-sitter-swift@*': peerDependenciesMeta: tree-sitter: diff --git a/apps/react-native-example/android/gradle.properties b/apps/react-native-example/android/gradle.properties index 23a4188e2..b47d73760 100644 --- a/apps/react-native-example/android/gradle.properties +++ b/apps/react-native-example/android/gradle.properties @@ -46,7 +46,7 @@ edgeToEdgeEnabled=false # Set to false to disable LaTeX math rendering and remove the RaTeX dependency. #enrichedMarkdown.enableMath=false -enrichedMarkdown.codeHighlightLanguages=json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,php,ruby,c-sharp +enrichedMarkdown.codeHighlightLanguages=json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,kotlin,dart,sql,php,ruby,c-sharp # Opt out of built-in kotlin and new DSL behavior that ships with AGP 9. # Starting from AGP 10.x these opt outs will be removed. diff --git a/apps/react-native-example/ios/Podfile b/apps/react-native-example/ios/Podfile index 27976deaa..6f2b0cd66 100644 --- a/apps/react-native-example/ios/Podfile +++ b/apps/react-native-example/ios/Podfile @@ -6,7 +6,7 @@ ENV['RCT_NEW_ARCH_ENABLED'] = '1' # ENV['ENRICHED_MARKDOWN_ENABLE_MATH'] = '0' # Keep project default to full list of languages -ENV['ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES'] = 'json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,php,ruby,c-sharp' +ENV['ENRICHED_MARKDOWN_CODE_HIGHLIGHT_LANGUAGES'] = 'json,html,css,markdown,yaml,go,java,javascript,python,c,rust,bash,typescript,tsx,cpp,swift,kotlin,dart,sql,php,ruby,c-sharp' # ENV['ENRICHED_MARKDOWN_ENABLE_CODE_HIGHLIGHT'] = '0' # Resolve react_native_pods.rb with node to allow for hoisting diff --git a/docs-md/CODE_HIGHLIGHT.md b/docs-md/CODE_HIGHLIGHT.md index 44ff8b663..7cfdd17d7 100644 --- a/docs-md/CODE_HIGHLIGHT.md +++ b/docs-md/CODE_HIGHLIGHT.md @@ -27,16 +27,16 @@ def greet(name: str) -> str: comment: '#7F848E', function: '#61AFEF', type: '#E5C07B', - // ...any of the 14 token types + // ...any of the 16 token types }, }, }} /> ```` -Token colors are set through `codeBlock.syntaxColors`. The 14 token types are: `keyword`, +Token colors are set through `codeBlock.syntaxColors`. The 16 token types are: `keyword`, `operator`, `punctuation`, `string`, `number`, `constant`, `comment`, `function`, `type`, -`variable`, `property`, `tag`, `attribute`, `embedded`. Any type left unset uses the built-in +`variable`, `property`, `tag`, `attribute`, `embedded`, `inserted`, `deleted`. Any type left unset uses the built-in (GitHub-dark) palette; `operator`, `punctuation`, `variable` and `embedded` default to the code block's `color`. @@ -63,12 +63,16 @@ derive from), so the table below tracks that manifest: | Default (on) | Opt-in (heavier) | |---|---| -| json, html, css, markdown, yaml, go, java, javascript, python, c, rust, bash, typescript, tsx | cpp, swift, php, ruby, c-sharp | +| json, html, css, markdown, yaml, go, java, javascript, python, c, rust, bash, typescript, tsx | cpp, swift, kotlin, dart, sql, php, ruby, c-sharp | Grammar sources are large (generated parsers run to several MB each); the opt-in grammars include the largest ones and are only compiled when you list them explicitly. A block whose language is not compiled in simply renders as plain (uncolored) code. +`diff` and `patch` blocks are highlighted without a grammar whenever highlighting is compiled in: +added and removed lines take the `inserted` and `deleted` colors, hunk ranges the `function` color +and file headers the `constant` color. + ## Choosing languages / reducing binary size Only the grammars you compile end up in your binary, so trimming the list is the main size lever. diff --git a/docs-md/STYLES.md b/docs-md/STYLES.md index 76d296887..a765a87b0 100644 --- a/docs-md/STYLES.md +++ b/docs-md/STYLES.md @@ -343,6 +343,8 @@ Per-token foreground colors for syntax highlighting, keyed on the highlight toke | `tag` | `string` | Markup tags | | `attribute` | `string` | Markup attributes | | `embedded` | `string` | Embedded/injected language regions | +| `inserted` | `string` | Added lines in a `diff` block | +| `deleted` | `string` | Removed lines in a `diff` block | > [!NOTE] > Inside list items, code blocks (background included) indent to the item's content column. diff --git a/docs/docs/android/guides/code-highlighting.md b/docs/docs/android/guides/code-highlighting.md index 8a52bb73b..cf27ebaef 100644 --- a/docs/docs/android/guides/code-highlighting.md +++ b/docs/docs/android/guides/code-highlighting.md @@ -135,7 +135,7 @@ Tokens are cached in memory, keyed by the language and the code, in one process- ## Colors {#styling} -Each of the 14 token types takes its color from the first of: +Each of the 16 token types takes its color from the first of: 1. a color you set; 2. GitHub's dark palette, made for the default dark code block (`#1F2937`); @@ -150,6 +150,8 @@ Each of the 14 token types takes its color from the first of: | `function` | `#D2A8FF` | | `type` | `#FFA657` | | `tag` | `#7EE787` | +| `inserted` | `#3FB950` | +| `deleted` | `#F85149` | | `operator`, `punctuation`, `variable`, `embedded` | `codeBlock` color | :::caution @@ -176,7 +178,7 @@ val oneDarkStyle = markdownStyle { } ``` -The properties are `keyword`, `operator`, `punctuation`, `string`, `number`, `constant`, `comment`, `function`, `type`, `variable`, `property`, `tag`, `attribute`, and `embedded`, each a `Color?`. A token you leave unset keeps its palette color. To pick the token at runtime, index the scope with a `SyntaxTokenType` (from `com.swmansion.enriched.markdown.codehighlight`): `this[SyntaxTokenType.KEYWORD] = color`. +The properties are `keyword`, `operator`, `punctuation`, `string`, `number`, `constant`, `comment`, `function`, `type`, `variable`, `property`, `tag`, `attribute`, `embedded`, `inserted`, and `deleted`, each a `Color?`. A token you leave unset keeps its palette color. To pick the token at runtime, index the scope with a `SyntaxTokenType` (from `com.swmansion.enriched.markdown.codehighlight`): `this[SyntaxTokenType.KEYWORD] = color`. Repeating the block merges into the earlier one, exactly like the built-in blocks, so `MarkdownStyle.merge { codeHighlight { … } }` layers over a base style and changes only what it names. diff --git a/docs/docs/ios/api-reference/style-properties.md b/docs/docs/ios/api-reference/style-properties.md index 1fdbfb2bc..5ee59a3c8 100644 --- a/docs/docs/ios/api-reference/style-properties.md +++ b/docs/docs/ios/api-reference/style-properties.md @@ -672,7 +672,7 @@ struct CodeSampleView: View { } ``` -`SyntaxTokenType` has fourteen cases: `.keyword`, `.operator`, `.punctuation`, `.string`, `.number`, `.constant`, `.comment`, `.function`, `.type`, `.variable`, `.property`, `.tag`, `.attribute`, and `.embedded`. +`SyntaxTokenType` has sixteen cases: `.keyword`, `.operator`, `.punctuation`, `.string`, `.number`, `.constant`, `.comment`, `.function`, `.type`, `.variable`, `.property`, `.tag`, `.attribute`, `.embedded`, `.inserted`, and `.deleted`. Turning highlighting on layers `MarkdownTheme.syntaxHighlightingDefault` - GitHub's palette - above `MarkdownTheme.default` and below your own themes, so a `SyntaxToken` you set overrides one type and leaves the rest. The palette switches between light and dark with the color scheme: @@ -685,6 +685,8 @@ Turning highlighting on layers `MarkdownTheme.syntaxHighlightingDefault` - GitHu | `.function` | `#8250DF` | `#D2A8FF` | | `.type` | `#953800` | `#FFA657` | | `.tag` | `#116329` | `#7EE787` | +| `.inserted` | `#1A7F37` | `#3FB950` | +| `.deleted` | `#D1242F` | `#F85149` | `.operator`, `.punctuation`, `.variable`, and `.embedded` have no default color, so they keep the code block's color until you give them one. diff --git a/docs/docs/ios/guides/syntax-highlighting.md b/docs/docs/ios/guides/syntax-highlighting.md index af5fa34a0..9953ba8f7 100644 --- a/docs/docs/ios/guides/syntax-highlighting.md +++ b/docs/docs/ios/guides/syntax-highlighting.md @@ -89,6 +89,7 @@ Each run is classified as one of the `SyntaxTokenType` cases: public enum SyntaxTokenType: UInt8, CaseIterable, Sendable { case keyword, `operator`, punctuation, string, number, constant, comment case function, type, variable, property, tag, attribute, embedded + case inserted, deleted } ``` diff --git a/docs/docs/react-native/api-reference/style-properties.md b/docs/docs/react-native/api-reference/style-properties.md index fe735495f..5e6381714 100644 --- a/docs/docs/react-native/api-reference/style-properties.md +++ b/docs/docs/react-native/api-reference/style-properties.md @@ -306,6 +306,8 @@ To learn more about how code highlighting works, see the [Code highlighting](/ri | `tag` | `#7EE787` | Markup tags (e.g. HTML or JSX element names) | | `attribute` | `#79C0FF` | Markup attributes | | `embedded` | Base `color` | Embedded content such as interpolations. Inherits the base `color` when omitted | +| `inserted` | `#3FB950` | Added lines in a `diff` block | +| `deleted` | `#F85149` | Removed lines in a `diff` block | :::note Syntax colors only take visible effect when the optional syntax-highlighting module is compiled in; otherwise code blocks render uncolored. The module is native-only, so `syntaxColors` has no effect on the web build. diff --git a/docs/docs/rich-text-formatting/code-highlighting.md b/docs/docs/rich-text-formatting/code-highlighting.md index aae6e34c0..4c10e5d60 100644 --- a/docs/docs/rich-text-formatting/code-highlighting.md +++ b/docs/docs/rich-text-formatting/code-highlighting.md @@ -32,7 +32,7 @@ In the React Native package, set token colors through the `syntaxColors` map on iOS and Android only - the web build renders code blocks uncolored, so syntaxColors has no visible effect here.} /> -There are 14 token types, any of which you can color. Most of them ship with a color of their own, from a GitHub-dark palette tuned for the default `#1F2937` code background: +There are 16 token types, any of which you can color. Most of them ship with a color of their own, from a GitHub-dark palette tuned for the default `#1F2937` code background: | Token | Default | | --- | --- | @@ -43,9 +43,10 @@ There are 14 token types, any of which you can color. Most of them ship with a c | `function` | `#D2A8FF` | | `type` | `#FFA657` | | `tag` | `#7EE787` | +| `inserted`, `deleted` | `#3FB950`, `#F85149` | | `operator`, `punctuation`, `variable`, `embedded` | inherit `codeBlock.color` | -Only those last four fall back to the normal code color when left unset; the other ten keep their palette default until you override them. +Only those last four fall back to the normal code color when left unset; the other twelve keep their palette default until you override them. ## Supported languages @@ -53,11 +54,13 @@ A fence's info string maps to a grammar (for example `js` and `jsx` both select | Default (compiled in) | Opt-in (add explicitly) | |---|---| -| json, html, css, markdown, yaml, go, java, javascript, python, c, rust, bash, typescript, tsx | cpp, swift, php, ruby, c-sharp | +| json, html, css, markdown, yaml, go, java, javascript, python, c, rust, bash, typescript, tsx | cpp, swift, kotlin, dart, sql, php, ruby, c-sharp | The default set is the smaller-footprint tier; the opt-in grammars are heavier and only compiled when you list them explicitly. -Several fence tags are aliases of the same grammar - `sh`, `shell` and `zsh` all select `bash`; `cc`, `cpp` and `cxx` select `cpp`; `cs` and `csharp` select `c-sharp`; and likewise `golang`, `js`/`jsx`, `md`, `py`, `rb`, `rs`, `ts`, `yml`. A handful of tags are recognized as **labels only** and never highlighted: `dockerfile`, `graphql`, `objc`, `objectivec`, `scss`, `sql`, `toml`, `xml`. +Several fence tags are aliases of the same grammar - `sh`, `shell` and `zsh` all select `bash`; `cc`, `cpp` and `cxx` select `cpp`; `cs` and `csharp` select `c-sharp`; `kt` and `kts` select `kotlin`; `mysql`, `postgres`, `postgresql`, `pgsql` and `sqlite` select `sql`; and likewise `golang`, `js`/`jsx`, `md`, `py`, `rb`, `rs`, `ts`, `yml`. A handful of tags are recognized as **labels only** and never highlighted: `dockerfile`, `graphql`, `objc`, `objectivec`, `scss`, `toml`, `xml`. + +`diff` and `patch` blocks need no grammar: whenever highlighting is compiled in, added and removed lines take the `inserted` and `deleted` colors, hunk ranges (`@@ -1,2 +1,3 @@`) the `function` color, and file headers the `constant` color. Context lines keep the code color. :::caution The alias table and the `codeHighlightLanguages` ids are **not** the same list. `c-sharp` is the grammar id you must write in `codeHighlightLanguages`, but it is not a valid fence tag - in Markdown, write `cs` or `csharp`. diff --git a/package.json b/package.json index f1aa9ee92..ed24e8d74 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,9 @@ }, "packageManager": "yarn@4.11.0", "dependenciesMeta": { + "@derekstride/tree-sitter-sql": { + "built": false + }, "@tree-sitter-grammars/tree-sitter-markdown": { "built": false }, @@ -105,6 +108,9 @@ "tree-sitter-css": { "built": false }, + "tree-sitter-dart": { + "built": false + }, "tree-sitter-go": { "built": false }, @@ -120,6 +126,9 @@ "tree-sitter-json": { "built": false }, + "tree-sitter-kotlin": { + "built": false + }, "tree-sitter-php": { "built": false }, diff --git a/packages/core/cpp/highlight/CodeBlockHighlighter.cpp b/packages/core/cpp/highlight/CodeBlockHighlighter.cpp index c3a97f6b0..5ee85d7b4 100644 --- a/packages/core/cpp/highlight/CodeBlockHighlighter.cpp +++ b/packages/core/cpp/highlight/CodeBlockHighlighter.cpp @@ -22,6 +22,7 @@ std::vector highlightCode(const std::string & /*code*/, const st #include #include +#include #include #include #include @@ -157,6 +158,161 @@ std::vector buildUtf16PrefixMap(const std::string &code) { return prefix; } +bool startsWith(const std::string &code, size_t at, size_t end, const char *prefix) { + const size_t length = std::strlen(prefix); + return end - at >= length && code.compare(at, length, prefix) == 0; +} + +// Reads a hunk range count ("12" or "12,3") at pos, advancing past it. The +// count after the comma is the line count; without one the range spans 1 line. +bool readRange(const std::string &code, size_t &pos, size_t end, long &count) { + size_t digits = pos; + while (digits < end && std::isdigit(static_cast(code[digits]))) { + ++digits; + } + if (digits == pos) { + return false; + } + count = 1; + pos = digits; + if (pos < end && code[pos] == ',') { + size_t countStart = ++pos; + while (pos < end && std::isdigit(static_cast(code[pos]))) { + ++pos; + } + if (pos == countStart || pos - countStart > 9) { + return false; + } + count = std::stol(code.substr(countStart, pos - countStart)); + } + return true; +} + +// Parses "@@ -a,b +c,d @@" into the old and new line counts the hunk spans. +bool parseHunkHeader(const std::string &code, size_t at, size_t end, long &oldLines, long &newLines) { + size_t pos = at + 2; + if (!startsWith(code, pos, end, " -")) { + return false; + } + pos += 2; + if (!readRange(code, pos, end, oldLines) || !startsWith(code, pos, end, " +")) { + return false; + } + pos += 2; + return readRange(code, pos, end, newLines) && startsWith(code, pos, end, " @@"); +} + +bool isFileHeader(const std::string &code, size_t at, size_t end) { + static constexpr const char *kPrefixes[] = { + "diff ", "index ", "new file mode ", "deleted file mode ", + "old mode ", "new mode ", "similarity index ", "dissimilarity index ", + "rename from ", "rename to ", "copy from ", "copy to ", + "Binary files ", "Only in ", + }; + for (const char *prefix : kPrefixes) { + if (startsWith(code, at, end, prefix)) { + return true; + } + } + return false; +} + +// Unified diffs are line-oriented, so they are classified line by line rather +// than parsed with a grammar. Inside a hunk whose header gave line counts, +// "--- x" and "+++ x" are a removed "-- x" and an added "++ x"; elsewhere a +// "--- " line directly followed by a "+++ " line is a file header pair. Lines +// keep their base color when they are context or not diff syntax at all. +std::vector highlightDiff(const std::string &code) { + struct Line { + size_t start; + size_t end; // Excludes the line break. + }; + std::vector lines; + for (size_t start = 0; start <= code.size();) { + size_t newline = code.find('\n', start); + size_t next = newline == std::string::npos ? code.size() + 1 : newline + 1; + size_t end = newline == std::string::npos ? code.size() : newline; + if (end > start && code[end - 1] == '\r') { + --end; + } + lines.push_back({start, end}); + start = next; + } + + struct Span { + size_t start; + size_t end; + HighlightTokenType type; + }; + std::vector spans; + long oldLeft = 0; + long newLeft = 0; + for (size_t i = 0; i < lines.size(); ++i) { + const size_t start = lines[i].start; + const size_t end = lines[i].end; + const char first = end > start ? code[start] : '\0'; + + if (oldLeft > 0 || newLeft > 0) { + if (first == '-' && oldLeft > 0) { + spans.push_back({start, end, HighlightTokenType::Deleted}); + --oldLeft; + continue; + } + if (first == '+' && newLeft > 0) { + spans.push_back({start, end, HighlightTokenType::Inserted}); + --newLeft; + continue; + } + if (first == ' ' || first == '\0') { + --oldLeft; + --newLeft; + continue; + } + if (first == '\\') { + spans.push_back({start, end, HighlightTokenType::Comment}); + continue; + } + oldLeft = newLeft = 0; + } + + if (startsWith(code, start, end, "@@")) { + size_t close = code.find(" @@", start + 2); + size_t rangeEnd = close != std::string::npos && close < end ? close + 3 : end; + spans.push_back({start, rangeEnd, HighlightTokenType::Function}); + long oldLines = 0; + long newLines = 0; + if (parseHunkHeader(code, start, end, oldLines, newLines)) { + oldLeft = oldLines; + newLeft = newLines; + } + } else if (startsWith(code, start, end, "--- ") && i + 1 < lines.size() && + startsWith(code, lines[i + 1].start, lines[i + 1].end, "+++ ")) { + spans.push_back({start, end, HighlightTokenType::Constant}); + spans.push_back({lines[i + 1].start, lines[i + 1].end, HighlightTokenType::Constant}); + ++i; + } else if (isFileHeader(code, start, end)) { + spans.push_back({start, end, HighlightTokenType::Constant}); + } else if (first == '-') { + spans.push_back({start, end, HighlightTokenType::Deleted}); + } else if (first == '+') { + spans.push_back({start, end, HighlightTokenType::Inserted}); + } else if (first == '\\') { + spans.push_back({start, end, HighlightTokenType::Comment}); + } + } + + if (spans.empty()) { + return {}; + } + std::vector u16 = buildUtf16PrefixMap(code); + std::vector tokens; + tokens.reserve(spans.size()); + for (const Span &span : spans) { + tokens.push_back({u16[span.start], u16[span.end], span.type}); + } + return tokens; +} + using ParserPtr = std::unique_ptr; using TreePtr = std::unique_ptr; using CursorPtr = std::unique_ptr; @@ -178,10 +334,6 @@ std::vector highlightCode(const std::string &code, const std::st if (grammarId.empty()) { return {}; } - const GrammarEntry *grammar = findGrammar(grammarId.c_str()); - if (grammar == nullptr) { - return {}; - } if (code.size() > kMaxBytes) { return {}; @@ -193,6 +345,14 @@ std::vector highlightCode(const std::string &code, const std::st } } + if (grammarId == "diff") { + return highlightDiff(code); + } + const GrammarEntry *grammar = findGrammar(grammarId.c_str()); + if (grammar == nullptr) { + return {}; + } + ParserPtr parser(ts_parser_new(), ts_parser_delete); if (!parser) { return {}; diff --git a/packages/core/cpp/highlight/CodeBlockHighlighter.hpp b/packages/core/cpp/highlight/CodeBlockHighlighter.hpp index 23770b3f3..1391a952e 100644 --- a/packages/core/cpp/highlight/CodeBlockHighlighter.hpp +++ b/packages/core/cpp/highlight/CodeBlockHighlighter.hpp @@ -25,7 +25,8 @@ // // Token types follow tree-sitter's standard highlight capture names, // flattened to one level. Values are explicit because they cross the JNI -// boundary as plain integers. +// boundary as plain integers. Diffs have no grammar: their lines are +// classified directly, as additions, deletions, hunk ranges and headers. // // Both markdown flavors call this seam: the github flavor highlights the code // string in its container view, and the commonmark renderers @@ -35,32 +36,35 @@ namespace Markdown { enum class HighlightTokenType : uint8_t { - Keyword = 0, - Operator = 1, - Punctuation = 2, - String = 3, - Number = 4, - Constant = 5, - Comment = 6, - Function = 7, - Type = 8, - Variable = 9, - Property = 10, - Tag = 11, - Attribute = 12, - Embedded = 13, + Keyword = 0, + Operator = 1, + Punctuation = 2, + String = 3, + Number = 4, + Constant = 5, + Comment = 6, + Function = 7, + Type = 8, + Variable = 9, + Property = 10, + Tag = 11, + Attribute = 12, + Embedded = 13, + // Added and removed lines of a diff. + Inserted = 14, + Deleted = 15, }; struct HighlightToken { - uint32_t start; - uint32_t end; - HighlightTokenType type; + uint32_t start; + uint32_t end; + HighlightTokenType type; }; // code is the UTF-8 code block content without the fence lines; language is // the fence info string (for example "python"), empty when absent. Returns an // empty vector whenever highlighting is unavailable (module compiled out, // unknown language, parse failure). -std::vector highlightCode(const std::string& code, const std::string& language); +std::vector highlightCode(const std::string &code, const std::string &language); } // namespace Markdown diff --git a/packages/core/cpp/highlight/CodeBlockLanguages.cpp b/packages/core/cpp/highlight/CodeBlockLanguages.cpp index 031d25a35..89646ecf8 100644 --- a/packages/core/cpp/highlight/CodeBlockLanguages.cpp +++ b/packages/core/cpp/highlight/CodeBlockLanguages.cpp @@ -18,7 +18,7 @@ struct LanguageName { }; // Sorted by key; both lookups below binary-search this table. -constexpr std::array kLanguageNames{{ +constexpr std::array kLanguageNames{{ {"bash", "Bash", "bash"}, {"c", "C", "c"}, {"cc", "C++", "cpp"}, @@ -27,6 +27,8 @@ constexpr std::array kLanguageNames{{ {"csharp", "C#", "c-sharp"}, {"css", "CSS", "css"}, {"cxx", "C++", "cpp"}, + {"dart", "Dart", "dart"}, + {"diff", "Diff", "diff"}, {"dockerfile", "Dockerfile", ""}, {"go", "Go", "go"}, {"golang", "Go", "go"}, @@ -37,11 +39,19 @@ constexpr std::array kLanguageNames{{ {"js", "JavaScript", "javascript"}, {"json", "JSON", "json"}, {"jsx", "JSX", "javascript"}, + {"kotlin", "Kotlin", "kotlin"}, + {"kt", "Kotlin", "kotlin"}, + {"kts", "Kotlin", "kotlin"}, {"markdown", "Markdown", "markdown"}, {"md", "Markdown", "markdown"}, + {"mysql", "MySQL", "sql"}, {"objc", "Objective-C", ""}, {"objectivec", "Objective-C", ""}, + {"patch", "Patch", "diff"}, + {"pgsql", "PostgreSQL", "sql"}, {"php", "PHP", "php"}, + {"postgres", "PostgreSQL", "sql"}, + {"postgresql", "PostgreSQL", "sql"}, {"py", "Python", "python"}, {"python", "Python", "python"}, {"rb", "Ruby", "ruby"}, @@ -51,7 +61,8 @@ constexpr std::array kLanguageNames{{ {"scss", "SCSS", ""}, {"sh", "Shell", "bash"}, {"shell", "Shell", "bash"}, - {"sql", "SQL", ""}, + {"sql", "SQL", "sql"}, + {"sqlite", "SQLite", "sql"}, {"swift", "Swift", "swift"}, {"toml", "TOML", ""}, {"ts", "TypeScript", "typescript"}, diff --git a/packages/core/cpp/tests/CMakeLists.txt b/packages/core/cpp/tests/CMakeLists.txt index 825cd9023..1e27df602 100644 --- a/packages/core/cpp/tests/CMakeLists.txt +++ b/packages/core/cpp/tests/CMakeLists.txt @@ -77,3 +77,60 @@ target_compile_options(enriched_markdown_latex PRIVATE -Wall -Wextra -Wpedantic) add_core_test(repair enriched_markdown_repair) add_core_test(latex enriched_markdown_latex) + +# --- Code-block syntax highlighting ----------------------------------------- +# +# Needs the gitignored tree-sitter runtime and grammars, restored by +# `node vendor/vendor-grammars.mjs`. Only the grammars these tests exercise are +# compiled, through a registry generated for exactly that set. +set(CORE_HIGHLIGHT_LANGUAGES "python;swift;kotlin;dart;sql" CACHE STRING + "Grammar ids compiled into highlight_tests") +set(HIGHLIGHT_ROOT "${CPP_ROOT}/highlight") +set(HIGHLIGHT_VENDOR "${HIGHLIGHT_ROOT}/vendor") +if(EXISTS "${HIGHLIGHT_VENDOR}/tree-sitter/src/lib.c") + enable_language(C) + set(CMAKE_C_STANDARD 11) + find_program(NODE_EXECUTABLE node REQUIRED) + set(HIGHLIGHT_REGISTRY_DIR "${CMAKE_CURRENT_BINARY_DIR}/highlight-registry") + list(JOIN CORE_HIGHLIGHT_LANGUAGES "," HIGHLIGHT_LANGUAGE_LIST) + execute_process( + COMMAND "${NODE_EXECUTABLE}" "${CPP_ROOT}/../../../vendor/gen-registry.mjs" + --vendor-dir "${HIGHLIGHT_VENDOR}" --languages "${HIGHLIGHT_LANGUAGE_LIST}" --out "${HIGHLIGHT_REGISTRY_DIR}" + RESULT_VARIABLE HIGHLIGHT_REGISTRY_RESULT) + if(NOT HIGHLIGHT_REGISTRY_RESULT EQUAL 0) + message(FATAL_ERROR "gen-registry.mjs failed for [${HIGHLIGHT_LANGUAGE_LIST}]; run node vendor/vendor-grammars.mjs") + endif() + + set(HIGHLIGHT_GRAMMAR_SOURCES "") + foreach(lang IN LISTS CORE_HIGHLIGHT_LANGUAGES) + set(gdir "${HIGHLIGHT_VENDOR}/grammars/${lang}") + set(gsrcs "${gdir}/parser.c") + if(EXISTS "${gdir}/scanner.c") + list(APPEND gsrcs "${gdir}/scanner.c") + endif() + add_library(ts_grammar_${lang} OBJECT ${gsrcs}) + target_include_directories(ts_grammar_${lang} PRIVATE "${gdir}") + # Generated parser tables are not ours to warn about, and their size makes + # sanitizer instrumentation the slowest part of the build for no benefit. + target_compile_options(ts_grammar_${lang} PRIVATE -w -fno-sanitize=all) + list(APPEND HIGHLIGHT_GRAMMAR_SOURCES $) + endforeach() + + add_library(enriched_markdown_highlight STATIC + "${HIGHLIGHT_VENDOR}/tree-sitter/src/lib.c" + "${HIGHLIGHT_ROOT}/CodeBlockHighlighter.cpp" + "${HIGHLIGHT_ROOT}/CodeBlockLanguages.cpp" + "${HIGHLIGHT_REGISTRY_DIR}/generated_registry.cpp" + ${HIGHLIGHT_GRAMMAR_SOURCES}) + set_source_files_properties("${HIGHLIGHT_VENDOR}/tree-sitter/src/lib.c" PROPERTIES COMPILE_OPTIONS "-w") + target_include_directories(enriched_markdown_highlight + PUBLIC "${HIGHLIGHT_ROOT}" "${HIGHLIGHT_VENDOR}/tree-sitter/include" + PRIVATE "${HIGHLIGHT_VENDOR}/tree-sitter/src" "${HIGHLIGHT_REGISTRY_DIR}") + target_compile_definitions(enriched_markdown_highlight PRIVATE ENRICHED_MARKDOWN_CODE_HIGHLIGHT=1) + + add_core_test(highlight enriched_markdown_highlight) + set_tests_properties(highlight_tests PROPERTIES + ENVIRONMENT "LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/highlight/lsan.supp") +else() + message(STATUS "highlight_tests skipped: tree-sitter is not vendored; run node vendor/vendor-grammars.mjs") +endif() diff --git a/packages/core/cpp/tests/highlight/HighlightTests.cpp b/packages/core/cpp/tests/highlight/HighlightTests.cpp new file mode 100644 index 000000000..584e59b1a --- /dev/null +++ b/packages/core/cpp/tests/highlight/HighlightTests.cpp @@ -0,0 +1,275 @@ +#include +#include +#include + +#include "CodeBlockHighlighter.hpp" +#include "CodeBlockLanguages.hpp" +#include "doctest/doctest.h" + +namespace Markdown { +bool operator==(const HighlightToken &a, const HighlightToken &b) { + return a.start == b.start && a.end == b.end && a.type == b.type; +} +} // namespace Markdown + +using Markdown::highlightCode; +using Markdown::HighlightToken; +using Markdown::HighlightTokenType; + +namespace { + +// Encodes code as UTF-16 so token ranges, which are UTF-16 offsets, can be +// sliced back into text and compared as UTF-8. +std::u16string toUtf16(const std::string &utf8) { + std::u16string out; + size_t i = 0; + while (i < utf8.size()) { + unsigned char c = static_cast(utf8[i]); + uint32_t cp; + size_t len; + if (c < 0x80) { + cp = c; + len = 1; + } else if ((c >> 5) == 0x6) { + cp = c & 0x1F; + len = 2; + } else if ((c >> 4) == 0xE) { + cp = c & 0x0F; + len = 3; + } else { + cp = c & 0x07; + len = 4; + } + for (size_t k = 1; k < len; ++k) { + cp = (cp << 6) | (static_cast(utf8[i + k]) & 0x3F); + } + if (cp >= 0x10000) { + cp -= 0x10000; + out.push_back(static_cast(0xD800 + (cp >> 10))); + out.push_back(static_cast(0xDC00 + (cp & 0x3FF))); + } else { + out.push_back(static_cast(cp)); + } + i += len; + } + return out; +} + +std::string toUtf8(const std::u16string &utf16) { + std::string out; + for (size_t i = 0; i < utf16.size(); ++i) { + uint32_t cp = utf16[i]; + if (cp >= 0xD800 && cp < 0xDC00 && i + 1 < utf16.size()) { + cp = 0x10000 + ((cp - 0xD800) << 10) + (utf16[++i] - 0xDC00); + } + if (cp < 0x80) { + out.push_back(static_cast(cp)); + } else if (cp < 0x800) { + out.push_back(static_cast(0xC0 | (cp >> 6))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + } else if (cp < 0x10000) { + out.push_back(static_cast(0xE0 | (cp >> 12))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + } else { + out.push_back(static_cast(0xF0 | (cp >> 18))); + out.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + } + } + return out; +} + +// The text of every token of type, in order. +std::vector texts(const std::string &code, const std::string &language, HighlightTokenType type) { + const std::u16string units = toUtf16(code); + std::vector out; + for (const HighlightToken &token : highlightCode(code, language)) { + if (token.type == type) { + out.push_back(toUtf8(units.substr(token.start, token.end - token.start))); + } + } + return out; +} + +bool contains(const std::vector &values, const std::string &value) { + for (const std::string &v : values) { + if (v == value) { + return true; + } + } + return false; +} + +void checkOrderedAndInBounds(const std::string &code, const std::string &language) { + const std::vector tokens = highlightCode(code, language); + const uint32_t length = static_cast(toUtf16(code).size()); + REQUIRE_FALSE(tokens.empty()); + uint32_t previousEnd = 0; + for (const HighlightToken &token : tokens) { + CHECK(token.start >= previousEnd); + CHECK(token.end > token.start); + CHECK(token.end <= length); + previousEnd = token.end; + } +} + +} // namespace + +TEST_CASE("fence aliases resolve to their grammar and display name") { + struct Alias { + const char *fence; + const char *grammar; + const char *name; + }; + const Alias aliases[] = { + {"kotlin", "kotlin", "Kotlin"}, {"kt", "kotlin", "Kotlin"}, {"KTS", "kotlin", "Kotlin"}, + {"swift", "swift", "Swift"}, {"dart", "dart", "Dart"}, {"sql", "sql", "SQL"}, + {"mysql", "sql", "MySQL"}, {"postgres", "sql", "PostgreSQL"}, {"postgresql", "sql", "PostgreSQL"}, + {"pgsql", "sql", "PostgreSQL"}, {"sqlite", "sql", "SQLite"}, {"diff", "diff", "Diff"}, + {"patch", "diff", "Patch"}, {"Diff", "diff", "Diff"}, + }; + for (const Alias &alias : aliases) { + CAPTURE(alias.fence); + CHECK(Markdown::canonicalGrammarId(alias.fence) == alias.grammar); + CHECK(Markdown::displayNameForLanguage(alias.fence) == alias.name); + } +} + +TEST_CASE("languages without a grammar stay plain") { + CHECK(highlightCode("let x = 1", "klingon").empty()); + CHECK(highlightCode("let x = 1", "").empty()); + CHECK(highlightCode("[a]\nb = 1", "toml").empty()); + CHECK(Markdown::displayNameForLanguage("klingon") == "Klingon"); + // A grammar the language table knows but this build did not compile. + CHECK(highlightCode("x := 1", "go").empty()); +} + +TEST_CASE("kotlin") { + const std::string code = + "fun greet(name: String): String {\n" + " val n = 42\n" + " return \"hi $name\" // note\n" + "}"; + CHECK(contains(texts(code, "kotlin", HighlightTokenType::Keyword), "fun")); + CHECK(contains(texts(code, "kotlin", HighlightTokenType::Keyword), "val")); + CHECK(contains(texts(code, "kotlin", HighlightTokenType::Function), "greet")); + CHECK(contains(texts(code, "kotlin", HighlightTokenType::Type), "String")); + CHECK(contains(texts(code, "kotlin", HighlightTokenType::Number), "42")); + CHECK(contains(texts(code, "kotlin", HighlightTokenType::Comment), "// note")); + CHECK(highlightCode(code, "kt") == highlightCode(code, "kotlin")); + checkOrderedAndInBounds(code, "kotlin"); +} + +TEST_CASE("swift") { + const std::string code = + "func greet(name: String) -> String {\n" + " let n = 42\n" + " return \"hi\" // note\n" + "}"; + CHECK(contains(texts(code, "swift", HighlightTokenType::Keyword), "func")); + CHECK(contains(texts(code, "swift", HighlightTokenType::Keyword), "let")); + CHECK(contains(texts(code, "swift", HighlightTokenType::Keyword), "return")); + CHECK(contains(texts(code, "swift", HighlightTokenType::Function), "greet")); + CHECK(contains(texts(code, "swift", HighlightTokenType::Number), "42")); + CHECK(contains(texts(code, "swift", HighlightTokenType::Comment), "// note")); + checkOrderedAndInBounds(code, "swift"); +} + +TEST_CASE("dart") { + const std::string code = + "void main() {\n" + " var n = 42;\n" + " print('hi'); // note\n" + "}"; + CHECK(contains(texts(code, "dart", HighlightTokenType::Keyword), "var")); + CHECK(contains(texts(code, "dart", HighlightTokenType::Type), "void")); + CHECK(contains(texts(code, "dart", HighlightTokenType::String), "'hi'")); + CHECK(contains(texts(code, "dart", HighlightTokenType::Number), "42")); + CHECK(contains(texts(code, "dart", HighlightTokenType::Comment), "// note")); + checkOrderedAndInBounds(code, "dart"); +} + +TEST_CASE("sql") { + const std::string code = "SELECT id, name FROM users WHERE id = 42 AND name = 'a'; -- note"; + CHECK(contains(texts(code, "sql", HighlightTokenType::Keyword), "SELECT")); + CHECK(contains(texts(code, "sql", HighlightTokenType::Keyword), "FROM")); + CHECK(contains(texts(code, "sql", HighlightTokenType::Keyword), "WHERE")); + CHECK(contains(texts(code, "sql", HighlightTokenType::Number), "42")); + CHECK(contains(texts(code, "sql", HighlightTokenType::Comment), "-- note")); + CHECK(highlightCode(code, "postgresql") == highlightCode(code, "sql")); + checkOrderedAndInBounds(code, "sql"); +} + +TEST_CASE("diff colors file headers, hunk ranges, additions and deletions") { + const std::string code = + "diff --git a/app.kt b/app.kt\n" + "index 83db48f..bf269f4 100644\n" + "--- a/app.kt\n" + "+++ b/app.kt\n" + "@@ -1,4 +1,4 @@ fun main() {\n" + " context\n" + "-removed\n" + "--- removed, not a header\n" + "+added\n" + "+++ added, not a header\n" + "\\ No newline at end of file"; + using T = HighlightTokenType; + CHECK(texts(code, "diff", T::Constant) == std::vector{"diff --git a/app.kt b/app.kt", + "index 83db48f..bf269f4 100644", "--- a/app.kt", + "+++ b/app.kt"}); + CHECK(texts(code, "diff", T::Function) == std::vector{"@@ -1,4 +1,4 @@"}); + CHECK(texts(code, "diff", T::Deleted) == std::vector{"-removed", "--- removed, not a header"}); + CHECK(texts(code, "diff", T::Inserted) == std::vector{"+added", "+++ added, not a header"}); + CHECK(texts(code, "diff", T::Comment) == std::vector{"\\ No newline at end of file"}); + CHECK(highlightCode(code, "patch") == highlightCode(code, "diff")); + checkOrderedAndInBounds(code, "diff"); + + // Context lines and the hunk's trailing function context keep the base color. + for (const HighlightToken &token : highlightCode(code, "diff")) { + const std::string text = toUtf8(toUtf16(code).substr(token.start, token.end - token.start)); + CHECK(text.find("context") == std::string::npos); + CHECK(text.find("fun main") == std::string::npos); + } +} + +TEST_CASE("diff without hunk headers still colors changed lines") { + const std::string code = "--- before\n+++ after\n unchanged\n-old\n+new\r\n+\n-\n"; + using T = HighlightTokenType; + CHECK(texts(code, "diff", T::Constant) == std::vector{"--- before", "+++ after"}); + CHECK(texts(code, "diff", T::Deleted) == std::vector{"-old", "-"}); + CHECK(texts(code, "diff", T::Inserted) == std::vector{"+new", "+"}); +} + +TEST_CASE("token ranges are UTF-16 offsets") { + using T = HighlightTokenType; + const std::string diff = "-caf\xC3\xA9 \xF0\x9F\x8E\x89\n+caf\xC3\xA9 \xF0\x9F\x9A\x80 done\n same"; + CHECK(texts(diff, "diff", T::Deleted) == std::vector{"-caf\xC3\xA9 \xF0\x9F\x8E\x89"}); + CHECK(texts(diff, "diff", T::Inserted) == std::vector{"+caf\xC3\xA9 \xF0\x9F\x9A\x80 done"}); + + const std::string kotlin = "val s = \"\xF0\x9F\x8E\x89\xC3\xA9\" // \xF0\x9F\x8E\x89\nval n = 1"; + CHECK(contains(texts(kotlin, "kotlin", T::Comment), "// \xF0\x9F\x8E\x89")); + CHECK(contains(texts(kotlin, "kotlin", T::Number), "1")); + checkOrderedAndInBounds(kotlin, "kotlin"); +} + +TEST_CASE("code past the size caps renders plain") { + const std::string line = "+val x = 1\n"; + std::string underCap; + std::string overLineCap; + for (int i = 0; i < 1000; ++i) { + underCap += line; + } + for (int i = 0; i < 2500; ++i) { + overLineCap += line; + } + const std::string overByteCap(50 * 1024 + 1, 'x'); + + for (const char *language : {"diff", "kotlin", "sql"}) { + CAPTURE(language); + CHECK_FALSE(highlightCode(underCap, language).empty()); + CHECK(highlightCode(overLineCap, language).empty()); + CHECK(highlightCode("+" + overByteCap, language).empty()); + } +} diff --git a/packages/core/cpp/tests/highlight/lsan.supp b/packages/core/cpp/tests/highlight/lsan.supp new file mode 100644 index 000000000..e2b8241a4 --- /dev/null +++ b/packages/core/cpp/tests/highlight/lsan.supp @@ -0,0 +1,2 @@ +# Compiled highlight queries are cached for the life of the process on purpose. +leak:queryForLanguage diff --git a/packages/enriched-markdown-android/README.md b/packages/enriched-markdown-android/README.md index 8febdd1ca..111b231ce 100644 --- a/packages/enriched-markdown-android/README.md +++ b/packages/enriched-markdown-android/README.md @@ -730,7 +730,7 @@ markdownStyle { } ``` -The properties are `keyword`, `operator`, `punctuation`, `string`, `number`, `constant`, `comment`, `function`, `type`, `variable`, `property`, `tag`, `attribute` and `embedded`; `this[SyntaxTokenType.KEYWORD] = color` sets a type chosen at runtime. Repeating the block merges into the earlier one, exactly like the built-in blocks, so `MarkdownStyle.merge { codeHighlight { … } }` layers over a base style. +The properties are `keyword`, `operator`, `punctuation`, `string`, `number`, `constant`, `comment`, `function`, `type`, `variable`, `property`, `tag`, `attribute`, `embedded`, `inserted` and `deleted`; `this[SyntaxTokenType.KEYWORD] = color` sets a type chosen at runtime. Repeating the block merges into the earlier one, exactly like the built-in blocks, so `MarkdownStyle.merge { codeHighlight { … } }` layers over a base style. ### Size and licenses diff --git a/packages/enriched-markdown-android/code-highlight/src/main/cpp/jni/HighlightJni.cpp b/packages/enriched-markdown-android/code-highlight/src/main/cpp/jni/HighlightJni.cpp index f69c285e3..a7a9e0917 100644 --- a/packages/enriched-markdown-android/code-highlight/src/main/cpp/jni/HighlightJni.cpp +++ b/packages/enriched-markdown-android/code-highlight/src/main/cpp/jni/HighlightJni.cpp @@ -18,7 +18,8 @@ constexpr HighlightTokenType kTokenTypeOrder[] = { HighlightTokenType::String, HighlightTokenType::Number, HighlightTokenType::Constant, HighlightTokenType::Comment, HighlightTokenType::Function, HighlightTokenType::Type, HighlightTokenType::Variable, HighlightTokenType::Property, HighlightTokenType::Tag, - HighlightTokenType::Attribute, HighlightTokenType::Embedded, + HighlightTokenType::Attribute, HighlightTokenType::Embedded, HighlightTokenType::Inserted, + HighlightTokenType::Deleted, }; constexpr bool tokenTypeOrderMatches() { @@ -30,7 +31,7 @@ constexpr bool tokenTypeOrderMatches() { return true; } -static_assert(sizeof(kTokenTypeOrder) / sizeof(kTokenTypeOrder[0]) == 14, +static_assert(sizeof(kTokenTypeOrder) / sizeof(kTokenTypeOrder[0]) == 16, "HighlightTokenType count changed; update SyntaxTokenType.kt to match"); static_assert(tokenTypeOrderMatches(), "HighlightTokenType order changed; update SyntaxTokenType.kt to match"); diff --git a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlightStyle.kt b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlightStyle.kt index b112297c7..7adbc67a4 100644 --- a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlightStyle.kt +++ b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlightStyle.kt @@ -25,6 +25,8 @@ data class CodeHighlightStyle( val tag: Int? = null, val attribute: Int? = null, val embedded: Int? = null, + val inserted: Int? = null, + val deleted: Int? = null, ) { /** The color for [type], or null when this style leaves it unset. */ operator fun get(type: SyntaxTokenType): Int? = @@ -43,6 +45,8 @@ data class CodeHighlightStyle( SyntaxTokenType.TAG -> tag SyntaxTokenType.ATTRIBUTE -> attribute SyntaxTokenType.EMBEDDED -> embedded + SyntaxTokenType.INSERTED -> inserted + SyntaxTokenType.DELETED -> deleted } /** This style's colors, with [fallback]'s for every type this one leaves unset. */ @@ -66,6 +70,8 @@ data class CodeHighlightStyle( property = opaque(0x79C0FF), tag = opaque(0x7EE787), attribute = opaque(0x79C0FF), + inserted = opaque(0x3FB950), + deleted = opaque(0xF85149), ) /** A style with [color]'s result for every token type. */ @@ -85,6 +91,8 @@ data class CodeHighlightStyle( tag = color(SyntaxTokenType.TAG), attribute = color(SyntaxTokenType.ATTRIBUTE), embedded = color(SyntaxTokenType.EMBEDDED), + inserted = color(SyntaxTokenType.INSERTED), + deleted = color(SyntaxTokenType.DELETED), ) private fun opaque(rgb: Int): Int = 0xFF000000.toInt() or rgb diff --git a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/SyntaxTokenType.kt b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/SyntaxTokenType.kt index 45a50e2ef..3a42bf588 100644 --- a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/SyntaxTokenType.kt +++ b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/SyntaxTokenType.kt @@ -20,4 +20,6 @@ enum class SyntaxTokenType { TAG, ATTRIBUTE, EMBEDDED, + INSERTED, + DELETED, } diff --git a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDsl.kt b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDsl.kt index 08646e39b..29e03683a 100644 --- a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDsl.kt +++ b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDsl.kt @@ -53,6 +53,8 @@ class CodeHighlightStyleScope internal constructor( var tag: Color? by token(SyntaxTokenType.TAG) var attribute: Color? by token(SyntaxTokenType.ATTRIBUTE) var embedded: Color? by token(SyntaxTokenType.EMBEDDED) + var inserted: Color? by token(SyntaxTokenType.INSERTED) + var deleted: Color? by token(SyntaxTokenType.DELETED) /** The color for [type], for code that picks the token at runtime. */ operator fun set( diff --git a/packages/enriched-markdown-android/code-highlight/src/test/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDslTest.kt b/packages/enriched-markdown-android/code-highlight/src/test/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDslTest.kt index 6fc69cff5..0951e0b11 100644 --- a/packages/enriched-markdown-android/code-highlight/src/test/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDslTest.kt +++ b/packages/enriched-markdown-android/code-highlight/src/test/java/com/swmansion/enriched/markdown/codehighlight/compose/CodeHighlightStyleDslTest.kt @@ -62,6 +62,8 @@ class CodeHighlightStyleDslTest { tag = colors.getValue(SyntaxTokenType.TAG) attribute = colors.getValue(SyntaxTokenType.ATTRIBUTE) embedded = colors.getValue(SyntaxTokenType.EMBEDDED) + inserted = colors.getValue(SyntaxTokenType.INSERTED) + deleted = colors.getValue(SyntaxTokenType.DELETED) } } diff --git a/packages/enriched-markdown-ios/README.md b/packages/enriched-markdown-ios/README.md index 971323d70..5fce8ab2c 100644 --- a/packages/enriched-markdown-ios/README.md +++ b/packages/enriched-markdown-ios/README.md @@ -796,7 +796,7 @@ EnrichedMarkdownText(content) The types are `.keyword`, `.operator`, `.punctuation`, `.string`, `.number`, `.constant`, `.comment`, `.function`, `.type`, `.variable`, `.property`, -`.tag`, `.attribute`, and `.embedded`. +`.tag`, `.attribute`, `.embedded`, `.inserted`, and `.deleted`. `.markdownSyntaxHighlighting()` layers `MarkdownTheme.syntaxHighlightingDefault` — GitHub's palette, light or dark with the color scheme — directly above diff --git a/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxToken.swift b/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxToken.swift index 9eeeabdd9..74fb5ec4d 100644 --- a/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxToken.swift +++ b/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxToken.swift @@ -42,6 +42,8 @@ public extension MarkdownTheme { SyntaxToken(.property).foregroundStyle(light: 0x0550AE, dark: 0x79C0FF) SyntaxToken(.tag).foregroundStyle(light: 0x116329, dark: 0x7EE787) SyntaxToken(.attribute).foregroundStyle(light: 0x0550AE, dark: 0x79C0FF) + SyntaxToken(.inserted).foregroundStyle(light: 0x1A7F37, dark: 0x3FB950) + SyntaxToken(.deleted).foregroundStyle(light: 0xD1242F, dark: 0xF85149) } } diff --git a/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxTokenType.swift b/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxTokenType.swift index 82d2af1eb..5595339fb 100644 --- a/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxTokenType.swift +++ b/packages/enriched-markdown-ios/Sources/EnrichedMarkdownSyntaxHighlighting/SyntaxTokenType.swift @@ -13,4 +13,6 @@ public enum SyntaxTokenType: UInt8, CaseIterable, Sendable { case tag case attribute case embedded + case inserted + case deleted } diff --git a/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlighterTests.swift b/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlighterTests.swift index b50481e7b..d2af6ee37 100644 --- a/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlighterTests.swift +++ b/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlighterTests.swift @@ -89,6 +89,14 @@ final class SyntaxHighlighterTests: XCTestCase { XCTAssertEqual(texts(of: .keyword, in: code, language: "python"), ["import"]) } + func testDiffLinesNeedNoGrammar() { + let code = "@@ -1,2 +1,2 @@\n same\n-old 🎉\n+new" + + XCTAssertEqual(texts(of: .function, in: code, language: "diff"), ["@@ -1,2 +1,2 @@"]) + XCTAssertEqual(texts(of: .deleted, in: code, language: "diff"), ["-old 🎉"]) + XCTAssertEqual(texts(of: .inserted, in: code, language: "patch"), ["+new"]) + } + func testUnknownLanguageHasNoTokens() { XCTAssertTrue(SyntaxHighlighter.tokens(in: "let x = 1", language: "klingon").isEmpty) XCTAssertTrue(SyntaxHighlighter.tokens(in: "let x = 1", language: "").isEmpty) diff --git a/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlightingRenderingTests.swift b/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlightingRenderingTests.swift index 406b5903f..bee647fc0 100644 --- a/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlightingRenderingTests.swift +++ b/packages/enriched-markdown-ios/Tests/EnrichedMarkdownSyntaxHighlightingTests/SyntaxHighlightingRenderingTests.swift @@ -169,6 +169,8 @@ final class SyntaxHighlightingRenderingTests: XCTestCase { XCTAssertEqual(hex(dark[.property]), 0x79C0FF) XCTAssertEqual(hex(dark[.tag]), 0x7EE787) XCTAssertEqual(hex(dark[.attribute]), 0x79C0FF) + XCTAssertEqual(hex(dark[.inserted]), 0x3FB950) + XCTAssertEqual(hex(dark[.deleted]), 0xF85149) } func testDefaultPaletteLeavesInheritingTypesUnset() { @@ -190,10 +192,11 @@ final class SyntaxHighlightingRenderingTests: XCTestCase { } func testTokenTypesMatchTheSharedSeam() { - XCTAssertEqual(SyntaxTokenType.allCases.map(\.rawValue), Array(0...13)) + XCTAssertEqual(SyntaxTokenType.allCases.map(\.rawValue), Array(0...15)) XCTAssertEqual(SyntaxTokenType.keyword.rawValue, 0) XCTAssertEqual(SyntaxTokenType.comment.rawValue, 6) XCTAssertEqual(SyntaxTokenType.embedded.rawValue, 13) + XCTAssertEqual(SyntaxTokenType.deleted.rawValue, 15) } } diff --git a/packages/enriched-markdown-ios/highlight/SwiftHighlightShim.cpp b/packages/enriched-markdown-ios/highlight/SwiftHighlightShim.cpp index eb4c4a09f..e9ac1cc88 100644 --- a/packages/enriched-markdown-ios/highlight/SwiftHighlightShim.cpp +++ b/packages/enriched-markdown-ios/highlight/SwiftHighlightShim.cpp @@ -8,7 +8,7 @@ static_assert(static_cast(Markdown::HighlightTokenType::Keyword) == 0, "HighlightTokenType enum must stay in sync with Swift SyntaxTokenType"); -static_assert(static_cast(Markdown::HighlightTokenType::Embedded) == 13, +static_assert(static_cast(Markdown::HighlightTokenType::Deleted) == 15, "HighlightTokenType enum must stay in sync with Swift SyntaxTokenType"); extern "C" { diff --git a/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/styles/CodeBlockStyle.kt b/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/styles/CodeBlockStyle.kt index 907389bf6..ef6eab365 100644 --- a/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/styles/CodeBlockStyle.kt +++ b/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/styles/CodeBlockStyle.kt @@ -34,6 +34,8 @@ data class CodeBlockStyle( "tag", "attribute", "embedded", + "inserted", + "deleted", ) fun fromReadableMap( diff --git a/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/CodeBlockHighlighter.kt b/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/CodeBlockHighlighter.kt index 6ef15f6bf..0d639c33e 100644 --- a/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/CodeBlockHighlighter.kt +++ b/packages/react-native-enriched-markdown/android/src/main/java/com/swmansion/enriched/markdown/utils/common/CodeBlockHighlighter.kt @@ -26,6 +26,8 @@ enum class HighlightTokenType { Tag, Attribute, Embedded, + Inserted, + Deleted, } /** diff --git a/packages/react-native-enriched-markdown/ios/styles/StyleConfig.mm b/packages/react-native-enriched-markdown/ios/styles/StyleConfig.mm index b63d2f80a..d35fc1bd5 100644 --- a/packages/react-native-enriched-markdown/ios/styles/StyleConfig.mm +++ b/packages/react-native-enriched-markdown/ios/styles/StyleConfig.mm @@ -6,7 +6,7 @@ #import static const NSInteger kENRMCodeBlockSyntaxColorCount = - static_cast(Markdown::HighlightTokenType::Embedded) + 1; + static_cast(Markdown::HighlightTokenType::Deleted) + 1; static inline NSString *normalizedFontWeight(NSString *fontWeight) { diff --git a/packages/react-native-enriched-markdown/ios/utils/StylePropsUtils.h b/packages/react-native-enriched-markdown/ios/utils/StylePropsUtils.h index dc242fce3..884e41724 100644 --- a/packages/react-native-enriched-markdown/ios/utils/StylePropsUtils.h +++ b/packages/react-native-enriched-markdown/ios/utils/StylePropsUtils.h @@ -989,6 +989,8 @@ BOOL applyMarkdownStyleToConfig(StyleConfig *config, const MarkdownStyle &newSty ENRM_SET_SYNTAX_COLOR(tag, Tag) ENRM_SET_SYNTAX_COLOR(attribute, Attribute) ENRM_SET_SYNTAX_COLOR(embedded, Embedded) + ENRM_SET_SYNTAX_COLOR(inserted, Inserted) + ENRM_SET_SYNTAX_COLOR(deleted, Deleted) #undef ENRM_SET_SYNTAX_COLOR // ── Thematic Break ───────────────────────────────────────────────────────── diff --git a/packages/react-native-enriched-markdown/package.json b/packages/react-native-enriched-markdown/package.json index 75136bf48..686809544 100644 --- a/packages/react-native-enriched-markdown/package.json +++ b/packages/react-native-enriched-markdown/package.json @@ -97,6 +97,7 @@ }, "devDependencies": { "@babel/core": "^7.25.0", + "@derekstride/tree-sitter-sql": "0.3.11", "@react-native/babel-preset": "0.88.0", "@react-native/jest-preset": "0.88.0", "@tree-sitter-grammars/tree-sitter-markdown": "0.3.2", @@ -120,11 +121,13 @@ "tree-sitter-c-sharp": "0.23.1", "tree-sitter-cpp": "0.23.4", "tree-sitter-css": "0.23.2", + "tree-sitter-dart": "1.0.0", "tree-sitter-go": "0.23.4", "tree-sitter-html": "0.23.2", "tree-sitter-java": "0.23.5", "tree-sitter-javascript": "0.23.1", "tree-sitter-json": "0.24.8", + "tree-sitter-kotlin": "0.3.8", "tree-sitter-php": "0.24.2", "tree-sitter-python": "0.23.6", "tree-sitter-ruby": "0.23.1", diff --git a/packages/react-native-enriched-markdown/src/EnrichedMarkdownNativeComponent.ts b/packages/react-native-enriched-markdown/src/EnrichedMarkdownNativeComponent.ts index 31e21dbdf..acc02325a 100644 --- a/packages/react-native-enriched-markdown/src/EnrichedMarkdownNativeComponent.ts +++ b/packages/react-native-enriched-markdown/src/EnrichedMarkdownNativeComponent.ts @@ -73,6 +73,8 @@ interface CodeBlockSyntaxColorsInternal { tag: ColorValue; attribute: ColorValue; embedded: ColorValue; + inserted: ColorValue; + deleted: ColorValue; } interface CodeBlockStyleInternal extends BaseBlockStyleInternal { diff --git a/packages/react-native-enriched-markdown/src/EnrichedMarkdownTextNativeComponent.ts b/packages/react-native-enriched-markdown/src/EnrichedMarkdownTextNativeComponent.ts index 58e569744..5bf3457ed 100644 --- a/packages/react-native-enriched-markdown/src/EnrichedMarkdownTextNativeComponent.ts +++ b/packages/react-native-enriched-markdown/src/EnrichedMarkdownTextNativeComponent.ts @@ -73,6 +73,8 @@ interface CodeBlockSyntaxColorsInternal { tag: ColorValue; attribute: ColorValue; embedded: ColorValue; + inserted: ColorValue; + deleted: ColorValue; } interface CodeBlockStyleInternal extends BaseBlockStyleInternal { diff --git a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts index f36748212..fc57f04a6 100644 --- a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts +++ b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.ts @@ -56,6 +56,8 @@ const DEFAULT_CODE_BLOCK_SYNTAX_COLORS = { tag: normalizeColor('#7EE787')!, attribute: normalizeColor('#79C0FF')!, embedded: codeBlockTextColor, + inserted: normalizeColor('#3FB950')!, + deleted: normalizeColor('#F85149')!, }; const INHERIT_SYNTAX_TOKENS = new Set([ diff --git a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts index 170e9f1dc..ab40b463e 100644 --- a/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts +++ b/packages/react-native-enriched-markdown/src/normalizeMarkdownStyle.web.ts @@ -145,6 +145,8 @@ export const DEFAULT_NORMALIZED_STYLE: MarkdownStyleInternal = Object.freeze({ tag: '#7EE787', attribute: '#79C0FF', embedded: '#F3F4F6', + inserted: '#3FB950', + deleted: '#F85149', }, }, link: { diff --git a/packages/react-native-enriched-markdown/src/types/MarkdownStyle.ts b/packages/react-native-enriched-markdown/src/types/MarkdownStyle.ts index 1fda59e97..fa7e95686 100644 --- a/packages/react-native-enriched-markdown/src/types/MarkdownStyle.ts +++ b/packages/react-native-enriched-markdown/src/types/MarkdownStyle.ts @@ -104,6 +104,10 @@ interface CodeBlockSyntaxColors { tag?: string; attribute?: string; embedded?: string; + /** Added lines in a `diff` block. */ + inserted?: string; + /** Removed lines in a `diff` block. */ + deleted?: string; } interface CodeBlockStyle extends BaseBlockStyle { diff --git a/packages/react-native-enriched-markdown/src/types/MarkdownStyleInternal.ts b/packages/react-native-enriched-markdown/src/types/MarkdownStyleInternal.ts index 9117812ac..0fe3694be 100644 --- a/packages/react-native-enriched-markdown/src/types/MarkdownStyleInternal.ts +++ b/packages/react-native-enriched-markdown/src/types/MarkdownStyleInternal.ts @@ -86,6 +86,8 @@ export interface CodeBlockSyntaxColorsInternal { tag: string; attribute: string; embedded: string; + inserted: string; + deleted: string; } interface CodeBlockStyleInternal extends BaseBlockStyleInternal { diff --git a/vendor/grammar-versions.json b/vendor/grammar-versions.json index 5633b06e7..d4f89c233 100644 --- a/vendor/grammar-versions.json +++ b/vendor/grammar-versions.json @@ -131,6 +131,28 @@ "default": false, "size": "18 MB" }, + "kotlin": { + "package": "tree-sitter-kotlin", + "version": "0.3.8", + "scanner": true, + "default": false, + "size": "23 MB", + "$comment": "fwcd's grammar. @tree-sitter-grammars/tree-sitter-kotlin does not ship queries/highlights.scm in its npm package." + }, + "dart": { + "package": "tree-sitter-dart", + "version": "1.0.0", + "scanner": true, + "default": false, + "size": "4.6 MB" + }, + "sql": { + "package": "@derekstride/tree-sitter-sql", + "version": "0.3.11", + "scanner": true, + "default": false, + "size": "17 MB" + }, "php": { "package": "tree-sitter-php", "version": "0.24.2", diff --git a/yarn.lock b/yarn.lock index 70caa1cba..94e42c10b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3347,6 +3347,22 @@ __metadata: languageName: node linkType: hard +"@derekstride/tree-sitter-sql@npm:0.3.11": + version: 0.3.11 + resolution: "@derekstride/tree-sitter-sql@npm:0.3.11" + dependencies: + node-addon-api: "npm:^7.1.0" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.0" + peerDependencies: + tree-sitter: ^0.21.0 + peerDependenciesMeta: + tree_sitter: + optional: true + checksum: 10c0/094757836c4f0dd036c8ce3c986e4fb372e21b5c012d8e7da9580a0b962d6f5f01f3e02d92f4c1d3198783c276c6e94480402aec53f5ebc5995c2bc1fe21ae3f + languageName: node + linkType: hard + "@egjs/hammerjs@npm:^2.0.17": version: 2.0.17 resolution: "@egjs/hammerjs@npm:2.0.17" @@ -10572,6 +10588,8 @@ __metadata: turbo: "npm:^2.8.21" typescript: "npm:^6.0.2" dependenciesMeta: + "@derekstride/tree-sitter-sql": + built: false "@tree-sitter-grammars/tree-sitter-markdown": built: false "@tree-sitter-grammars/tree-sitter-yaml": @@ -10588,6 +10606,8 @@ __metadata: built: false tree-sitter-css: built: false + tree-sitter-dart: + built: false tree-sitter-go: built: false tree-sitter-html: @@ -10598,6 +10618,8 @@ __metadata: built: false tree-sitter-json: built: false + tree-sitter-kotlin: + built: false tree-sitter-php: built: false tree-sitter-python: @@ -15730,6 +15752,15 @@ __metadata: languageName: node linkType: hard +"nan@npm:^2.15.0": + version: 2.30.0 + resolution: "nan@npm:2.30.0" + dependencies: + node-gyp: "npm:latest" + checksum: 10c0/563bf97c84099feeccae9aeaee1f1e14043e0ab0e8525497058495373e9a53abd46b00a6f5d178931911c2739c8ee94604da13d1245ab9e8b689339e61e98c23 + languageName: node + linkType: hard + "nanoid@npm:^3.3.1": version: 3.3.12 resolution: "nanoid@npm:3.3.12" @@ -15816,6 +15847,15 @@ __metadata: languageName: node linkType: hard +"node-addon-api@npm:^7.1.0": + version: 7.1.1 + resolution: "node-addon-api@npm:7.1.1" + dependencies: + node-gyp: "npm:latest" + checksum: 10c0/fb32a206276d608037fa1bcd7e9921e177fe992fc610d098aa3128baca3c0050fc1e014fa007e9b3874cf865ddb4f5bd9f43ccb7cbbbe4efaff6a83e920b17e9 + languageName: node + linkType: hard + "node-addon-api@npm:^8.0.0, node-addon-api@npm:^8.1.0, node-addon-api@npm:^8.2.1, node-addon-api@npm:^8.2.2, node-addon-api@npm:^8.3.0, node-addon-api@npm:^8.3.1": version: 8.9.1 resolution: "node-addon-api@npm:8.9.1" @@ -17300,6 +17340,7 @@ __metadata: resolution: "react-native-enriched-markdown@workspace:packages/react-native-enriched-markdown" dependencies: "@babel/core": "npm:^7.25.0" + "@derekstride/tree-sitter-sql": "npm:0.3.11" "@react-native/babel-preset": "npm:0.88.0" "@react-native/jest-preset": "npm:0.88.0" "@tree-sitter-grammars/tree-sitter-markdown": "npm:0.3.2" @@ -17323,11 +17364,13 @@ __metadata: tree-sitter-c-sharp: "npm:0.23.1" tree-sitter-cpp: "npm:0.23.4" tree-sitter-css: "npm:0.23.2" + tree-sitter-dart: "npm:1.0.0" tree-sitter-go: "npm:0.23.4" tree-sitter-html: "npm:0.23.2" tree-sitter-java: "npm:0.23.5" tree-sitter-javascript: "npm:0.23.1" tree-sitter-json: "npm:0.24.8" + tree-sitter-kotlin: "npm:0.3.8" tree-sitter-php: "npm:0.24.2" tree-sitter-python: "npm:0.23.6" tree-sitter-ruby: "npm:0.23.1" @@ -19494,6 +19537,16 @@ __metadata: languageName: node linkType: hard +"tree-sitter-dart@npm:1.0.0": + version: 1.0.0 + resolution: "tree-sitter-dart@npm:1.0.0" + dependencies: + nan: "npm:^2.15.0" + node-gyp: "npm:latest" + checksum: 10c0/cb339a0be975850d84065c642e27a464dc7fc7a023c547283fefff28b115e3739b04733e9b6bf6a00188e40de4660884d2bbebf6db67fc2e8dee5bdd74802026 + languageName: node + linkType: hard + "tree-sitter-go@npm:0.23.4": version: 0.23.4 resolution: "tree-sitter-go@npm:0.23.4" @@ -19574,6 +19627,22 @@ __metadata: languageName: node linkType: hard +"tree-sitter-kotlin@npm:0.3.8": + version: 0.3.8 + resolution: "tree-sitter-kotlin@npm:0.3.8" + dependencies: + node-addon-api: "npm:^7.1.0" + node-gyp: "npm:latest" + node-gyp-build: "npm:^4.8.0" + peerDependencies: + tree-sitter: ^0.21.0 + peerDependenciesMeta: + tree_sitter: + optional: true + checksum: 10c0/2dee5cf5123df38d13ad5d42f78f8910fab4058430f7b510d44d0cba31089eba34a7de3ff36a40363b56723c81396990654f272acc71efd9f652d26966c71ba8 + languageName: node + linkType: hard + "tree-sitter-php@npm:0.24.2": version: 0.24.2 resolution: "tree-sitter-php@npm:0.24.2" From 23ce6af473d93ce8fc66fb760d447c36a11ba18e Mon Sep 17 00:00:00 2001 From: shivam <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 10 Nov 2026 02:31:21 +0000 Subject: [PATCH 2/2] fix(highlight): preserve Kotlin and Dart semantic tokens --- .../core/cpp/highlight/CodeBlockHighlighter.cpp | 13 +++++++++++++ .../core/cpp/tests/highlight/HighlightTests.cpp | 13 +++++++++++++ .../markdown/codehighlight/CodeHighlighterNative.kt | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/core/cpp/highlight/CodeBlockHighlighter.cpp b/packages/core/cpp/highlight/CodeBlockHighlighter.cpp index 5ee85d7b4..130fd6cbc 100644 --- a/packages/core/cpp/highlight/CodeBlockHighlighter.cpp +++ b/packages/core/cpp/highlight/CodeBlockHighlighter.cpp @@ -49,6 +49,10 @@ struct CaptureMapping { constexpr CaptureMapping kCaptureMappings[] = { {"keyword", HighlightTokenType::Keyword}, + {"conditional", HighlightTokenType::Keyword}, + {"repeat", HighlightTokenType::Keyword}, + {"exception", HighlightTokenType::Keyword}, + {"include", HighlightTokenType::Keyword}, {"operator", HighlightTokenType::Operator}, {"punctuation", HighlightTokenType::Punctuation}, {"string", HighlightTokenType::String}, @@ -390,6 +394,15 @@ std::vector highlightCode(const std::string &code, const std::st TSQueryMatch match; uint32_t captureIndex = 0; while (ts_query_cursor_next_capture(cursor.get(), &match, &captureIndex)) { + // Dart's predicate rules capture every identifier without evaluation and + // overwrite its structural function/type captures. + if (grammarId == "dart") { + uint32_t predicateCount = 0; + ts_query_predicates_for_pattern(query, match.pattern_index, &predicateCount); + if (predicateCount > 0) { + continue; + } + } const TSQueryCapture &capture = match.captures[captureIndex]; uint32_t nameLength = 0; const char *name = ts_query_capture_name_for_id(query, capture.index, &nameLength); diff --git a/packages/core/cpp/tests/highlight/HighlightTests.cpp b/packages/core/cpp/tests/highlight/HighlightTests.cpp index 584e59b1a..be8fb1ce9 100644 --- a/packages/core/cpp/tests/highlight/HighlightTests.cpp +++ b/packages/core/cpp/tests/highlight/HighlightTests.cpp @@ -148,12 +148,17 @@ TEST_CASE("languages without a grammar stay plain") { TEST_CASE("kotlin") { const std::string code = + "import kotlin.math.abs\n" "fun greet(name: String): String {\n" " val n = 42\n" + " if (n > 0) { for (i in 1..n) { try { abs(i) } catch (e: Exception) {} } }\n" " return \"hi $name\" // note\n" "}"; CHECK(contains(texts(code, "kotlin", HighlightTokenType::Keyword), "fun")); CHECK(contains(texts(code, "kotlin", HighlightTokenType::Keyword), "val")); + for (const char *keyword : {"import", "if", "for", "try", "catch"}) { + CHECK(contains(texts(code, "kotlin", HighlightTokenType::Keyword), keyword)); + } CHECK(contains(texts(code, "kotlin", HighlightTokenType::Function), "greet")); CHECK(contains(texts(code, "kotlin", HighlightTokenType::Type), "String")); CHECK(contains(texts(code, "kotlin", HighlightTokenType::Number), "42")); @@ -179,11 +184,19 @@ TEST_CASE("swift") { TEST_CASE("dart") { const std::string code = + "import 'dart:math';\n" + "class Greeter {}\n" "void main() {\n" " var n = 42;\n" + " if (n > 0) { for (var i = 0; i < n; i++) { try {} catch (e) {} } }\n" " print('hi'); // note\n" "}"; CHECK(contains(texts(code, "dart", HighlightTokenType::Keyword), "var")); + for (const char *keyword : {"import", "if", "for", "try", "catch"}) { + CHECK(contains(texts(code, "dart", HighlightTokenType::Keyword), keyword)); + } + CHECK(contains(texts(code, "dart", HighlightTokenType::Function), "main")); + CHECK(contains(texts(code, "dart", HighlightTokenType::Type), "Greeter")); CHECK(contains(texts(code, "dart", HighlightTokenType::Type), "void")); CHECK(contains(texts(code, "dart", HighlightTokenType::String), "'hi'")); CHECK(contains(texts(code, "dart", HighlightTokenType::Number), "42")); diff --git a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlighterNative.kt b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlighterNative.kt index f75148219..bfd3bfc6c 100644 --- a/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlighterNative.kt +++ b/packages/enriched-markdown-android/code-highlight/src/main/java/com/swmansion/enriched/markdown/codehighlight/CodeHighlighterNative.kt @@ -9,7 +9,7 @@ package com.swmansion.enriched.markdown.codehighlight */ internal object CodeHighlighterNative { /** Token types in the seam's `HighlightTokenType` order; `HighlightJni.cpp` pins the count. */ - const val TOKEN_TYPE_COUNT = 14 + const val TOKEN_TYPE_COUNT = 16 init { // Not caught here: the failure reaches the caller, which logs it once and stops calling.