Skip to content

parser/lexer: bump to Unicode 17, use faster unicode-ident #148321

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bors merged 2 commits into from
Dec 29, 2025
Merged

parser/lexer: bump to Unicode 17, use faster unicode-ident #148321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ca64688
parser/lexer: bump to Unicode 17, use faster unicode-ident
Marcondiro Oct 30, 2025
f7cb82e
lexer/parser: ensure deps use the same unicode version
Marcondiro Dec 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4146,8 +4146,8 @@ version = "0.0.0"
dependencies = [
"expect-test",
"memchr",
"unicode-ident",
"unicode-properties",
"unicode-xid",
]

[[package]]
Expand Down Expand Up @@ -5983,24 +5983,24 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"

[[package]]
name = "unicode-ident"
version = "1.0.18"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"

[[package]]
name = "unicode-normalization"
version = "0.1.24"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]

[[package]]
name = "unicode-properties"
version = "0.1.3"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"

[[package]]
name = "unicode-script"
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_lexer/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ Rust lexer used by rustc. No stability guarantees are provided.
# Note that this crate purposefully does not depend on other rustc crates
[dependencies]
memchr = "2.7.6"
unicode-properties = { version = "0.1.0", default-features = false, features = ["emoji"] }
unicode-xid = "0.2.0"
unicode-properties = { version = "0.1.4", default-features = false, features = ["emoji"] }
unicode-ident = "1.0.22"

[dev-dependencies]
expect-test = "1.4.0"
23 changes: 20 additions & 3 deletions compiler/rustc_lexer/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,25 @@ use LiteralKind::*;
use TokenKind::*;
use cursor::EOF_CHAR;
pub use cursor::{Cursor, FrontmatterAllowed};
pub use unicode_ident::UNICODE_VERSION;
use unicode_properties::UnicodeEmoji;
pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;

// Make sure that the Unicode version of the dependencies is the same.
const _: () = {
Copy link
Contributor

@GrigorenkoPV GrigorenkoPV Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yay, #149226 use-case spotted in the wild

let properties = unicode_properties::UNICODE_VERSION;
let ident = unicode_ident::UNICODE_VERSION;

if properties.0 != ident.0 as u64
|| properties.1 != ident.1 as u64
|| properties.2 != ident.2 as u64
{
panic!(
"unicode-properties and unicode-ident must use the same Unicode version, \
`unicode_properties::UNICODE_VERSION` and `unicode_ident::UNICODE_VERSION` are \
different."
);
}
};

/// Parsed token.
/// It doesn't contain information about data that has been parsed,
Expand Down Expand Up @@ -370,14 +387,14 @@ pub fn is_horizontal_whitespace(c: char) -> bool {
/// a formal definition of valid identifier name.
pub fn is_id_start(c: char) -> bool {
// This is XID_Start OR '_' (which formally is not a XID_Start).
c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
c == '_' || unicode_ident::is_xid_start(c)
}

/// True if `c` is valid as a non-first character of an identifier.
/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
/// a formal definition of valid identifier name.
pub fn is_id_continue(c: char) -> bool {
unicode_xid::UnicodeXID::is_xid_continue(c)
unicode_ident::is_xid_continue(c)
}

/// The passed string is lexically an identifier.
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_parse/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ rustc_session = { path = "../rustc_session" }
rustc_span = { path = "../rustc_span" }
thin-vec = "0.2.12"
tracing = "0.1"
unicode-normalization = "0.1.11"
unicode-width = "0.2.0"
unicode-normalization = "0.1.25"
unicode-width = "0.2.2"
# tidy-alphabetical-end

[dev-dependencies]
Expand Down
40 changes: 39 additions & 1 deletion compiler/rustc_parse/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ use rustc_ast::token;
use rustc_ast::tokenstream::TokenStream;
use rustc_ast_pretty::pprust;
use rustc_errors::{Diag, EmissionGuarantee, FatalError, PResult, pluralize};
pub use rustc_lexer::UNICODE_VERSION;
use rustc_session::parse::ParseSess;
use rustc_span::source_map::SourceMap;
use rustc_span::{FileName, SourceFile, Span};
pub use unicode_normalization::UNICODE_VERSION as UNICODE_NORMALIZATION_VERSION;

pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments");

Expand All @@ -39,6 +39,44 @@ pub mod lexer;

mod errors;

// Make sure that the Unicode version of the dependencies is the same.
const _: () = {
let rustc_lexer = rustc_lexer::UNICODE_VERSION;
let rustc_span = rustc_span::UNICODE_VERSION;
let normalization = unicode_normalization::UNICODE_VERSION;
let width = unicode_width::UNICODE_VERSION;

if rustc_lexer.0 != rustc_span.0
|| rustc_lexer.1 != rustc_span.1
|| rustc_lexer.2 != rustc_span.2
{
panic!(
"rustc_lexer and rustc_span must use the same Unicode version, \
`rustc_lexer::UNICODE_VERSION` and `rustc_span::UNICODE_VERSION` are \
different."
);
}

if rustc_lexer.0 != normalization.0
|| rustc_lexer.1 != normalization.1
|| rustc_lexer.2 != normalization.2
{
panic!(
"rustc_lexer and unicode-normalization must use the same Unicode version, \
`rustc_lexer::UNICODE_VERSION` and `unicode_normalization::UNICODE_VERSION` are \
different."
);
}

if rustc_lexer.0 != width.0 || rustc_lexer.1 != width.1 || rustc_lexer.2 != width.2 {
panic!(
"rustc_lexer and unicode-width must use the same Unicode version, \
`rustc_lexer::UNICODE_VERSION` and `unicode_width::UNICODE_VERSION` are \
different."
);
}
};

rustc_fluent_macro::fluent_messages! { "../messages.ftl" }

// Unwrap the result if `Ok`, otherwise emit the diagnostics and abort.
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_span/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@ scoped-tls = "1.0"
sha1 = "0.10.0"
sha2 = "0.10.1"
tracing = "0.1"
unicode-width = "0.2.0"
unicode-width = "0.2.2"
# tidy-alphabetical-end
1 change: 1 addition & 0 deletions compiler/rustc_span/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ use rustc_macros::{Decodable, Encodable, HashStable_Generic};
use rustc_serialize::opaque::{FileEncoder, MemDecoder};
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
use tracing::debug;
pub use unicode_width::UNICODE_VERSION;

mod caching_source_map_view;
pub mod source_map;
Expand Down
1 change: 0 additions & 1 deletion src/tools/tidy/src/deps.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,6 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
"unicode-script",
"unicode-security",
"unicode-width",
"unicode-xid",
"utf8parse",
"valuable",
"version_check",
Expand Down
4 changes: 1 addition & 3 deletions tests/ui-fulldeps/lexer/unicode-version.rs
View file
Open in desktop
Copy link
Contributor Author

@Marcondiro Marcondiro Dec 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since all the relevant crates seem to export some flavour of the UNICODE_VERSION constant included in the standard library, perhaps there could be a simple test somewhere that asserts that all these constants are identical? That way manual review of the versions isn't required.

At the moment there is this test in place, checking that rustc_lexer::UNICODE_IDENT_VERSION == rustc_parse::UNICODE_NORMALIZATION_VERSION

Would expanding this test to all the Unicode-version dependent crates used in /compiler/* be enough?

Copy link
Contributor

@clarfonthey clarfonthey Dec 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, to clarify, this is exactly what I was proposing in this comment: #148321 (comment)

I was looking into it, and my particular method I was going with was just adding an anonymous constant in each of the crates to make a static assertion. So, for example, in rustc_lexer, this is what I currently have right now in my (unpublished) branch:

// ensure that unicode version is same as libstd
const _: () = {
    let internal = std::char::UNICODE_VERSION;
    let properties = unicode_properties::UNICODE_VERSION;
    assert!(internal.0 as u64 == properties.0);
    assert!(internal.1 as u64 == properties.1);
    assert!(internal.2 as u64 == properties.2);
};
const _: () = {
    let internal = std::char::UNICODE_VERSION;
    let xid = unicode_xid::UNICODE_VERSION;
    assert!(internal.0 as u64 == xid.0);
    assert!(internal.1 as u64 == xid.1);
    assert!(internal.2 as u64 == xid.2);
};

(note: I'm using libstd's version as an anchor because it's the most convenient: if all of them match, then libstd should match. also, while I could use unstable features to make this simpler with assert_eq, I chose to go with something that works on stable atm just for ease of maintenance)

Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
#![feature(rustc_private)]

extern crate rustc_driver;
extern crate rustc_lexer;
extern crate rustc_parse;

fn main() {
Expand All @@ -22,6 +21,5 @@ fn main() {
it should also be updated in the reference at \
https://github.com/rust-lang/reference/blob/HEAD/src/identifiers.md."
);
println!("Unicode XID version is: {:?}", rustc_lexer::UNICODE_XID_VERSION);
println!("Unicode normalization version is: {:?}", rustc_parse::UNICODE_NORMALIZATION_VERSION);
println!("Unicode version used in rustc_parse is: {:?}", rustc_parse::UNICODE_VERSION);
}
3 changes: 1 addition & 2 deletions tests/ui-fulldeps/lexer/unicode-version.run.stdout
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
Checking if Unicode version changed.
If the Unicode version changes are intentional, it should also be updated in the reference at https://github.com/rust-lang/reference/blob/HEAD/src/identifiers.md.
Unicode XID version is: (16, 0, 0)
Unicode normalization version is: (16, 0, 0)
Unicode version used in rustc_parse is: (17, 0, 0)
Loading