From a17324a830a2b13806db68a796c3877c354b3058 Mon Sep 17 00:00:00 2001 From: Aram Hammoudeh Date: Sat, 20 Jun 2026 19:18:33 -0600 Subject: [PATCH] feat(codegen): infer workspace token modules --- Cargo.lock | 1 + crates/plumb-cli/tests/init_from.rs | 89 +++ crates/plumb-codegen/Cargo.toml | 1 + crates/plumb-codegen/src/lib.rs | 135 +++- crates/plumb-codegen/src/ts_tokens.rs | 903 ++++++++++++++++++++++++++ crates/plumb-codegen/src/walk.rs | 218 ++++++- 6 files changed, 1326 insertions(+), 21 deletions(-) create mode 100644 crates/plumb-codegen/src/ts_tokens.rs diff --git a/Cargo.lock b/Cargo.lock index 769ff1b..feceb45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2172,6 +2172,7 @@ dependencies = [ "insta", "plumb-config", "plumb-core", + "serde_json", "tempfile", "thiserror", "toml 1.1.2+spec-1.1.0", diff --git a/crates/plumb-cli/tests/init_from.rs b/crates/plumb-cli/tests/init_from.rs index 44f44cf..abbdd29 100644 --- a/crates/plumb-cli/tests/init_from.rs +++ b/crates/plumb-cli/tests/init_from.rs @@ -164,3 +164,92 @@ fn init_from_empty_dir_writes_blank_starter() -> Result<(), Box Result<(), Box> { + let workspace = TempDir::new()?; + fs::write( + workspace.path().join("package.json"), + r#"{ "private": true, "workspaces": ["apps/*", "packages/*"] }"#, + )?; + fs::create_dir_all(workspace.path().join("apps/web"))?; + let tokens = workspace.path().join("packages/types/src/tokens"); + fs::create_dir_all(&tokens)?; + fs::write( + tokens.join("spacing.ts"), + r" + export const SPACING = { + 0.5: '2px', + 1: '4px', + 1.5: '6px', + } as const; + + export const RADIUS = { + sm: '4px', + md: '6px', + '2xl': '16px', + } as const; + ", + )?; + fs::write( + tokens.join("colors.ts"), + r" + export const COLOR_TOKENS = { + navy: '#0A3D5C', + } as const; + + export const STATUS_COLORS = { + success: '#22c55e', + } as const; + + export const DESIGN_TOKENS = { + colors: COLOR_TOKENS, + } as const; + ", + )?; + fs::write( + tokens.join("typography.ts"), + r#" + export const FONT_FAMILY = { + heading: '"Poppins", sans-serif', + body: '"apertura", "Inter", system-ui, sans-serif', + } as const; + + export const FONT_SIZE = { + '2xs': '9px', + xs: '10px', + } as const; + + export const FONT_WEIGHT = { + normal: 400, + semibold: 600, + bold: 700, + extrabold: 800, + } as const; + "#, + )?; + + let outdir = TempDir::new()?; + Command::cargo_bin("plumb")? + .arg("init") + .arg("--from") + .arg(workspace.path().join("apps/web")) + .current_dir(outdir.path()) + .assert() + .success() + .stdout(contains("Inferred from")); + + let written = fs::read_to_string(outdir.path().join("plumb.toml"))?; + assert!(written.contains("\"0.5\" = 2")); + assert!(written.contains("\"1.5\" = 6")); + assert!(written.contains("navy = \"#0A3D5C\"")); + assert!(written.contains("success = \"#22c55e\"")); + assert!(written.contains("weights = [\n 400,\n 600,\n 700,\n 800,\n]")); + assert!(written.contains("2,\n 4,\n 6,")); + assert!(written.contains("../../packages/types/src/tokens/spacing.ts")); + let workspace_path = workspace.path().to_string_lossy(); + assert!(!written.contains(workspace_path.as_ref())); + + Ok(()) +} diff --git a/crates/plumb-codegen/Cargo.toml b/crates/plumb-codegen/Cargo.toml index 11fafdf..1b1ef81 100644 --- a/crates/plumb-codegen/Cargo.toml +++ b/crates/plumb-codegen/Cargo.toml @@ -18,6 +18,7 @@ exclude = ["AGENTS.md", "CLAUDE.md"] plumb-core = { workspace = true } plumb-config = { workspace = true } indexmap = { workspace = true } +serde_json = { workspace = true } toml = { workspace = true } thiserror = { workspace = true } diff --git a/crates/plumb-codegen/src/lib.rs b/crates/plumb-codegen/src/lib.rs index 4789315..bbd3be2 100644 --- a/crates/plumb-codegen/src/lib.rs +++ b/crates/plumb-codegen/src/lib.rs @@ -25,6 +25,11 @@ //! - **DTCG token JSON files.** Files matching `*.tokens.json` or //! placed under a `tokens/` directory are merged via //! [`plumb_config::merge_dtcg`]. +//! - **Literal TypeScript/JavaScript token modules.** When `source_dir` +//! is inside a workspace, conventional package token modules under +//! `packages/*/src/tokens/**/*.{ts,tsx,js,jsx}` are parsed for +//! exported object constants with string/number leaves. The parser +//! never evaluates JavaScript or resolves aliases. //! //! ## Determinism contract //! @@ -48,9 +53,11 @@ mod classify; mod render; +mod ts_tokens; mod walk; -use std::path::{Path, PathBuf}; +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; use indexmap::IndexMap; use plumb_config::ConfigError; @@ -59,6 +66,8 @@ use thiserror::Error; pub use render::render_toml; +type SummaryEntry = (u8, String, String); + /// Maximum directory depth the walker descends into the source tree. /// /// Most design-token directories sit at depth ≤ 3 (`src/styles/tokens.css`). @@ -132,6 +141,9 @@ pub enum TokenSourceKind { TailwindConfig, /// CSS file containing one or more `:root` blocks. CssCustomProperties, + /// Literal TypeScript/JavaScript token module from a workspace + /// package token directory. + TokenModule, /// DTCG token document (`*.tokens.json` or `tokens/*.json`). Dtcg, } @@ -144,6 +156,7 @@ impl TokenSourceKind { Self::TailwindConfig => "tailwind", Self::CssCustomProperties => "css", Self::Dtcg => "dtcg", + Self::TokenModule => "token-module", } } } @@ -188,7 +201,7 @@ pub fn infer_config(source_dir: &Path) -> Result { let walked = walk::walk(source_dir)?; let mut config = Config::default(); - let mut summary: Vec<(u8, String, String)> = Vec::new(); + let mut summary: Vec = Vec::new(); let mut sources: Vec = Vec::new(); // Tailwind config — record presence only. Theme resolution is the @@ -270,11 +283,20 @@ pub fn infer_config(source_dir: &Path) -> Result { )); } + merge_token_modules( + source_dir, + &walked.ts_token_modules, + &mut config, + &mut sources, + &mut summary, + )?; + // Sort scales ascending with duplicates removed — deterministic // output regardless of file walk order. sort_and_dedup(&mut config.spacing.scale); sort_and_dedup(&mut config.type_scale.scale); sort_and_dedup(&mut config.radius.scale); + sort_and_dedup(&mut config.type_scale.weights); // Stable summary order: `(kind tag, relative path)`. summary.sort(); @@ -287,13 +309,52 @@ pub fn infer_config(source_dir: &Path) -> Result { }) } +fn merge_token_modules( + source_dir: &Path, + token_module_paths: &[PathBuf], + config: &mut Config, + sources: &mut Vec, + summary: &mut Vec, +) -> Result<(), CodegenError> { + for token_module_path in token_module_paths { + let contents = + std::fs::read_to_string(token_module_path).map_err(|source| CodegenError::Io { + path: token_module_path.display().to_string(), + source, + })?; + let relative = relative_to(source_dir, token_module_path); + let import = ts_tokens::merge_literal_token_module(config, &relative, &contents); + sources.push(TokenSource { + kind: TokenSourceKind::TokenModule, + relative_path: relative.clone(), + }); + summary.push(( + order_tag(TokenSourceKind::TokenModule), + display_path(&relative), + format!( + "literal token module from {} (+{} colors, +{} spacing, +{} type sizes, +{} type families, +{} type weights, +{} radii)", + display_path(&relative), + import.colors, + import.spacing, + import.type_sizes, + import.type_families, + import.type_weights, + import.radii, + ), + )); + } + Ok(()) +} + /// Lower numbers sort earlier in the rendered summary. Tailwind first -/// (it's the framework signal), then CSS, then DTCG. +/// (it's the framework signal), then CSS, then literal modules, then +/// DTCG. fn order_tag(kind: TokenSourceKind) -> u8 { match kind { TokenSourceKind::TailwindConfig => 0, TokenSourceKind::CssCustomProperties => 1, - TokenSourceKind::Dtcg => 2, + TokenSourceKind::TokenModule => 2, + TokenSourceKind::Dtcg => 3, } } @@ -301,8 +362,69 @@ fn order_tag(kind: TokenSourceKind) -> u8 { /// strip fails (e.g. an absolute path the walker handed back verbatim /// because canonicalization was not possible). fn relative_to(base: &Path, path: &Path) -> PathBuf { - path.strip_prefix(base) - .map_or_else(|_| path.to_path_buf(), Path::to_path_buf) + path.strip_prefix(base).map_or_else( + |_| lexical_relative_to(base, path).unwrap_or_else(|| path.to_path_buf()), + Path::to_path_buf, + ) +} + +fn lexical_relative_to(base: &Path, path: &Path) -> Option { + let base_parts = lexical_parts(base); + let path_parts = lexical_parts(path); + if base_parts.prefix != path_parts.prefix || base_parts.rooted != path_parts.rooted { + return None; + } + + let common_len = base_parts + .segments + .iter() + .zip(&path_parts.segments) + .take_while(|(left, right)| left == right) + .count(); + + let mut out = PathBuf::new(); + for _ in common_len..base_parts.segments.len() { + out.push(".."); + } + for segment in &path_parts.segments[common_len..] { + out.push(segment); + } + Some(out) +} + +#[derive(Debug, PartialEq, Eq)] +struct LexicalParts { + prefix: Option, + rooted: bool, + segments: Vec, +} + +fn lexical_parts(path: &Path) -> LexicalParts { + let mut parts = LexicalParts { + prefix: None, + rooted: false, + segments: Vec::new(), + }; + + for component in path.components() { + match component { + Component::Prefix(prefix) => { + parts.prefix = Some(prefix.as_os_str().to_os_string()); + } + Component::RootDir => { + parts.rooted = true; + } + Component::CurDir => {} + Component::ParentDir => { + parts.segments.push(OsString::from("..")); + } + Component::Normal(segment) => { + parts.segments.push(segment.to_os_string()); + } + } + } + + parts } /// Render a path with forward slashes regardless of host OS so summaries @@ -475,6 +597,7 @@ mod tests { fn label_lookup_is_stable() { assert_eq!(TokenSourceKind::TailwindConfig.label(), "tailwind"); assert_eq!(TokenSourceKind::CssCustomProperties.label(), "css"); + assert_eq!(TokenSourceKind::TokenModule.label(), "token-module"); assert_eq!(TokenSourceKind::Dtcg.label(), "dtcg"); } } diff --git a/crates/plumb-codegen/src/ts_tokens.rs b/crates/plumb-codegen/src/ts_tokens.rs new file mode 100644 index 0000000..2eae057 --- /dev/null +++ b/crates/plumb-codegen/src/ts_tokens.rs @@ -0,0 +1,903 @@ +//! Conservative parser for literal TypeScript/JavaScript token modules. + +// Items here are crate-private but live inside a private module; the +// `redundant_pub_crate` lint flips between deny on `pub(crate)` and the +// rust-level `unreachable_pub` lint on bare `pub`. Allow the former +// scoped to this module so the items keep the explicit visibility. +#![allow(clippy::redundant_pub_crate)] + +use std::path::Path; + +use plumb_core::Config; + +/// Summary of token module values inserted into a [`Config`]. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct TokenModuleImport { + /// Number of color tokens added. + pub(crate) colors: usize, + /// Number of spacing tokens added. + pub(crate) spacing: usize, + /// Number of typography size tokens added. + pub(crate) type_sizes: usize, + /// Number of font families added. + pub(crate) type_families: usize, + /// Number of font weights added. + pub(crate) type_weights: usize, + /// Number of radius values added. + pub(crate) radii: usize, +} + +/// Merge conservative literal exports from `contents` into `config`. +pub(crate) fn merge_literal_token_module( + config: &mut Config, + path: &Path, + contents: &str, +) -> TokenModuleImport { + let mut exports = Parser::new(contents).parse_exported_objects(); + exports.sort_by(|a, b| { + token_sort_key(&a.name) + .cmp(&token_sort_key(&b.name)) + .then_with(|| a.name.cmp(&b.name)) + }); + + let mut import = TokenModuleImport::default(); + for export in exports { + let mut leaves = Vec::new(); + collect_leaves(&export.properties, &mut Vec::new(), &mut leaves); + leaves.sort_by(|a, b| { + token_path_key(&a.path) + .cmp(&token_path_key(&b.path)) + .then_with(|| a.path.cmp(&b.path)) + }); + for leaf in leaves { + merge_leaf(config, path, &export.name, &leaf, &mut import); + } + } + + config.spacing.scale.sort_unstable(); + config.spacing.scale.dedup(); + config.radius.scale.sort_unstable(); + config.radius.scale.dedup(); + config.type_scale.scale.sort_unstable(); + config.type_scale.scale.dedup(); + config.type_scale.weights.sort_unstable(); + config.type_scale.weights.dedup(); + + import +} + +#[derive(Debug)] +struct ExportedObject { + name: String, + properties: Vec, +} + +#[derive(Debug)] +struct Property { + key: String, + value: LiteralValue, +} + +#[derive(Debug)] +enum LiteralValue { + String(String), + Number(String), + Object(Vec), +} + +#[derive(Debug)] +struct TokenLeaf<'a> { + path: Vec, + value: &'a LiteralValue, +} + +struct Parser<'a> { + source: &'a str, + pos: usize, +} + +impl<'a> Parser<'a> { + fn new(source: &'a str) -> Self { + Self { source, pos: 0 } + } + + fn parse_exported_objects(&mut self) -> Vec { + let mut exports = Vec::new(); + while !self.is_eof() { + self.skip_ws_and_comments(); + if self.consume_keyword("export") { + self.skip_ws_and_comments(); + if self.consume_keyword("const") + && let Some(export) = self.parse_const_export() + { + exports.push(export); + } + continue; + } + self.skip_non_code_char(); + } + exports + } + + fn parse_const_export(&mut self) -> Option { + self.skip_ws_and_comments(); + let name = self.parse_identifier()?; + if !self.consume_until_equals() { + return None; + } + self.skip_ws_and_comments(); + let properties = self.parse_object()?; + Some(ExportedObject { name, properties }) + } + + fn parse_object(&mut self) -> Option> { + if !self.consume_byte(b'{') { + return None; + } + let mut properties = Vec::new(); + + loop { + self.skip_ws_and_comments(); + if self.consume_byte(b'}') { + return Some(properties); + } + if self.is_eof() { + return None; + } + + let Some(key) = self.parse_key() else { + self.skip_unsupported_value(); + let _ = self.consume_byte(b','); + continue; + }; + self.skip_ws_and_comments(); + if !self.consume_byte(b':') { + self.skip_unsupported_value(); + let _ = self.consume_byte(b','); + continue; + } + self.skip_ws_and_comments(); + if let Some(value) = self.parse_value() { + properties.push(Property { key, value }); + } + self.skip_ws_and_comments(); + if self.consume_byte(b',') { + continue; + } + if self.consume_byte(b'}') { + return Some(properties); + } + } + } + + fn parse_key(&mut self) -> Option { + self.skip_ws_and_comments(); + match self.current_byte()? { + b'\'' | b'"' => self.parse_string(), + b'-' | b'0'..=b'9' => self.parse_number_literal(), + _ => self.parse_identifier(), + } + } + + fn parse_value(&mut self) -> Option { + self.skip_ws_and_comments(); + match self.current_byte()? { + b'\'' | b'"' => self.parse_string().map(LiteralValue::String), + b'{' => self.parse_object().map(LiteralValue::Object), + b'-' | b'0'..=b'9' => { + if let Some(value) = self.parse_number_literal() { + Some(LiteralValue::Number(value)) + } else { + self.skip_unsupported_value(); + None + } + } + _ => { + self.skip_unsupported_value(); + None + } + } + } + + fn consume_until_equals(&mut self) -> bool { + let mut depth = 0usize; + while !self.is_eof() { + self.skip_ws_and_comments(); + let Some(byte) = self.current_byte() else { + return false; + }; + match byte { + b'=' if depth == 0 => { + self.pos += 1; + return true; + } + b';' if depth == 0 => return false, + b'\'' | b'"' => self.skip_string_literal(), + b'`' => self.skip_template_literal(), + b'(' | b'[' | b'{' | b'<' => { + depth = depth.saturating_add(1); + self.pos += 1; + } + b')' | b']' | b'}' | b'>' => { + depth = depth.saturating_sub(1); + self.pos += 1; + } + _ => { + let _ = self.bump_char(); + } + } + } + false + } + + fn skip_unsupported_value(&mut self) { + let mut depth = 0usize; + while !self.is_eof() { + self.skip_ws_and_comments(); + let Some(byte) = self.current_byte() else { + return; + }; + match byte { + b',' | b'}' if depth == 0 => return, + b'\'' | b'"' => self.skip_string_literal(), + b'`' => self.skip_template_literal(), + b'(' | b'[' | b'{' => { + depth = depth.saturating_add(1); + self.pos += 1; + } + b')' | b']' | b'}' => { + depth = depth.saturating_sub(1); + self.pos += 1; + } + _ => { + let _ = self.bump_char(); + } + } + } + } + + fn parse_string(&mut self) -> Option { + let quote = self.current_byte()?; + if quote != b'\'' && quote != b'"' { + return None; + } + self.pos += 1; + let mut out = String::new(); + while !self.is_eof() { + let ch = self.bump_char()?; + if ch == char::from(quote) { + return Some(out); + } + if ch == '\\' { + let escaped = self.bump_char()?; + match escaped { + 'n' => out.push('\n'), + 'r' => out.push('\r'), + 't' => out.push('\t'), + '\\' | '\'' | '"' => out.push(escaped), + other => out.push(other), + } + } else { + out.push(ch); + } + } + None + } + + fn parse_number_literal(&mut self) -> Option { + let start = self.pos; + if self.current_byte() == Some(b'-') { + self.pos += 1; + } + + let integer_start = self.pos; + while self.current_byte().is_some_and(|b| b.is_ascii_digit()) { + self.pos += 1; + } + if self.pos == integer_start { + self.pos = start; + return None; + } + + if self.current_byte() == Some(b'.') { + self.pos += 1; + let fraction_start = self.pos; + while self.current_byte().is_some_and(|b| b.is_ascii_digit()) { + self.pos += 1; + } + if self.pos == fraction_start { + self.pos = start; + return None; + } + } + if !self.at_number_literal_delimiter() { + self.pos = start; + return None; + } + + Some(self.source[start..self.pos].to_owned()) + } + + fn at_number_literal_delimiter(&self) -> bool { + match self.current_byte() { + Some(byte) if byte.is_ascii_whitespace() => true, + None | Some(b',' | b'}' | b']' | b')' | b':' | b';') => true, + Some(_) => false, + } + } + + fn parse_identifier(&mut self) -> Option { + let mut chars = self.source[self.pos..].char_indices(); + let (_, first) = chars.next()?; + if !is_ident_start(first) { + return None; + } + let mut end = self.pos + first.len_utf8(); + for (offset, ch) in chars { + if !is_ident_continue(ch) { + break; + } + end = self.pos + offset + ch.len_utf8(); + } + let ident = self.source[self.pos..end].to_owned(); + self.pos = end; + Some(ident) + } + + fn skip_ws_and_comments(&mut self) { + loop { + while self + .source + .get(self.pos..) + .and_then(|rest| rest.chars().next()) + .is_some_and(char::is_whitespace) + { + let _ = self.bump_char(); + } + + if self.starts_with("//") { + self.pos += 2; + while !self.is_eof() && self.current_byte() != Some(b'\n') { + let _ = self.bump_char(); + } + continue; + } + if self.starts_with("/*") { + self.pos += 2; + while !self.is_eof() && !self.starts_with("*/") { + let _ = self.bump_char(); + } + if self.starts_with("*/") { + self.pos += 2; + } + continue; + } + break; + } + } + + fn skip_non_code_char(&mut self) { + match self.current_byte() { + Some(b'\'' | b'"') => self.skip_string_literal(), + Some(b'`') => self.skip_template_literal(), + Some(_) => { + let _ = self.bump_char(); + } + None => {} + } + } + + fn skip_string_literal(&mut self) { + let Some(quote) = self.current_byte() else { + return; + }; + if quote != b'\'' && quote != b'"' { + return; + } + self.pos += 1; + while !self.is_eof() { + let Some(ch) = self.bump_char() else { + return; + }; + if ch == '\\' { + let _ = self.bump_char(); + } else if ch == char::from(quote) { + return; + } + } + } + + fn skip_template_literal(&mut self) { + if self.current_byte() != Some(b'`') { + return; + } + self.pos += 1; + while !self.is_eof() { + let Some(ch) = self.bump_char() else { + return; + }; + if ch == '\\' { + let _ = self.bump_char(); + } else if ch == '`' { + return; + } + } + } + + fn consume_keyword(&mut self, keyword: &str) -> bool { + if !self.keyword_at(keyword) { + return false; + } + self.pos += keyword.len(); + true + } + + fn keyword_at(&self, keyword: &str) -> bool { + if !self.starts_with(keyword) { + return false; + } + let before_ok = self.source[..self.pos] + .chars() + .next_back() + .is_none_or(|ch| !is_ident_continue(ch)); + let after_pos = self.pos + keyword.len(); + let after_ok = self + .source + .get(after_pos..) + .and_then(|rest| rest.chars().next()) + .is_none_or(|ch| !is_ident_continue(ch)); + before_ok && after_ok + } + + fn consume_byte(&mut self, byte: u8) -> bool { + if self.current_byte() != Some(byte) { + return false; + } + self.pos += 1; + true + } + + fn current_byte(&self) -> Option { + self.source.as_bytes().get(self.pos).copied() + } + + fn starts_with(&self, value: &str) -> bool { + self.source + .get(self.pos..) + .is_some_and(|rest| rest.starts_with(value)) + } + + fn bump_char(&mut self) -> Option { + let ch = self.source.get(self.pos..)?.chars().next()?; + self.pos += ch.len_utf8(); + Some(ch) + } + + fn is_eof(&self) -> bool { + self.pos >= self.source.len() + } +} + +fn collect_leaves<'a>( + properties: &'a [Property], + prefix: &mut Vec, + out: &mut Vec>, +) { + for property in properties { + prefix.push(property.key.clone()); + match &property.value { + LiteralValue::Object(children) => collect_leaves(children, prefix, out), + LiteralValue::String(_) | LiteralValue::Number(_) => out.push(TokenLeaf { + path: prefix.clone(), + value: &property.value, + }), + } + let _ = prefix.pop(); + } +} + +fn merge_leaf( + config: &mut Config, + path: &Path, + object_name: &str, + leaf: &TokenLeaf<'_>, + import: &mut TokenModuleImport, +) { + let mut hints = hint_tokens(object_name, &leaf.path); + if !has_any_token_hint(&hints) + && let Some(stem) = path.file_stem().and_then(|s| s.to_str()) + { + hints.extend(split_hint(stem)); + } + let token_name = token_path_key(&leaf.path); + + if has_font_family_hint(&hints) + && let LiteralValue::String(value) = leaf.value + { + add_font_families(config, value, import); + return; + } + + if has_font_weight_hint(&hints) + && let Some(weight) = parse_weight(leaf.value) + { + if !config.type_scale.weights.contains(&weight) { + config.type_scale.weights.push(weight); + import.type_weights += 1; + } + return; + } + + if has_radius_hint(&hints) + && let Some(px) = parse_px(leaf.value) + { + if !config.radius.scale.contains(&px) { + config.radius.scale.push(px); + import.radii += 1; + } + return; + } + + if has_spacing_hint(&hints) + && let Some(px) = parse_px(leaf.value) + { + if !config.spacing.tokens.contains_key(&token_name) { + config.spacing.tokens.insert(token_name, px); + config.spacing.scale.push(px); + import.spacing += 1; + } + return; + } + + if has_type_size_hint(&hints) + && let Some(px) = parse_px(leaf.value) + { + if !config.type_scale.tokens.contains_key(&token_name) { + config.type_scale.tokens.insert(token_name, px); + config.type_scale.scale.push(px); + import.type_sizes += 1; + } + return; + } + + if has_color_hint(&hints) + && let LiteralValue::String(value) = leaf.value + && is_hex_color(value) + && !config.color.tokens.contains_key(&token_name) + { + config + .color + .tokens + .insert(token_name, value.trim().to_owned()); + import.colors += 1; + } +} + +fn hint_tokens(object_name: &str, token_path: &[String]) -> Vec { + let mut hints = Vec::new(); + hints.extend(split_hint(object_name)); + for segment in token_path { + hints.extend(split_hint(segment)); + } + hints +} + +fn split_hint(value: &str) -> Vec { + let mut out = Vec::new(); + let mut current = String::new(); + let mut previous_was_lower_or_digit = false; + + for ch in value.chars() { + if ch.is_ascii_alphanumeric() { + if ch.is_ascii_uppercase() && previous_was_lower_or_digit && !current.is_empty() { + out.push(std::mem::take(&mut current)); + } + current.push(ch.to_ascii_lowercase()); + previous_was_lower_or_digit = ch.is_ascii_lowercase() || ch.is_ascii_digit(); + } else { + if !current.is_empty() { + out.push(std::mem::take(&mut current)); + } + previous_was_lower_or_digit = false; + } + } + + if !current.is_empty() { + out.push(current); + } + out +} + +fn has_font_family_hint(hints: &[String]) -> bool { + has_joined(hints, "fontfamily") + || has_joined(hints, "fontfamilies") + || hints.iter().any(|hint| hint == "families") +} + +fn has_font_weight_hint(hints: &[String]) -> bool { + has_joined(hints, "fontweight") + || has_joined(hints, "fontweights") + || hints + .iter() + .any(|hint| hint == "weights" || hint == "weight") +} + +fn has_radius_hint(hints: &[String]) -> bool { + hints + .iter() + .any(|hint| hint == "radius" || hint == "radii" || hint == "borderradius") + || has_joined(hints, "borderradius") +} + +fn has_spacing_hint(hints: &[String]) -> bool { + hints + .iter() + .any(|hint| hint == "spacing" || hint == "space") +} + +fn has_type_size_hint(hints: &[String]) -> bool { + has_joined(hints, "fontsize") + || has_joined(hints, "fontsizes") + || hints + .iter() + .any(|hint| matches!(hint.as_str(), "typography" | "type" | "font" | "text")) +} + +fn has_color_hint(hints: &[String]) -> bool { + hints.iter().any(|hint| hint == "color" || hint == "colors") + || has_joined(hints, "color") + || has_joined(hints, "colors") +} + +fn has_any_token_hint(hints: &[String]) -> bool { + has_font_family_hint(hints) + || has_font_weight_hint(hints) + || has_radius_hint(hints) + || has_spacing_hint(hints) + || has_type_size_hint(hints) + || has_color_hint(hints) +} + +fn has_joined(hints: &[String], needle: &str) -> bool { + let joined = hints.join(""); + joined.contains(needle) +} + +fn parse_px(value: &LiteralValue) -> Option { + match value { + LiteralValue::String(raw) => { + let trimmed = raw.trim(); + let number = trimmed.strip_suffix("px")?.trim(); + decimal_to_u32(number) + } + LiteralValue::Number(raw) => decimal_to_u32(raw), + LiteralValue::Object(_) => None, + } +} + +fn decimal_to_u32(raw: &str) -> Option { + let number = raw.parse::().ok()?; + if !number.is_finite() || number.is_sign_negative() || number > f64::from(u32::MAX) { + return None; + } + // The finite/sign/range checks above make the rounded value fit in u32. + #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] + Some(number.round() as u32) +} + +fn parse_weight(value: &LiteralValue) -> Option { + let parsed = match value { + LiteralValue::Number(raw) => raw.parse::().ok(), + LiteralValue::String(raw) => match raw.trim().to_ascii_lowercase().as_str() { + "thin" | "hairline" => Some(100), + "extra-light" | "extralight" | "ultralight" => Some(200), + "light" => Some(300), + "regular" | "normal" => Some(400), + "medium" => Some(500), + "semi-bold" | "semibold" | "demibold" => Some(600), + "bold" => Some(700), + "extra-bold" | "extrabold" | "ultrabold" => Some(800), + "black" | "heavy" => Some(900), + other => other.parse::().ok(), + }, + LiteralValue::Object(_) => None, + }?; + u16::try_from(parsed).ok() +} + +fn add_font_families(config: &mut Config, raw: &str, import: &mut TokenModuleImport) { + for family in split_font_stack(raw) { + if !config.type_scale.families.iter().any(|f| f == &family) { + config.type_scale.families.push(family); + import.type_families += 1; + } + } +} + +fn split_font_stack(raw: &str) -> Vec { + raw.split(',') + .filter_map(|part| { + let trimmed = part.trim().trim_matches(['\'', '"']).trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed.to_owned()) + } + }) + .collect() +} + +fn is_hex_color(raw: &str) -> bool { + let trimmed = raw.trim(); + let Some(body) = trimmed.strip_prefix('#') else { + return false; + }; + matches!(body.len(), 3 | 4 | 6 | 8) && body.chars().all(|ch| ch.is_ascii_hexdigit()) +} + +fn token_path_key(path: &[String]) -> String { + path.join("/") +} + +fn token_sort_key(value: &str) -> String { + split_hint(value).join("") +} + +fn is_ident_start(ch: char) -> bool { + ch == '_' || ch == '$' || ch.is_ascii_alphabetic() +} + +fn is_ident_continue(ch: char) -> bool { + is_ident_start(ch) || ch.is_ascii_digit() +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + + #[test] + fn merges_flappy_gouda_style_literal_exports() { + let source = r#" + export const SPACING = { + 0.5: '2px', + 1: '4px', + 1.5: '6px', + } as const; + + export const RADIUS = { + sm: '4px', + md: '6px', + lg: '8px', + xl: '12px', + '2xl': '16px', + pill: '100px', + } as const; + + export const COLOR_TOKENS = { + navy: '#0A3D5C', + violet: '#5AAFA5', + } as const; + + export const STATUS_COLORS = { + success: '#22c55e', + } as const; + + export const COLOR_RGB = { + navy: '10 61 92', + } as const; + + export const RGBA_TOKENS = { + overlay: `rgba(${COLOR_RGB.navy} / 0.45)`, + } as const; + + export const FONT_FAMILY = { + heading: '"Poppins", sans-serif', + body: '"apertura", "Inter", system-ui, sans-serif', + } as const; + + export const FONT_SIZE = { + '2xs': '9px', + xs: '10px', + } as const; + + export const FONT_WEIGHT = { + normal: 400, + semibold: 600, + bold: 700, + extrabold: 800, + } as const; + + export const DESIGN_TOKENS = { + colors: COLOR_TOKENS, + } as const; + "#; + let mut config = Config::default(); + + let import = merge_literal_token_module( + &mut config, + Path::new("packages/types/src/tokens/spacing.ts"), + source, + ); + + assert_eq!(import.spacing, 3); + assert_eq!(config.spacing.tokens["0.5"], 2); + assert_eq!(config.spacing.tokens["1.5"], 6); + assert_eq!(config.spacing.scale, vec![2, 4, 6]); + + assert_eq!(import.radii, 6); + assert_eq!(config.radius.scale, vec![4, 6, 8, 12, 16, 100]); + + assert_eq!(import.colors, 3); + assert_eq!(config.color.tokens["navy"], "#0A3D5C"); + assert_eq!(config.color.tokens["success"], "#22c55e"); + assert!(!config.color.tokens.contains_key("overlay")); + + assert_eq!(import.type_sizes, 2); + assert_eq!(config.type_scale.tokens["2xs"], 9); + assert_eq!(config.type_scale.tokens["xs"], 10); + assert_eq!(config.type_scale.scale, vec![9, 10]); + + assert_eq!(import.type_weights, 4); + assert_eq!(config.type_scale.weights, vec![400, 600, 700, 800]); + + assert_eq!(import.type_families, 5); + assert!(config.type_scale.families.contains(&"Poppins".to_owned())); + assert!(config.type_scale.families.contains(&"apertura".to_owned())); + assert!(config.type_scale.families.contains(&"Inter".to_owned())); + assert!(config.type_scale.families.contains(&"system-ui".to_owned())); + assert!( + config + .type_scale + .families + .contains(&"sans-serif".to_owned()) + ); + } + + #[test] + fn skips_unsupported_numeric_literals_without_truncating() { + let source = r" + export const SPACING = { + ok: 8, + decimal: 1.5, + exponent: 1e3, + separator: 1_000, + hex: 0x10, + binary: 0b10, + octal: 0o10, + bigint: 100n, + unit: 12px, + } as const; + + export const FONT_WEIGHT = { + regular: 400, + badBigInt: 700n, + } as const; + "; + let mut config = Config::default(); + + let import = merge_literal_token_module( + &mut config, + Path::new("packages/types/src/tokens/spacing.ts"), + source, + ); + + assert_eq!(import.spacing, 2); + assert_eq!(config.spacing.tokens["ok"], 8); + assert_eq!(config.spacing.tokens["decimal"], 2); + for key in [ + "exponent", + "separator", + "hex", + "binary", + "octal", + "bigint", + "unit", + ] { + assert!(!config.spacing.tokens.contains_key(key)); + } + + assert_eq!(import.type_weights, 1); + assert_eq!(config.type_scale.weights, vec![400]); + } +} diff --git a/crates/plumb-codegen/src/walk.rs b/crates/plumb-codegen/src/walk.rs index 9fc804d..d27fa9c 100644 --- a/crates/plumb-codegen/src/walk.rs +++ b/crates/plumb-codegen/src/walk.rs @@ -32,6 +32,19 @@ const SKIPPED_DIRS: &[&str] = &[ "target", ]; +/// Workspace-root marker files. A `package.json` marker is handled +/// separately because it must declare a `workspaces` field. +const WORKSPACE_MARKER_FILES: &[&str] = &[ + "pnpm-workspace.yaml", + "pnpm-workspace.yml", + "lerna.json", + "nx.json", + "rush.json", +]; + +/// TypeScript/JavaScript extensions accepted under package token dirs. +const TOKEN_MODULE_EXTENSIONS: &[&str] = &["js", "jsx", "ts", "tsx"]; + /// Discovered token-source paths, grouped by kind. /// /// Each list is sorted so the caller-visible output is deterministic. @@ -46,6 +59,9 @@ pub(crate) struct Walked { pub(crate) css_files: Vec, /// DTCG token JSON files (`*.tokens.json` or under `tokens/`). pub(crate) dtcg_files: Vec, + /// Literal TypeScript/JavaScript token modules discovered from a + /// workspace package token directory. + pub(crate) ts_token_modules: Vec, } /// Walk `source_dir` and return a [`Walked`] with token-source paths @@ -63,9 +79,12 @@ pub(crate) fn walk(source_dir: &Path) -> Result { walked.tailwind_configs.sort(); walk_dir(source_dir, source_dir, 0, &mut walked)?; + discover_workspace_token_modules(source_dir, &mut walked)?; walked.css_files.sort(); walked.dtcg_files.sort(); + walked.ts_token_modules.sort(); + walked.ts_token_modules.dedup(); Ok(walked) } @@ -80,15 +99,42 @@ fn walk_dir( return Ok(()); } - let entries = match std::fs::read_dir(dir) { - Ok(entries) => entries, - Err(source) => { - return Err(CodegenError::Io { - path: dir.display().to_string(), - source, - }); + let sorted = read_sorted_paths(dir)?; + for path in sorted { + let file_type = match std::fs::symlink_metadata(&path) { + Ok(meta) => meta.file_type(), + Err(_) => continue, + }; + if file_type.is_symlink() { + // Skip symlinks; they could escape the source tree, and + // following them would risk cycles. + continue; } - }; + if file_type.is_dir() { + let name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default(); + if name.is_empty() || name.starts_with('.') || SKIPPED_DIRS.contains(&name) { + continue; + } + walk_dir(root, &path, depth + 1, walked)?; + continue; + } + if !file_type.is_file() { + continue; + } + classify_file(root, &path, walked); + } + + Ok(()) +} + +fn read_sorted_paths(dir: &Path) -> Result, CodegenError> { + let entries = std::fs::read_dir(dir).map_err(|source| CodegenError::Io { + path: dir.display().to_string(), + source, + })?; let mut sorted: Vec = Vec::new(); for entry in entries { @@ -99,15 +145,59 @@ fn walk_dir( sorted.push(entry.path()); } sorted.sort(); + Ok(sorted) +} - for path in sorted { +fn discover_workspace_token_modules( + source_dir: &Path, + walked: &mut Walked, +) -> Result<(), CodegenError> { + let Some(workspace_root) = find_workspace_root(source_dir) else { + return Ok(()); + }; + let packages_dir = workspace_root.join("packages"); + if !packages_dir.is_dir() { + return Ok(()); + } + + for package_path in read_sorted_paths(&packages_dir)? { + let file_type = match std::fs::symlink_metadata(&package_path) { + Ok(meta) => meta.file_type(), + Err(_) => continue, + }; + if file_type.is_symlink() || !file_type.is_dir() { + continue; + } + let Some(name) = package_path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + if name.is_empty() || name.starts_with('.') || SKIPPED_DIRS.contains(&name) { + continue; + } + let token_dir = package_path.join("src").join("tokens"); + if token_dir.is_dir() { + collect_token_modules(&token_dir, 0, walked)?; + } + } + + Ok(()) +} + +fn collect_token_modules( + dir: &Path, + depth: usize, + walked: &mut Walked, +) -> Result<(), CodegenError> { + if depth > MAX_WALK_DEPTH { + return Ok(()); + } + + for path in read_sorted_paths(dir)? { let file_type = match std::fs::symlink_metadata(&path) { Ok(meta) => meta.file_type(), Err(_) => continue, }; if file_type.is_symlink() { - // Skip symlinks; they could escape the source tree, and - // following them would risk cycles. continue; } if file_type.is_dir() { @@ -118,18 +208,69 @@ fn walk_dir( if name.is_empty() || name.starts_with('.') || SKIPPED_DIRS.contains(&name) { continue; } - walk_dir(root, &path, depth + 1, walked)?; + collect_token_modules(&path, depth + 1, walked)?; continue; } - if !file_type.is_file() { - continue; + if file_type.is_file() && is_token_module_file(&path) { + walked.ts_token_modules.push(path); } - classify_file(root, &path, walked); } Ok(()) } +fn is_token_module_file(path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + return false; + }; + let lower_name = name.to_ascii_lowercase(); + if lower_name.ends_with(".d.ts") { + return false; + } + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| { + TOKEN_MODULE_EXTENSIONS + .iter() + .any(|candidate| ext.eq_ignore_ascii_case(candidate)) + }) +} + +fn find_workspace_root(source_dir: &Path) -> Option { + let mut current = source_dir.to_path_buf(); + loop { + if has_workspace_marker(¤t) { + if current.as_os_str().is_empty() { + return Some(PathBuf::from(".")); + } + return Some(current); + } + if !current.pop() { + return None; + } + } +} + +fn has_workspace_marker(dir: &Path) -> bool { + if WORKSPACE_MARKER_FILES + .iter() + .any(|marker| dir.join(marker).is_file()) + { + return true; + } + package_json_declares_workspaces(&dir.join("package.json")) +} + +fn package_json_declares_workspaces(path: &Path) -> bool { + let Ok(contents) = std::fs::read_to_string(path) else { + return false; + }; + let Ok(value) = serde_json::from_str::(&contents) else { + return false; + }; + value.get("workspaces").is_some() +} + /// Decide which bucket (if any) a single file belongs to. fn classify_file(root: &Path, path: &Path, walked: &mut Walked) { let Some(name) = path.file_name().and_then(|n| n.to_str()) else { @@ -272,4 +413,51 @@ mod tests { let walked = walk(dir.path()).unwrap(); assert!(walked.css_files.is_empty()); } + + #[test] + fn walk_from_app_subdir_finds_workspace_token_modules() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("pnpm-workspace.yaml"), + "packages:\n - apps/*\n - packages/*\n", + ) + .unwrap(); + let app = dir.path().join("apps/web"); + std::fs::create_dir_all(&app).unwrap(); + let tokens = dir.path().join("packages/types/src/tokens"); + std::fs::create_dir_all(&tokens).unwrap(); + std::fs::write( + tokens.join("spacing.ts"), + "export const SPACING = {} as const;\n", + ) + .unwrap(); + std::fs::write( + tokens.join("colors.jsx"), + "export const COLORS = {} as const;\n", + ) + .unwrap(); + + let walked = walk(&app).unwrap(); + + assert_eq!( + walked.ts_token_modules, + vec![tokens.join("colors.jsx"), tokens.join("spacing.ts")] + ); + } + + #[test] + fn package_json_workspace_marker_requires_top_level_workspaces_field() { + let dir = tempfile::tempdir().unwrap(); + let package_json = dir.path().join("package.json"); + + std::fs::write(&package_json, r#"{ "scripts": { "echo": "workspaces" } }"#).unwrap(); + assert!(!package_json_declares_workspaces(&package_json)); + + std::fs::write( + &package_json, + r#"{ "private": true, "workspaces": { "packages": ["apps/*"] } }"#, + ) + .unwrap(); + assert!(package_json_declares_workspaces(&package_json)); + } }