Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/forge_markdown_stream/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ streamdown-render = "0.1.4"
syntect.workspace = true
colored.workspace = true
unicode-width = "0.2"
unicode-segmentation = "1.12"
terminal-colorsaurus = "1.0.3"

[dev-dependencies]
insta.workspace = true
strip-ansi-escapes.workspace = true
pretty_assertions.workspace = true
28 changes: 25 additions & 3 deletions crates/forge_markdown_stream/src/heading.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
//! Heading rendering with theme-based styling.

use streamdown_render::simple_wrap;

use crate::inline::render_inline_content;
use crate::style::{HeadingStyler, InlineStyler};
use crate::utils::simple_wrap_preserving_spaces;

/// Render a heading with appropriate styling.
pub fn render_heading<S: InlineStyler + HeadingStyler>(
Expand All @@ -30,7 +29,7 @@ pub fn render_heading<S: InlineStyler + HeadingStyler>(
// chars, etc.)
let prefix_display_width = level as usize + 1;
let content_width = width.saturating_sub(prefix_display_width);
let lines = simple_wrap(&rendered_content, content_width);
let lines = simple_wrap_preserving_spaces(&rendered_content, content_width);
let mut result = Vec::new();

for line in lines {
Expand Down Expand Up @@ -201,6 +200,29 @@ mod tests {
");
}

#[test]
fn test_h3_wrapping_preserves_korean_word_spaces() {
let actual = render_with_width(3, "한글 공백 보존 확인", 12);

insta::assert_snapshot!(actual, @r"
<dim><h3>###</h3></dim> <h3>한글</h3>
<dim><h3>###</h3></dim> <h3>공백</h3>
<dim><h3>###</h3></dim> <h3>보존</h3>
<dim><h3>###</h3></dim> <h3>확인</h3>
");
}

#[test]
fn test_h3_wrapping_splits_long_tokens() {
let actual = render_with_width(3, "supercalifragilistic", 12);

insta::assert_snapshot!(actual, @r"
<dim><h3>###</h3></dim> <h3>supercal</h3>
<dim><h3>###</h3></dim> <h3>ifragili</h3>
<dim><h3>###</h3></dim> <h3>stic</h3>
");
}

#[test]
fn test_special_characters() {
insta::assert_snapshot!(render(2, "Hello & Goodbye < World >"), @"<dim><h2>##</h2></dim> <h2>Hello & Goodbye < World ></h2>");
Expand Down
124 changes: 122 additions & 2 deletions crates/forge_markdown_stream/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
//!
//! fn main() -> io::Result<()> {
//! let mut renderer = StreamdownRenderer::new(io::stdout(), 80);
//!
//!
//! // Push tokens as they arrive from LLM
//! renderer.push("Hello ")?;
//! renderer.push("**world**!\n")?;
//!
//!
//! // Finish rendering
//! let _ = renderer.finish()?;
//! Ok(())
Expand Down Expand Up @@ -109,3 +109,123 @@ impl<W: Write> StreamdownRenderer<W> {
Ok(())
}
}

#[cfg(test)]
mod tests {
use pretty_assertions::assert_eq;

use super::StreamdownRenderer;

fn fixture_rendered_output(markdown: &str, width: usize) -> String {
let mut output = Vec::new();
let mut fixture = StreamdownRenderer::new(&mut output, width);
fixture.push(markdown).unwrap();
fixture.finish().unwrap();

let actual = strip_ansi_escapes::strip(output);
String::from_utf8(actual).unwrap().trim_matches('\n').to_string()
}

fn fixture_rendered_output_from_chunks(chunks: &[&str], width: usize) -> String {
let mut output = Vec::new();
let mut fixture = StreamdownRenderer::new(&mut output, width);
for chunk in chunks {
fixture.push(chunk).unwrap();
}
fixture.finish().unwrap();

let actual = strip_ansi_escapes::strip(output);
String::from_utf8(actual).unwrap().trim_matches('\n').to_string()
}

#[test]
fn test_streaming_renderer_preserves_korean_spacing_in_structured_markdown() {
let fixture = concat!(
"## 구현 요약\n",
"- 각 서비스에서 metadata key를 개별 수정하지 않고, object storage 공통 레이어에서 일괄 정규화하도록 반영했습니다.\n",
"## 검토 사항\n",
"- 본 수정은 업로드 시 metadata header 이름 문제를 해결합니다.\n",
"- 추가적인 권한 정책, bucket policy, reverse proxy 제한이 있으면 별도 오류가 발생할 수 있습니다.\n",
);
let actual = fixture_rendered_output(fixture, 200);
let expected = concat!(
"## 구현 요약\n",
"• 각 서비스에서 metadata key를 개별 수정하지 않고, object storage 공통 레이어에서 일괄 정규화하도록 반영했습니다.\n",
"\n",
"## 검토 사항\n",
"• 본 수정은 업로드 시 metadata header 이름 문제를 해결합니다.\n",
"• 추가적인 권한 정책, bucket policy, reverse proxy 제한이 있으면 별도 오류가 발생할 수 있습니다.",
);

assert_eq!(actual, expected);
}

#[test]
fn test_streaming_renderer_preserves_korean_spacing_when_structured_tail_arrives_in_chunks() {
let fixture = [
"## 검토 결과\n",
"- 본 사례는 스트리밍 마크다운 렌더링의 공백 재조합 문제와 관련이 있습니다.\n",
"- 핵심 구현은 공백 보존 래퍼에 위치합니다.\n",
"- 회귀 테스트는 스트리밍 렌더러 검증 항목에 추가되어 있습니다.\n\n",
"후속 작업은 다음과 같습니다.\n",
"1. 변경 사항을 검토 가능한 형식으로 정리합니다.\n",
"2. 실제 대화 출력과 유사한 통합 테스트 범위를 ",
"확장합니다.",
];
let actual = fixture_rendered_output_from_chunks(&fixture, 200);
let expected = concat!(
"## 검토 결과\n",
"• 본 사례는 스트리밍 마크다운 렌더링의 공백 재조합 문제와 관련이 있습니다.\n",
"• 핵심 구현은 공백 보존 래퍼에 위치합니다.\n",
"• 회귀 테스트는 스트리밍 렌더러 검증 항목에 추가되어 있습니다.\n",
"\n",
"후속 작업은 다음과 같습니다.\n",
"1. 변경 사항을 검토 가능한 형식으로 정리합니다.\n",
"2. 실제 대화 출력과 유사한 통합 테스트 범위를 확장합니다.",
);

assert_eq!(actual, expected);
}

#[test]
fn test_streaming_renderer_wraps_blockquotes_with_prefix_width_and_long_tokens() {
let fixture = "> supercalifragilistic\n> 한글 공백\n";
let actual = fixture_rendered_output(fixture, 10);
let expected = concat!(
"│ supercal\n",
"│ ifragili\n",
"│ stic\n",
"│ 한글\n",
"│ 공백"
);

assert_eq!(actual, expected);
}

#[test]
fn test_streaming_renderer_wraps_blockquote_links_without_losing_separator() {
let fixture = "> [링크](https://example.com/very/long/path) 설명\n";
let actual = fixture_rendered_output(fixture, 20);
let expected = concat!(
"│ 링크\n",
"│ (https://example.c\n",
"│ om/very/long/path)\n",
"│ 설명"
);

assert_eq!(actual, expected);
}

#[test]
fn test_streaming_renderer_wraps_nested_blockquotes_with_correct_prefix_width() {
let fixture = ">> supercalifragilistic\n";
let actual = fixture_rendered_output(fixture, 12);
let expected = concat!(
"│ │ supercal\n",
"│ │ ifragili\n",
"│ │ stic"
);

assert_eq!(actual, expected);
}
}
122 changes: 112 additions & 10 deletions crates/forge_markdown_stream/src/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

use streamdown_ansi::utils::visible_length;
use streamdown_parser::ListBullet;
use streamdown_render::text::text_wrap;

use crate::inline::render_inline_content;
use crate::style::{InlineStyler, ListStyler};
use crate::utils::wrap_text_preserving_spaces;

/// Bullet characters for dash lists at different nesting levels.
const BULLETS_DASH: [&str; 4] = ["•", "◦", "▪", "‣"];
Expand Down Expand Up @@ -183,27 +183,25 @@ pub fn render_list_item<S: InlineStyler + ListStyler>(
let next_prefix = format!("{}{}", margin, " ".repeat(content_indent));

// Wrap the content
let wrapped = text_wrap(
let wrapped = wrap_text_preserving_spaces(
&rendered_content,
width,
0,
width.saturating_sub(visible_length(&first_prefix)),
width.saturating_sub(visible_length(&next_prefix)),
&first_prefix,
&next_prefix,
false,
true,
);

if wrapped.is_empty() {
vec![first_prefix]
} else {
wrapped.lines
wrapped
}
}

#[cfg(test)]
mod tests {
use super::*;
use crate::theme::TagStyler;
use crate::theme::{TagStyler, Theme};

fn render(indent: usize, bullet: ListBullet, content: &str) -> String {
let mut state = ListState::default();
Expand All @@ -227,6 +225,28 @@ mod tests {
.join("\n")
}

fn render_visible_with_width(
indent: usize,
bullet: ListBullet,
content: &str,
width: usize,
) -> String {
let mut state = ListState::default();
let actual = render_list_item(
indent,
&bullet,
content,
width,
" ",
&Theme::default(),
&mut state,
)
.join("\n");
let stripped = strip_ansi_escapes::strip(actual.as_bytes());

String::from_utf8(stripped).unwrap()
}

#[test]
fn test_unordered_dash() {
insta::assert_snapshot!(render(0, ListBullet::Dash, "Item one"), @" <dash>•</dash> Item one");
Expand Down Expand Up @@ -333,11 +353,93 @@ mod tests {
40,
);
insta::assert_snapshot!(result, @r"
<dash>•</dash> This is a very long list item that
should wrap to multiple lines
<dash>•</dash> This is a very long
list item that should wrap to
multiple lines
");
}

#[test]
fn test_wrapping_preserves_korean_word_spaces() {
let actual = render_with_width(0, ListBullet::Dash, "한글 공백 보존 확인", 8);
let expected = " <dash>•</dash> 한글\n 공백\n 보존\n 확인";

pretty_assertions::assert_eq!(actual, expected);
}

#[test]
fn test_wrapping_respects_bullet_prefix_width() {
let actual = render_with_width(0, ListBullet::Dash, "한글 공백", 6);
let expected = " <dash>•</dash> 한\n 글\n 공\n 백";

pretty_assertions::assert_eq!(actual, expected);
}

#[test]
fn test_wrapping_respects_checkbox_prefix_width() {
let actual = render_with_width(0, ListBullet::Dash, "[ ] 한글 공백", 8);
let expected = " <dash>•</dash> <unchecked></unchecked> 한\n 글\n 공\n 백";

pretty_assertions::assert_eq!(actual, expected);
}

#[test]
fn test_wrapping_respects_multidigit_ordered_prefix_width() {
let mut fixture = ListState::default();
for index in 1..10 {
let _ = render_list_item(
0,
&ListBullet::Ordered(1),
&format!("예시 {index}"),
8,
" ",
&TagStyler,
&mut fixture,
);
}
let actual = render_list_item(
0,
&ListBullet::Ordered(1),
"한글 공백",
8,
" ",
&TagStyler,
&mut fixture,
)
.join("\n");
let expected = " <num>10.</num> 한\n 글\n 공\n 백";

pretty_assertions::assert_eq!(actual, expected);
}

#[test]
fn test_wrapping_splits_long_tokens() {
let actual = render_with_width(0, ListBullet::Dash, "supercalifragilistic", 10);
let expected = " <dash>•</dash> superc\n alifra\n gilist\n ic";

pretty_assertions::assert_eq!(actual, expected);
}

#[test]
fn test_wrapping_preserves_link_breaks() {
let actual = render_visible_with_width(
0,
ListBullet::Dash,
"[링크](https://example.com/very/long/path) 설명",
14,
);
let expected = concat!(
" • 링크\n",
" (https://e\n",
" xample.com\n",
" /very/long\n",
" /path)\n",
" 설명"
);

pretty_assertions::assert_eq!(actual, expected);
}

#[test]
fn test_list_state_reset() {
let mut state = ListState::default();
Expand Down
Loading
Loading