review of #190: each language of the newsletter line says which it is, and two smaller fixes

From the CodeRabbit review of #190, each remark checked first.

The newsletter's page says it is English since this pull request, so its
line in Polish and Japanese read as English to a screen reader. Each half
now says its language in a span of its own, the Polish one alone where the
character set holds no Japanese. On two lines where the Japanese half
comes: the spans made the line 83 characters, past the 78 RFC 5322 asks
for, and 8bit has no soft line break - the structural checker caught it. A
message laid out simple is unchanged. The checker asks for both marks, and
the newsletter canary breaks each.

MimeKit with no code page provider tries UTF-8 and then Latin-1, read from
TextPart.cs at 4.18.1, rather than reading the text as Latin-1 outright.
The charset description says so, in English and Polish.

The checker finds the title rather than matching a lazy pattern, which over
many unclosed title tags took 19.6 s for 20 000 of them and 0.8 s for 5000.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: donislawdev

CHANGELOG.md

@@ -34,8 +34,11 @@ because it turns other people's test suites red.
   name. The smallest catalogue grows from 6924 B to 6944 B.
 - **A newsletter's page says its language.** With `layout=newsletter` the
   HTML part opens with `<html lang="en">`, as the pages of the `html` format
-  do. The smallest newsletter grows by 10 B, and by 12 B in quoted-printable
-  and base64. A message laid out `simple` is unchanged.
+  do, and the line in Polish and Japanese marks each half with its own
+  language, `<span lang="pl">` and `<span lang="ja">`, so a screen reader
+  does not read them as English. The smallest newsletter grows by 10 B in
+  7bit, and by 57 B to 78 B with 8bit, quoted-printable and base64, which
+  carry that line. A message laid out `simple` is unchanged.
 
 ### Added
 
@@ -337,8 +340,8 @@ because it turns other people's test suites red.
 
 - **The `eml` settings say what Thunderbird and MimeKit do with them.**
   Measured with Thunderbird 157.0.1 and MimeKit 4.18.1. `charset` says that
-  MimeKit reads Latin-2 and Windows-1250 right once a code page provider is
-  registered. `message_depth` says that Thunderbird shows 32 levels in full,
+  MimeKit with no code page provider tries UTF-8 and then Latin-1, and reads
+  Latin-2 and Windows-1250 right once one is registered. `message_depth` says that Thunderbird shows 32 levels in full,
   and past that shows only the headers of the 32nd message inside, without a
   word. `filename_style` says that Thunderbird calls an attachment with no
   name Part 1.2, Part 1.3 and so on.

internal/format/eml/message.go

@@ -47,7 +47,7 @@ func newMessage(s settings, seed uint64, children, pics []child) *message {
 	return &message{
 		s: s, id: id, children: children, pictures: pics, seed: seed,
 		plainLine: s.internationalLine("", ""),
-		htmlLine:  s.internationalLine("<p>", "</p>"),
+		htmlLine:  s.htmlInternationalLine(),
 		images:    imageLines(s, pics),
 		frame:     newFrame(s, id),
 	}

internal/format/eml/settings.go

@@ -282,7 +282,7 @@ func charsetProperty() format.Property {
 		Default: CharsetUTF8,
 		Detail: "The character set of the text when text_encoding is not 7bit. iso-8859-2 and windows-1250 hold Polish and no Japanese, " +
 			"and they write ą, ś and ź as different bytes, so a program that takes one for the other shows the wrong letters. " +
-			"MimeKit reads both as Latin-1 until a code page provider is registered, and right once one is. Go's standard library converts no character set at all. " +
+			"MimeKit with no code page provider registered tries UTF-8 and then Latin-1, which shows the Polish letters of both wrong, and reads them right once one is. Go's standard library converts no character set at all. " +
 			"With 7bit the text is plain ASCII and says us-ascii, whatever this is set to.",
 	}
 }

internal/format/eml/text.go

@@ -30,7 +30,8 @@ const (
 // Windows-1250 write as different bytes.
 const (
 	polish        = "Zażółć gęślą jaźń."
-	international = polish + " 日本語のテキストです。"
+	japanese      = "日本語のテキストです。"
+	international = polish + " " + japanese
 )
 
 // singleByte is how ISO-8859-2 and Windows-1250 write the letters of the
@@ -97,7 +98,36 @@ func (s settings) internationalLine(open, close string) string {
 	if !s.international() {
 		return ""
 	}
-	line := open + s.encodedLine() + close
+	return s.textLine(open + s.encodedLine() + close)
+}
+
+// htmlInternationalLine is the line as the HTML part writes it. A newsletter's
+// page says it is English (<html lang="en">), so there each half says which
+// language it is in a span of its own - the Polish half, and the Japanese one
+// where the character set holds it - or a screen reader reads both as English
+// (review of #190, as the html format marks its paragraphs in other scripts).
+// A message laid out simple says no language anywhere and keeps the bytes it
+// always wrote.
+func (s settings) htmlInternationalLine() string {
+	if s.layout != Newsletter {
+		return s.internationalLine("<p>", "</p>")
+	}
+	if !s.international() {
+		return ""
+	}
+	// Two lines rather than one where the Japanese half comes: the spans make
+	// the line 83 characters, past the 78 RFC 5322 asks for, and 8bit has no
+	// soft line break to keep it under (measured by the structural checker).
+	// A line break inside a paragraph is white space, as the space it replaces.
+	if _, single := singleByte[s.charset]; single {
+		return s.textLine(`<p><span lang="pl">` + s.encodedLine() + `</span></p>`)
+	}
+	return s.textLine(`<p><span lang="pl">`+polish+`</span>`) + s.textLine(`<span lang="ja">`+japanese+`</span></p>`)
+}
+
+// textLine is one line of the text in the encoding the text is written in,
+// with the text's line ending.
+func (s settings) textLine(line string) string {
 	if s.encoding == QuotedPrintable {
 		line = quotedPrintable(line, s.eol)
 	}

internal/guard/emlnewsletter_test.go

@@ -139,6 +139,12 @@ func TestTheEMLCheckRefusesANewsletterBrokenOnPurpose(t *testing.T) {
 		// nothing read before (docs/HTML-6-2026-10-15.md).
 		{newsletter, newsletter, replaced(`<html lang="en">`, "<html>"),
 			"layout=newsletter and the HTML lacks: language of the page"},
+		// Each half of the line in Polish and Japanese says its language, since
+		// the page says English (review of #190).
+		{newsletter, newsletter, replaced(`<span lang="pl">`, "<span>"),
+			"layout=newsletter and the HTML lacks: the Polish line marked as Polish"},
+		{newsletter, newsletter, replaced(`<span lang="ja">`, "<span>"),
+			"layout=newsletter and the HTML lacks: the Japanese line marked as Japanese"},
 	}
 	dir := t.TempDir()
 	for i, c := range cases {

internal/guard/testdata/generator-golden.json

@@ -558,14 +558,14 @@
     },
     "eml_8kib_newsletter_alternative_quoted": {
       "bytes": 8192,
-      "sha256": "27e3688910812ba5b68a4ef0f595ee98fb0556eda9f2c18ef391fc486d6f2ff3",
+      "sha256": "5b4918a133ccb3708018ca9bc9c2e8882340d1073f408ae91a7c2ba164f2b528",
       "measured_on": "2026-10-15"
     }
   },
   "remeasured": [
     {
       "on": "2026-10-15",
-      "why": "A page with no label to fill its title carries Page there now, where it carried an empty title - the smallest document and every page under --clean, in every size. The Nu HTML Checker calls an empty title an error, so every such page was invalid HTML since 2026-08-01 (O277). Page rather than the tool's name, because --clean is there to keep the tool's description out of the file. The smallest html page goes from 118 B to 122 B. The catalogue of every element names its forms article tfg-forms, and its two links to it, because DOMPurify takes out an id that shadows document.forms and the links then led nowhere (O282) - the reason the links list is tfg-links. Its hidden field is named tfg-hidden, because a name hidden shadows document.hidden and DOMPurify took that out too, leaving the field out of a submitted form. The page of a newsletter says its language, <html lang=\"en\">, as the html format's pages always did, written in the frame so quoted-printable gives its = as =3D. A message laid out simple is unchanged. This is a breaking change by the owner's decision of 2026-10-15, docs/HTML-6-2026-10-15.md. The two placeholder pages had no golden case and have one from today. Guarded from now on by the structural checker, which refuses a document without a title with text in it and a newsletter page without its language.",
+      "why": "A page with no label to fill its title carries Page there now, where it carried an empty title - the smallest document and every page under --clean, in every size. The Nu HTML Checker calls an empty title an error, so every such page was invalid HTML since 2026-08-01 (O277). Page rather than the tool's name, because --clean is there to keep the tool's description out of the file. The smallest html page goes from 118 B to 122 B. The catalogue of every element names its forms article tfg-forms, and its two links to it, because DOMPurify takes out an id that shadows document.forms and the links then led nowhere (O282) - the reason the links list is tfg-links. Its hidden field is named tfg-hidden, because a name hidden shadows document.hidden and DOMPurify took that out too, leaving the field out of a submitted form. The page of a newsletter says its language, <html lang=\"en\">, as the html format's pages always did, written in the frame so quoted-printable gives its = as =3D. Inside it, the line in Polish and Japanese marks each half with its own lang in a span, on two lines where the Japanese half comes, so the page saying English does not make a screen reader read them as English (review of #190). A message laid out simple is unchanged. This is a breaking change by the owner's decision of 2026-10-15, docs/HTML-6-2026-10-15.md. The two placeholder pages had no golden case and have one from today. Guarded from now on by the structural checker, which refuses a document without a title with text in it and a newsletter page without its language.",
       "files": [
         "eml_8kib_newsletter_alternative_quoted",
         "html_16kib_every",

internal/gui/text/locale/registry/en.json

@@ -1206,8 +1206,8 @@
   },
   "Detail.format/eml.charset": {
     "description": "The sentence under the charset setting of the eml format, after what it takes.",
-    "hash": "sha256-5bc40079d024",
-    "other": "The character set of the text when text_encoding is not 7bit. iso-8859-2 and windows-1250 hold Polish and no Japanese, and they write ą, ś and ź as different bytes, so a program that takes one for the other shows the wrong letters. MimeKit reads both as Latin-1 until a code page provider is registered, and right once one is. Go's standard library converts no character set at all. With 7bit the text is plain ASCII and says us-ascii, whatever this is set to."
+    "hash": "sha256-636bf249f436",
+    "other": "The character set of the text when text_encoding is not 7bit. iso-8859-2 and windows-1250 hold Polish and no Japanese, and they write ą, ś and ź as different bytes, so a program that takes one for the other shows the wrong letters. MimeKit with no code page provider registered tries UTF-8 and then Latin-1, which shows the Polish letters of both wrong, and reads them right once one is. Go's standard library converts no character set at all. With 7bit the text is plain ASCII and says us-ascii, whatever this is set to."
   },
   "Detail.format/eml.filename_style": {
     "description": "The sentence under the filename_style setting of the eml format, after what it takes.",

internal/gui/text/locale/registry/pl.json

@@ -240,7 +240,7 @@
   "Detail.format/eml.attachment_size": { "hash": "sha256-c6ca8985e21d", "other": "Jak duży jest każdy załączony plik. Wiadomość rośnie o mniej więcej jedną trzecią więcej, bo załączone pliki są przesyłane w base64." },
   "Detail.format/eml.attachments": { "hash": "sha256-43c96b5ef075", "other": "Ile plików niesie wiadomość. Gdy pliki nie są jednakowe, użyj zamiast tego contains. mailparser odmawia wiadomości z więcej niż 1000 encji MIME, a manifest podaje ich liczbę dla tej wiadomości jako mime_entities." },
   "Detail.format/eml.body": { "hash": "sha256-5dc479375b75", "other": "Jaki tekst niesie wiadomość. zwykły tekst to jedna część tekstowa, a html jedna część HTML. tekst i html niesie obie z tymi samymi słowami, a program pocztowy pokazuje jedną z nich." },
-  "Detail.format/eml.charset": { "hash": "sha256-5bc40079d024", "other": "Zestaw znaków tekstu, gdy kodowanie tekstu nie jest 7bit. iso-8859-2 i windows-1250 mają polski i nie mają japońskiego, a litery ą, ś i ź zapisują innymi bajtami, więc program, który bierze jeden za drugi, pokazuje złe litery. MimeKit czyta oba jako Latin-1, dopóki nie zarejestruje się dostawcy stron kodowych, a poprawnie, gdy jest zarejestrowany. Biblioteka standardowa Go nie przekształca żadnego zestawu znaków. Przy 7bit tekst jest czystym ASCII i mówi us-ascii, bez względu na to ustawienie." },
+  "Detail.format/eml.charset": { "hash": "sha256-636bf249f436", "other": "Zestaw znaków tekstu, gdy kodowanie tekstu nie jest 7bit. iso-8859-2 i windows-1250 mają polski i nie mają japońskiego, a litery ą, ś i ź zapisują innymi bajtami, więc program, który bierze jeden za drugi, pokazuje złe litery. MimeKit bez zarejestrowanego dostawcy stron kodowych próbuje UTF-8, a potem Latin-1, więc w obu pokazuje polskie litery źle, a poprawnie, gdy dostawca jest zarejestrowany. Biblioteka standardowa Go nie przekształca żadnego zestawu znaków. Przy 7bit tekst jest czystym ASCII i mówi us-ascii, bez względu na to ustawienie." },
   "Detail.format/eml.filename_style": { "hash": "sha256-adf72ea5fcb6", "other": "Jak zapisane są nazwy załączonych plików i obrazków. rfc2231 to sposób standardowy. rfc2047 to forma kodowana, której RFC 2047 zabrania wewnątrz nazwy, a wiele czytników i tak ją dekoduje. oba zapisuje obie, a content-type umieszcza nazwę tylko tam, gdzie szukały jej stare programy pocztowe. brak nie zapisuje nazwy wcale, a program, który jej potrzebuje, musi ją wymyślić - Thunderbird 157 nazywa je Part 1.2, Part 1.3 i tak dalej. Biblioteka standardowa Go nie czyta nazwy z content-type, a z rfc2047 oddaje sam zakodowany tekst. Gdy headers nie jest ustawione na kodowane, nazwa jest zapisana tak, jak jest, i zmienia się tylko jej miejsce, więc rfc2047 i oba zapisują te same bajty." },
   "Detail.format/eml.headers": { "hash": "sha256-5002e2b42e63", "other": "Czy temat, nazwy nadawcy i odbiorcy oraz nazwy załączonych plików mają litery spoza ASCII, i jak są zapisane. kodowane zapisuje je tak, jak każą RFC 2047 i RFC 2231. utf8 zapisuje je tak, jak są (RFC 6532), a pakiet email Pythona czyta to i zgłasza defekt w liniach From i To." },
   "Detail.format/eml.inline": { "hash": "sha256-65d689f88414", "other": "Ile obrazków pokazuje tekst. Przy html oraz tekst i html każdy leży obok HTML-a, a HTML wskazuje go przez cid:. Przy zwykłym tekście nie ma HTML-a, z którego można by wskazać, więc obrazki idą za tekstem jako obrazki do pokazania w treści. mailparser wpisuje je na listę załączników, pakiet email Pythona robi to, gdy HTML jest sam, a MimeKit nigdy." },

internal/oracle/strict.py

@@ -1096,8 +1096,12 @@ def check_html(data, settings):
         # A title with text in it. The standard asks a title for text, and the
         # Nu HTML Checker calls an empty one an error - which every page with no
         # label had until 2026-10-15, the smallest and every --clean one (O277).
-        title = re.search(r"<title>(.*?)</title>", text, re.S | re.I)
-        if not title or not title.group(1).strip():
+        # Found rather than matched: a lazy pattern over a page of many
+        # unclosed <title> tags reads the rest of the page from each of them,
+        # the quadratic shape eml_head left behind (review of #190).
+        start = text.find("<title>")
+        end = text.find("</title>", start) if start >= 0 else -1
+        if end < 0 or not text[start + len("<title>"):end].strip():
             fail("the document has no title with text in it, which the HTML standard asks of every document")
     else:
         # A fragment is what a content field or the body of an email holds. The
@@ -3212,6 +3216,12 @@ def eml_frame(text, s, unsubscribe):
         (re.search(r'<a class="button" href="' + re.escape(EML_READ_MORE) + r'"[^>]*>\s*Read more</a>', body), "button"),
         (f'<a href="{unsubscribe}">Unsubscribe</a>' in body, "link to unsubscribe at the address in List-Unsubscribe"),
     ]
+    if s["text_encoding"] != "7bit":
+        # Each half of the line in Polish and Japanese says its language, since
+        # the page around it says English (review of #190).
+        page.append(('<span lang="pl">' in body, "the Polish line marked as Polish"))
+        if s["charset"] == "utf-8":
+            page.append(('<span lang="ja">' in body, "the Japanese line marked as Japanese"))
     missing = [what for present, what in page if not present]
     if missing:
         fail(f"layout=newsletter and the HTML lacks: {', '.join(missing)}")