From 29a3f4fd8d3c48517b77e19f26f81cc950cb6582 Mon Sep 17 00:00:00 2001 From: kirill237 Date: Fri, 1 Aug 2025 09:22:53 +0300 Subject: [PATCH 1/2] Adding encoding specification parameter when opening yaml file Adding the ability to specify in which encoding the file should be opened to generate a table from yaml --- src/mkdocs_table_reader_plugin/readers.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/mkdocs_table_reader_plugin/readers.py b/src/mkdocs_table_reader_plugin/readers.py index e60f25d..f6f113b 100644 --- a/src/mkdocs_table_reader_plugin/readers.py +++ b/src/mkdocs_table_reader_plugin/readers.py @@ -133,15 +133,23 @@ def read_excel(*args, **kwargs) -> str: @ParseArgs def pd_read_yaml(*args, **kwargs) -> str: + defaultEncoding = "UTF-8" + if "encoding" in kwargs: + defaultEncoding = kwargs["encoding"] + del kwargs["encoding"] json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0]) as f: + with open(args[0], "r", encoding=defaultEncoding) as f: df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) return df @ParseArgs def read_yaml(*args, **kwargs) -> str: + defaultEncoding = "UTF-8" + if "encoding" in kwargs: + defaultEncoding = kwargs["encoding"] + del kwargs["encoding"] json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0]) as f: + with open(args[0], "r", encoding=defaultEncoding) as f: df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.json_normalize) @@ -194,4 +202,4 @@ def read_raw(*args, **kwargs) -> str: "pd_read_json": pd_read_json, "pd_read_feather": pd_read_feather, } -MACROS = {**READERS, **MACRO_ONLY} \ No newline at end of file +MACROS = {**READERS, **MACRO_ONLY} From d5b976e49ca093ccef25735be6c55ad7cf25b93d Mon Sep 17 00:00:00 2001 From: Tim Vink Date: Mon, 14 Sep 2026 12:20:59 +0200 Subject: [PATCH 2/2] Support encoding argument in read_yaml and read_raw Simplify the encoding handling, add it to read_raw() as well, document it and add a test with cp1251 encoded files. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JCJnK1rDt8rmDsXTFc7AHE --- docs/readers.md | 6 ++++++ src/mkdocs_table_reader_plugin/readers.py | 17 ++++++--------- .../encoding/assets/tables/cp1251_table.md | 3 +++ .../encoding/assets/tables/cp1251_table.yml | 4 ++++ tests/fixtures/encoding/docs/index.md | 11 ++++++++++ tests/fixtures/encoding/mkdocs.yml | 6 ++++++ tests/test_build.py | 21 +++++++++++++++++++ 7 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 tests/fixtures/encoding/assets/tables/cp1251_table.md create mode 100644 tests/fixtures/encoding/assets/tables/cp1251_table.yml create mode 100644 tests/fixtures/encoding/docs/index.md create mode 100644 tests/fixtures/encoding/mkdocs.yml diff --git a/docs/readers.md b/docs/readers.md index 50ed597..fefb4e5 100644 --- a/docs/readers.md +++ b/docs/readers.md @@ -80,6 +80,8 @@ Example: {{ read_yaml('assets/tables/yaml_table.yml') | add_indentation(spaces=4) }} +The file is read as UTF-8. Use the `encoding` argument for files in another encoding, for example {% raw %}`{{ read_yaml('assets/tables/yaml_table.yml', encoding='cp1251') }}`{% endraw %}. + ### `read_table` @@ -210,6 +212,8 @@ Example: {{ read_raw('assets/tables/markdown_table.md') | add_indentation(spaces=4) }} +The file is read as UTF-8. Use the `encoding` argument for files in another encoding, for example {% raw %}`{{ read_raw('assets/tables/markdown_table.md', encoding='cp1251') }}`{% endraw %}. + ## Macros @@ -269,6 +273,8 @@ Example: {{ pd_read_yaml('assets/tables/yaml_table.yml').to_markdown(tablefmt="pipe", index=False) | add_indentation(spaces=4) }} +The file is read as UTF-8. Use the `encoding` argument for files in another encoding, for example {% raw %}`{{ pd_read_yaml('assets/tables/yaml_table.yml', encoding='cp1251') }}`{% endraw %}. + ### `pd_read_table` diff --git a/src/mkdocs_table_reader_plugin/readers.py b/src/mkdocs_table_reader_plugin/readers.py index f6f113b..86bbea4 100644 --- a/src/mkdocs_table_reader_plugin/readers.py +++ b/src/mkdocs_table_reader_plugin/readers.py @@ -133,23 +133,17 @@ def read_excel(*args, **kwargs) -> str: @ParseArgs def pd_read_yaml(*args, **kwargs) -> str: - defaultEncoding = "UTF-8" - if "encoding" in kwargs: - defaultEncoding = kwargs["encoding"] - del kwargs["encoding"] + encoding = kwargs.pop("encoding", "utf-8") json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0], "r", encoding=defaultEncoding) as f: + with open(args[0], encoding=encoding) as f: df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) return df @ParseArgs def read_yaml(*args, **kwargs) -> str: - defaultEncoding = "UTF-8" - if "encoding" in kwargs: - defaultEncoding = kwargs["encoding"] - del kwargs["encoding"] + encoding = kwargs.pop("encoding", "utf-8") json_kwargs = kwargs_in_func(kwargs, pd.json_normalize) - with open(args[0], "r", encoding=defaultEncoding) as f: + with open(args[0], encoding=encoding) as f: df = pd.json_normalize(yaml.safe_load(f), **json_kwargs) markdown_kwargs = kwargs_not_in_func(kwargs, pd.json_normalize) @@ -178,7 +172,8 @@ def read_raw(*args, **kwargs) -> str: Returns: str: file contents """ - with open(args[0]) as f: + encoding = kwargs.pop("encoding", "utf-8") + with open(args[0], encoding=encoding) as f: return f.read() diff --git a/tests/fixtures/encoding/assets/tables/cp1251_table.md b/tests/fixtures/encoding/assets/tables/cp1251_table.md new file mode 100644 index 0000000..c894265 --- /dev/null +++ b/tests/fixtures/encoding/assets/tables/cp1251_table.md @@ -0,0 +1,3 @@ +| Òîâàð | Öåíà | +|--------|-----:| +| Ñûð | 539956 | diff --git a/tests/fixtures/encoding/assets/tables/cp1251_table.yml b/tests/fixtures/encoding/assets/tables/cp1251_table.yml new file mode 100644 index 0000000..08d7620 --- /dev/null +++ b/tests/fixtures/encoding/assets/tables/cp1251_table.yml @@ -0,0 +1,4 @@ +- Òîâàð: Õëåá + Öåíà: 531456 +- Òîâàð: Ìîëîêî + Öåíà: 80 diff --git a/tests/fixtures/encoding/docs/index.md b/tests/fixtures/encoding/docs/index.md new file mode 100644 index 0000000..79f7da3 --- /dev/null +++ b/tests/fixtures/encoding/docs/index.md @@ -0,0 +1,11 @@ +# Test page + +Files that are not UTF-8 encoded can be read by specifying the `encoding`: + +## read_yaml + +{{ read_yaml('assets/tables/cp1251_table.yml', encoding='cp1251') }} + +## read_raw + +{{ read_raw('assets/tables/cp1251_table.md', encoding='cp1251') }} diff --git a/tests/fixtures/encoding/mkdocs.yml b/tests/fixtures/encoding/mkdocs.yml new file mode 100644 index 0000000..cea9ccc --- /dev/null +++ b/tests/fixtures/encoding/mkdocs.yml @@ -0,0 +1,6 @@ +site_name: test git_table_reader site +use_directory_urls: false + +plugins: + - search + - table-reader diff --git a/tests/test_build.py b/tests/test_build.py index 46d50a7..004250b 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -387,3 +387,24 @@ def test_macros_jinja2_syntax(tmp_path): contents = page_with_tag.read_text() assert re.search(r"531456", contents) + +def test_non_utf8_encoding(tmp_path): + """ + A project where files are not UTF-8 encoded, and 'encoding' is specified. + """ + + tmp_proj = setup_clean_mkdocs_folder( + "tests/fixtures/encoding/mkdocs.yml", tmp_path + ) + + result = build_docs_setup(tmp_proj) + assert result.exit_code == 0, "'mkdocs build' command failed" + + page_with_tag = tmp_proj / "site/index.html" + contents = page_with_tag.read_text(encoding="utf-8") + # read_yaml() inserted the cp1251 encoded yaml file + assert re.search(r"531456", contents) + assert re.search(r"Хлеб", contents) + # read_raw() inserted the cp1251 encoded markdown file + assert re.search(r"539956", contents) + assert re.search(r"Сыр", contents)