diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..8240e96
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,41 @@
+name: CI
+
+on:
+ push:
+ branches: [master]
+ pull_request:
+ branches: [master]
+
+jobs:
+ lint:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - uses: astral-sh/ruff-action@v3
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.14"
+ allow-prereleases: true
+ - name: Install dependencies
+ run: pip install phpserialize>=1.3 setuptools>=42 mypy
+ - name: Run mypy
+ run: mypy wpparser/
+
+ test:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+ allow-prereleases: true
+ - name: Install dependencies
+ run: |
+ pip install --upgrade pip
+ pip install pytest phpserialize>=1.3
+ - name: Run tests
+ run: pytest -s tests/
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index f5bd85d..525c146 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
*.pyc
+__pycache__/
.DS_Store
.*.swp
tags
@@ -8,4 +9,6 @@ build
.coverage
*.egg-info/
venv*
-.tox/
\ No newline at end of file
+.tox/
+.pytest_cache/
+.ruff_cache/
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 579d423..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,10 +0,0 @@
-language: python
-python:
- - "pypy"
- - "3.6"
- - "3.7"
- - "3.8"
-install:
- - pip install -r requirements/tests.txt
-script:
- - python runtests.py
diff --git a/MANIFEST.in b/MANIFEST.in
deleted file mode 100644
index d7fd551..0000000
--- a/MANIFEST.in
+++ /dev/null
@@ -1 +0,0 @@
-include README.md LICENSE requirements/install.txt requirements/tests.txt test.py
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 43be683..0000000
--- a/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-init:
- pip install -r requirements.txt
-
-test:
- py.test test.py -s
diff --git a/README.md b/README.md
index b68a0b4..34cf60e 100644
--- a/README.md
+++ b/README.md
@@ -1,34 +1,32 @@
-[](https://travis-ci.org/marteinn/wpparser)
-[](http://badge.fury.io/py/wpparser)
-
# wpparser
-This library parses Wordpress xml based exports into a simple python dictionary.
-
+This library parses WordPress XML exports into a Python dictionary.
-## How it works
+## Installation
-The library uses ElementTree to traverse through the export file.
+```bash
+pip install wpparser
+```
## Usage
- import wpparser
-
- data = wpparser.parse("./blog.wordpress.2014-09-26.xml")
- >>> {"blog": {"tagline": "Tagline",...
+```python
+import wpparser
+data = wpparser.parse("./blog.wordpress.2014-09-26.xml")
+```
## What it returns
-It returns a well formatted dict, containing the following datatypes:
+A dictionary containing:
-- Blog: The general blog information, such as tagline, site url.
-- Authors: A list with the different authors.
-- Categories: The categories in use, organized as a nested array.
-- Tags: A list with the different tags.
-- Posts: An array that contains all posts, the post object might also contain the different comments belonging to the post.
+- **blog**: General blog information (title, tagline, site url, etc.)
+- **authors**: List of authors
+- **categories**: Categories organized as a nested tree
+- **tags**: List of tags
+- **posts**: List of posts, including comments and post metadata
-### Example:
+### Example
```python
{
@@ -37,85 +35,83 @@ It returns a well formatted dict, containing the following datatypes:
"site_url": "http://marteinn.se/blog",
"blog_url": "http://marteinn.se/blog",
"language": "en-US",
- "title": "Marteinn / Blog"
+ "title": "Marteinn / Blog",
},
- "authors: [{
- "login": "admin",
- "last_name": None,
- "display_name": "admin",
- "email": "martin@marteinn.se",
- "first_name": None}
+ "authors": [
+ {
+ "login": "admin",
+ "last_name": None,
+ "display_name": "admin",
+ "email": "martin@marteinn.se",
+ "first_name": None,
+ }
+ ],
+ "categories": [
+ {
+ "parent": None,
+ "term_id": "3",
+ "name": "Action Script",
+ "nicename": "action-script",
+ "children": [
+ {
+ "parent": "action-script",
+ "term_id": "20",
+ "name": "Flash related",
+ "nicename": "flash-related",
+ "children": [],
+ }
+ ],
+ }
+ ],
+ "tags": [{"term_id": "1", "slug": "bash", "name": "Bash"}],
+ "posts": [
+ {
+ "creator": "admin",
+ "excerpt": None,
+ "post_date_gmt": "2014-09-22 20:10:40",
+ "post_date": "2014-09-22 21:10:40",
+ "post_type": "post",
+ "menu_order": "0",
+ "guid": "http://marteinn.se/blog/?p=828",
+ "title": "Post Title",
+ "comments": [
+ {
+ "date_gmt": "2014-09-24 23:08:31",
+ "parent": "0",
+ "date": "2014-09-25 00:08:31",
+ "id": "85929",
+ "user_id": "0",
+ "author": "Author",
+ "author_email": None,
+ "author_ip": "111.111.111.111",
+ "approved": "1",
+ "content": "Comment title",
+ "author_url": "http://example.com",
+ "type": "pingback",
+ }
+ ],
+ "content": "Text",
+ "post_parent": "0",
+ "post_password": None,
+ "status": "publish",
+ "description": None,
+ "tags": ["tag"],
+ "ping_status": "open",
+ "post_id": "1",
+ "link": "http://www.marteinn.se/blog/slug/",
+ "pub_date": "Mon, 22 Sep 2014 20:10:40 +0000",
+ "categories": ["category"],
+ "is_sticky": "0",
+ "post_name": "slug",
+ }
],
- "categories": [{
- "parent": None,
- "term_id": "3",
- "name": "Action Script",
- "nicename": "action-script",
- "children": [{
- "parent": "action-script",
- "term_id": "20",
- "name": "Flash related",
- "nicename": "flash-related",
- "children": []
- }]
- }],
- "tags": [{
- "term_id": "1",
- "slug": "bash",
- "name": "Bash"
- }],
- "posts": [{
- "creator": "admin",
- "excerpt": None,
- "post_date_gmt": "2014-09-22 20:10:40",
- "post_date": "2014-09-22 21:10:40",
- "post_type": "post",
- "menu_order": "0",
- "guid": "http://marteinn.se/blog/?p=828",
- "title": "Post Title",
- "comments": [{
- "date_gmt": "2014-09-24 23:08:31",
- "parent": "0",
- "date": "2014-09-25 00:08:31",
- "id": "85929",
- "user_id": "0",
- "author": u"Author",
- "author_email": None,
- "author_ip": "111.111.111.111",
- "approved": "1",
- "content": u"Comment title",
- "author_url": "http://example.com",
- "type": "pingback"
- }],
- "content": "Text",
- "post_parent": "0",
- "post_password": None,
- "status": "publish",
- "description": None,
- "tags": ["tag"],
- "ping_status": "open",
- "post_id": "1",
- "link": "http://www.marteinn.se/blog/slug/",
- "pub_date": "Mon, 22 Sep 2014 20:10:40 +0000",
- "categories": ["category"],
- "is_sticky": "0",
- "post_name": "slug"
- }]
}
```
-## Installation
-
-wpparser can easily be installed through pip.
-
- $ pip install wpparser
-
-
## Contributing
Want to contribute? Awesome. Just send a pull request.
-
## License
wpparser is released under the [MIT License](http://www.opensource.org/licenses/MIT).
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..5f417a2
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,71 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wpparser"
+dynamic = ["version"]
+description = "Parse wordpress export files into a well formatted python dictionary"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+ { name = "Martin Sandström", email = "martin@marteinn.se" },
+]
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "Natural Language :: English",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3.14",
+]
+dependencies = [
+ "phpserialize>=1.3",
+]
+
+[project.urls]
+Homepage = "https://github.com/marteinn/wpparser"
+
+[tool.setuptools.dynamic]
+version = { attr = "wpparser.__version__" }
+
+[tool.ruff]
+target-version = "py314"
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+ "E", # pycodestyle errors
+ "W", # pycodestyle warnings
+ "F", # pyflakes
+ "I", # isort
+ "UP", # pyupgrade
+ "B", # flake8-bugbear
+ "SIM", # flake8-simplify
+ "TCH", # flake8-type-checking
+ "RUF", # ruff-specific rules
+]
+
+[tool.ruff.lint.isort]
+known-third-party = ["phpserialize"]
+
+[tool.mypy]
+python_version = "3.14"
+strict = true
+warn_unused_configs = true
+
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_unreachable = true
+
+show_error_codes = true
+pretty = true
+
+[[tool.mypy.overrides]]
+module = ["phpserialize"]
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = ["tests.*"]
+disallow_untyped_defs = false
\ No newline at end of file
diff --git a/requirements/dev.txt b/requirements/dev.txt
deleted file mode 100644
index 102ac7e..0000000
--- a/requirements/dev.txt
+++ /dev/null
@@ -1,2 +0,0 @@
--r tests.txt
-pypandoc==0.9.9
diff --git a/requirements/install.txt b/requirements/install.txt
deleted file mode 100644
index f8449aa..0000000
--- a/requirements/install.txt
+++ /dev/null
@@ -1 +0,0 @@
-phpserialize==1.3
diff --git a/requirements/tests.txt b/requirements/tests.txt
deleted file mode 100644
index 539e4ae..0000000
--- a/requirements/tests.txt
+++ /dev/null
@@ -1,2 +0,0 @@
--r install.txt
-pytest>=3
diff --git a/runtests.py b/runtests.py
deleted file mode 100644
index cc43b4e..0000000
--- a/runtests.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Tests"""
-
-import unittest
-from wpparser import parse
-
-
-class ParseTestCase(unittest.TestCase):
- def test_parse(self):
- result = parse("./blog.wordpress.2014-09-26.xml")
-
- assert len(result["posts"]) is 3
- assert result["blog"]["title"] == "Blog"
- assert len(result["categories"]) is 1
- assert len(result["tags"]) is 1
-
- def test_attachment_metadata(self):
- result = parse("./blog.wordpress.2014-09-26.xml")
-
- post = result["posts"][2]
-
- assert "postmeta" in post
- assert "attached_file" in post["postmeta"]
- assert "attachment_metadata" in post["postmeta"]
-
- attached_file = post["postmeta"]["attached_file"]
- assert attached_file == "logo-promo.png"
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 7205f32..0000000
--- a/setup.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import os
-import re
-import sys
-from setuptools import find_packages, setup
-
-import wpparser
-
-if sys.argv[-1] == "publish":
- os.system("python setup.py sdist upload")
- sys.exit()
-
-
-test_requirements = [
- "pytest>=3",
-]
-
-requires = [
- "phpserialize>=1.3",
-]
-
-version = ""
-with open("wpparser/__init__.py", "r") as fd:
- version = re.search(
- r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE
- ).group(1)
-
-# Convert markdown to rst
-try:
- from pypandoc import convert
- long_description = convert("README.md", "rst")
-except:
- long_description = ""
-
-setup(
- name="wpparser",
- version=version,
- description="Parse wordpress export files into a well formatted python dictionary", # NOQA
- long_description=long_description,
- author="Martin Sandström",
- author_email="martin@marteinn.se",
- url="https://github.com/marteinn/wpparser",
- packages=find_packages(),
- package_data={"": ["LICENSE", ], "wpparser": ["*.txt"]},
- package_dir={"wpparser": "wpparser"},
- include_package_data=True,
- install_requires=requires,
- tests_require=test_requirements,
- license="MIT",
- zip_safe=False,
- classifiers=[
- "Development Status :: 5 - Production/Stable",
- "Intended Audience :: Developers",
- "Natural Language :: English",
- "License :: OSI Approved :: MIT License",
- "Programming Language :: Python",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- ],
-)
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..87191b1
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,10 @@
+from pathlib import Path
+
+import pytest
+
+FIXTURES_DIR = Path(__file__).parent / "fixtures"
+
+
+@pytest.fixture
+def sample_export():
+ return str(FIXTURES_DIR / "blog.wordpress.2014-09-26.xml")
diff --git a/blog.wordpress.2014-09-26.xml b/tests/fixtures/blog.wordpress.2014-09-26.xml
similarity index 100%
rename from blog.wordpress.2014-09-26.xml
rename to tests/fixtures/blog.wordpress.2014-09-26.xml
diff --git a/tests/test_parser.py b/tests/test_parser.py
new file mode 100644
index 0000000..b711bb8
--- /dev/null
+++ b/tests/test_parser.py
@@ -0,0 +1,146 @@
+import pytest
+
+from wpparser import parse
+
+
+def test_parse(sample_export):
+ result = parse(sample_export)
+
+ assert len(result["posts"]) == 3
+ assert result["blog"]["title"] == "Blog"
+ assert len(result["categories"]) == 1
+ assert len(result["tags"]) == 1
+
+
+def test_attachment_metadata(sample_export):
+ result = parse(sample_export)
+
+ post = result["posts"][2]
+
+ assert "postmeta" in post
+ assert "attached_file" in post["postmeta"]
+ assert "attachment_metadata" in post["postmeta"]
+
+ attached_file = post["postmeta"]["attached_file"]
+ assert attached_file == "logo-promo.png"
+
+
+def test_blog_metadata(sample_export):
+ result = parse(sample_export)
+ blog = result["blog"]
+
+ assert blog["title"] == "Blog"
+ assert blog["tagline"] == "Just another WordPress site"
+ assert blog["language"] == "en-US"
+ assert blog["site_url"] == "http://marteinn.se/blog"
+ assert blog["blog_url"] == "http://marteinn.se/blog"
+
+
+def test_authors(sample_export):
+ result = parse(sample_export)
+ authors = result["authors"]
+
+ assert len(authors) == 1
+
+ author = authors[0]
+ assert author["login"] == "admin"
+ assert author["email"] == "martin@marteinn.se"
+ assert author["display_name"] == "admin"
+ assert author["first_name"] is None
+ assert author["last_name"] is None
+
+
+def test_category_structure(sample_export):
+ result = parse(sample_export)
+ categories = result["categories"]
+
+ assert len(categories) == 1
+ cat = categories[0]
+ assert cat["term_id"] == "1"
+ assert cat["nicename"] == "uncategorized"
+ assert cat["name"] == "Uncategorized"
+ assert cat["parent"] is None
+ assert cat["children"] == []
+
+
+def test_tags(sample_export):
+ result = parse(sample_export)
+ tags = result["tags"]
+
+ assert len(tags) == 1
+ tag = tags[0]
+ assert tag["term_id"] == "1"
+ assert tag["slug"] == "bash"
+ assert tag["name"] == "Bash"
+
+
+def test_post_fields(sample_export):
+ result = parse(sample_export)
+ post = result["posts"][0]
+
+ assert post["title"] == "Hello world!"
+ assert post["content"] == "Welcome to WordPress. This is your first post. Edit or delete it, then start blogging!"
+ assert post["post_date"] == "2014-09-26 18:47:05"
+ assert post["post_date_gmt"] == "2014-09-26 18:47:06"
+ assert post["status"] == "publish"
+ assert post["post_type"] == "post"
+ assert post["post_id"] == "1"
+ assert post["post_name"] == "hello-world"
+ assert post["creator"] == "admin"
+ assert post["ping_status"] == "open"
+ assert post["menu_order"] == "0"
+ assert post["is_sticky"] == "0"
+
+
+def test_post_modified_fields(sample_export):
+ result = parse(sample_export)
+ post = result["posts"][0]
+
+ assert "post_modified" in post
+ assert "post_modified_gmt" in post
+
+
+def test_page_post_type(sample_export):
+ result = parse(sample_export)
+ page = result["posts"][1]
+
+ assert page["title"] == "Sample Page"
+ assert page["post_type"] == "page"
+
+
+def test_comments(sample_export):
+ result = parse(sample_export)
+ post = result["posts"][0]
+ comments = post["comments"]
+
+ assert len(comments) == 1
+ comment = comments[0]
+ assert comment["id"] == "1"
+ assert comment["author"] == "Mr WordPress"
+ assert comment["author_url"] == "https://wordpress.org/"
+ assert comment["date"] == "2014-09-26 18:47:05"
+ assert comment["date_gmt"] == "2014-09-26 18:47:06"
+ assert comment["approved"] == "1"
+ assert comment["parent"] == "0"
+ assert comment["user_id"] == "0"
+ assert "Hi, this is a comment." in comment["content"]
+
+
+def test_missing_channel_raises_valueerror(tmp_path):
+ xml_content = ''
+ xml_file = tmp_path / "invalid.xml"
+ xml_file.write_text(xml_content)
+
+ with pytest.raises(ValueError, match="missing element"):
+ parse(str(xml_file))
+
+
+def test_post_categories_domain(sample_export):
+ result = parse(sample_export)
+ post = result["posts"][0]
+
+ assert "category_category" in post
+ cats = post["category_category"]
+ assert len(cats) == 1
+ assert cats[0]["nicename"] == "uncategorized"
+ assert cats[0]["text"] == "Uncategorized"
diff --git a/tox.ini b/tox.ini
deleted file mode 100644
index 04cb9dc..0000000
--- a/tox.ini
+++ /dev/null
@@ -1,8 +0,0 @@
-envlist=py36,py37,py38
-
-[testenv]
-extras = test
-deps =
- pytest
-commands =
- pytest -s runtests.py
diff --git a/wpparser/__init__.py b/wpparser/__init__.py
index 2f03527..f4a1ef4 100644
--- a/wpparser/__init__.py
+++ b/wpparser/__init__.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
"""
wpparser
---
@@ -18,5 +16,4 @@
__license__ = "MIT"
__copyright__ = "Copyright 2014-2020 Martin Sandström"
-
-from .parser import parse # NOQA
+from .parser import parse as parse
diff --git a/wpparser/parser.py b/wpparser/parser.py
index b800037..58992ca 100644
--- a/wpparser/parser.py
+++ b/wpparser/parser.py
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
"""
wpparser
---
@@ -7,16 +5,17 @@
Load and parse the wp export file into a readable dictionary.
"""
-try:
- from StringIO import StringIO
-except ImportError:
- from io import BytesIO as StringIO
+from __future__ import annotations
+
+import logging
+import xml.etree.ElementTree as ET
+from io import BytesIO
+from typing import TYPE_CHECKING, Any
-try:
- import xml.etree.cElementTree as ET
-except ImportError:
- import xml.etree.ElementTree as ET
+if TYPE_CHECKING:
+ from pathlib import Path
+import phpserialize
# Namespaces used by ElementTree with parsing wp xml.
EXCERPT_NAMESPACE = "http://wordpress.org/export/1.2/excerpt/"
@@ -26,7 +25,7 @@
WP_NAMESPACE = "http://wordpress.org/export/1.2/"
-def parse(path):
+def parse(path: str | Path) -> dict[str, Any]:
"""
Parses xml and returns a formatted dict.
@@ -100,6 +99,8 @@ def parse(path):
"pub_date": "Mon, 22 Sep 2014 20:10:40 +0000",
"categories": ["category"],
"is_sticky": "0",
+ "post_modified": "2014-09-22 21:10:40",
+ "post_modified_gmt": "2014-09-22 20:10:40",
"post_name": "slug"
}]
}
@@ -108,6 +109,8 @@ def parse(path):
doc = ET.parse(path).getroot()
channel = doc.find("./channel")
+ if channel is None:
+ raise ValueError("Invalid WordPress export: missing element")
blog = _parse_blog(channel)
authors = _parse_authors(channel)
@@ -124,16 +127,25 @@ def parse(path):
}
-def _parse_blog(element):
+def _find_text(element: ET.Element, path: str) -> str | None:
+ el = element.find(path)
+ return el.text if el is not None else None
+
+
+def _get_wp_element(element: ET.Element, name: str) -> str | None:
+ return _find_text(element, f"./{{{WP_NAMESPACE}}}{name}")
+
+
+def _parse_blog(element: ET.Element) -> dict[str, str | None]:
"""
- Parse and return genral blog data (title, tagline etc).
+ Parse and return general blog data (title, tagline etc).
"""
- title = element.find("./title").text
- tagline = element.find("./description").text
- language = element.find("./language").text
- site_url = element.find("./{%s}base_site_url" % WP_NAMESPACE).text
- blog_url = element.find("./{%s}base_blog_url" % WP_NAMESPACE).text
+ title = _find_text(element, "./title")
+ tagline = _find_text(element, "./description")
+ language = _find_text(element, "./language")
+ site_url = _get_wp_element(element, "base_site_url")
+ blog_url = _get_wp_element(element, "base_blog_url")
return {
"title": title,
@@ -144,59 +156,58 @@ def _parse_blog(element):
}
-def _parse_authors(element):
+def _parse_authors(element: ET.Element) -> list[dict[str, str | None]]:
"""
Returns a well formatted list of users that can be matched against posts.
"""
authors = []
- items = element.findall("./{%s}author" % WP_NAMESPACE)
- for item in items:
- login = item.find("./{%s}author_login" % WP_NAMESPACE).text
- email = item.find("./{%s}author_email" % WP_NAMESPACE).text
- first_name = item.find("./{%s}author_first_name" % WP_NAMESPACE).text
- last_name = item.find("./{%s}author_last_name" % WP_NAMESPACE).text
- display_name = item.find(
- "./{%s}author_display_name" % WP_NAMESPACE).text
-
- authors.append({
- "login": login,
- "email": email,
- "display_name": display_name,
- "first_name": first_name,
- "last_name": last_name
- })
+ for item in element.findall(f"./{{{WP_NAMESPACE}}}author"):
+ login = _get_wp_element(item, "author_login")
+ email = _get_wp_element(item, "author_email")
+ first_name = _get_wp_element(item, "author_first_name")
+ last_name = _get_wp_element(item, "author_last_name")
+ display_name = _get_wp_element(item, "author_display_name")
+
+ authors.append(
+ {
+ "login": login,
+ "email": email,
+ "display_name": display_name,
+ "first_name": first_name,
+ "last_name": last_name,
+ }
+ )
return authors
-def _parse_categories(element):
+def _parse_categories(element: ET.Element) -> list[dict[str, Any]]:
"""
Returns a list with categories with relations.
"""
- reference = {}
- items = element.findall("./{%s}category" % WP_NAMESPACE)
+ reference: dict[str, dict[str, Any]] = {}
- for item in items:
- term_id = item.find("./{%s}term_id" % WP_NAMESPACE).text
- nicename = item.find("./{%s}category_nicename" % WP_NAMESPACE).text
- name = item.find("./{%s}cat_name" % WP_NAMESPACE).text
- parent = item.find("./{%s}category_parent" % WP_NAMESPACE).text
+ for item in element.findall(f"./{{{WP_NAMESPACE}}}category"):
+ term_id = _get_wp_element(item, "term_id")
+ nicename = _get_wp_element(item, "category_nicename")
+ name = _get_wp_element(item, "cat_name")
+ parent = _get_wp_element(item, "category_parent")
- category = {
- "term_id": term_id,
- "nicename": nicename,
- "name": name,
- "parent": parent
- }
+ category: dict[str, Any] = {"term_id": term_id, "nicename": nicename, "name": name, "parent": parent}
- reference[nicename] = category
+ if nicename is not None:
+ reference[nicename] = category
return _build_category_tree(None, reference=reference)
-def _build_category_tree(slug, reference=None, items=None):
+def _build_category_tree(
+ slug: str | None,
+ reference: dict[str, dict[str, Any]],
+ items: list[dict[str, Any]] | None = None,
+) -> list[dict[str, Any]]:
"""
Builds a recursive tree with category relations as children.
"""
@@ -208,17 +219,15 @@ def _build_category_tree(slug, reference=None, items=None):
category = reference[key]
if category["parent"] == slug:
- children = _build_category_tree(category["nicename"],
- reference=reference)
- category["children"] = children
+ category["children"] = _build_category_tree(category["nicename"], reference=reference)
items.append(category)
return items
-def _parse_tags(element):
+def _parse_tags(element: ET.Element) -> list[dict[str, str | None]]:
"""
- Retrieves and parses tags into a array/dict.
+ Retrieves and parses tags into an array/dict.
Example:
@@ -227,12 +236,11 @@ def _parse_tags(element):
"""
tags = []
- items = element.findall("./{%s}tag" % WP_NAMESPACE)
- for item in items:
- term_id = item.find("./{%s}term_id" % WP_NAMESPACE).text
- slug = item.find("./{%s}tag_slug" % WP_NAMESPACE).text
- name = item.find("./{%s}tag_name" % WP_NAMESPACE).text
+ for item in element.findall(f"./{{{WP_NAMESPACE}}}tag"):
+ term_id = _get_wp_element(item, "term_id")
+ slug = _get_wp_element(item, "tag_slug")
+ name = _get_wp_element(item, "tag_name")
tag = {
"term_id": term_id,
@@ -245,7 +253,7 @@ def _parse_tags(element):
return tags
-def _parse_posts(element):
+def _parse_posts(element: ET.Element) -> list[dict[str, Any]]:
"""
Returns a list with posts.
"""
@@ -254,39 +262,39 @@ def _parse_posts(element):
items = element.findall("item")
for item in items:
- title = item.find("./title").text
- link = item.find("./link").text
- pub_date = item.find("./pubDate").text
- creator = item.find("./{%s}creator" % DC_NAMESPACE).text
- guid = item.find("./guid").text
- description = item.find("./description").text
- content = item.find("./{%s}encoded" % CONTENT_NAMESPACE).text
- excerpt = item.find("./{%s}encoded" % EXCERPT_NAMESPACE).text
- post_id = item.find("./{%s}post_id" % WP_NAMESPACE).text
- post_date = item.find("./{%s}post_date" % WP_NAMESPACE).text
- post_date_gmt = item.find("./{%s}post_date_gmt" % WP_NAMESPACE).text
- status = item.find("./{%s}status" % WP_NAMESPACE).text
- post_parent = item.find("./{%s}post_parent" % WP_NAMESPACE).text
- menu_order = item.find("./{%s}menu_order" % WP_NAMESPACE).text
- post_type = item.find("./{%s}post_type" % WP_NAMESPACE).text
- post_name = item.find("./{%s}post_name" % WP_NAMESPACE).text
- is_sticky = item.find("./{%s}is_sticky" % WP_NAMESPACE).text
- ping_status = item.find("./{%s}ping_status" % WP_NAMESPACE).text
- post_password = item.find("./{%s}post_password" % WP_NAMESPACE).text
+ title = _find_text(item, "./title")
+ link = _find_text(item, "./link")
+ pub_date = _find_text(item, "./pubDate")
+ creator = _find_text(item, f"./{{{DC_NAMESPACE}}}creator")
+ guid = _find_text(item, "./guid")
+ description = _find_text(item, "./description")
+ content = _find_text(item, f"./{{{CONTENT_NAMESPACE}}}encoded")
+ excerpt = _find_text(item, f"./{{{EXCERPT_NAMESPACE}}}encoded")
+ post_id = _get_wp_element(item, "post_id")
+ post_date = _get_wp_element(item, "post_date")
+ post_date_gmt = _get_wp_element(item, "post_date_gmt")
+ post_modified = _get_wp_element(item, "post_modified")
+ post_modified_gmt = _get_wp_element(item, "post_modified_gmt")
+ status = _get_wp_element(item, "status")
+ post_parent = _get_wp_element(item, "post_parent")
+ menu_order = _get_wp_element(item, "menu_order")
+ post_type = _get_wp_element(item, "post_type")
+ post_name = _get_wp_element(item, "post_name")
+ is_sticky = _get_wp_element(item, "is_sticky")
+ ping_status = _get_wp_element(item, "ping_status")
+ post_password = _get_wp_element(item, "post_password")
category_items = item.findall("./category")
- categories = []
- tags = []
+ category_domains: dict[str, list[dict[str, str | None]]] = {}
for category_item in category_items:
- if category_item.attrib["domain"] == "category":
- item_list = categories
- else:
- item_list = tags
+ if category_item.attrib["domain"] not in category_domains:
+ category_domains[category_item.attrib["domain"]] = []
+ category_domains[category_item.attrib["domain"]].append(
+ {"nicename": category_item.attrib["nicename"], "text": category_item.text}
+ )
- item_list.append(category_item.attrib["nicename"])
-
- post = {
+ post: dict[str, Any] = {
"title": title,
"link": link,
"pub_date": pub_date,
@@ -298,18 +306,22 @@ def _parse_posts(element):
"post_id": post_id,
"post_date": post_date,
"post_date_gmt": post_date_gmt,
+ "post_modified": post_modified,
+ "post_modified_gmt": post_modified_gmt,
"status": status,
"post_parent": post_parent,
"menu_order": menu_order,
"post_type": post_type,
"post_name": post_name,
- "categories": categories,
"is_sticky": is_sticky,
"ping_status": ping_status,
"post_password": post_password,
- "tags": tags,
}
+ # Include all categories with a prefix inorder to avoid collisions
+ for k, v in category_domains.items():
+ post[f"category_{k}"] = v
+
post["postmeta"] = _parse_postmeta(item)
post["comments"] = _parse_comments(item)
posts.append(post)
@@ -317,57 +329,58 @@ def _parse_posts(element):
return posts
-def _parse_postmeta(element):
- import phpserialize
-
+def _parse_postmeta(element: ET.Element) -> dict[str, Any]:
"""
- Retrive post metadata as a dictionary
+ Retrieve post metadata as a dictionary
"""
- metadata = {}
- fields = element.findall("./{%s}postmeta" % WP_NAMESPACE)
+ metadata: dict[str, Any] = {}
+ fields = element.findall(f"./{{{WP_NAMESPACE}}}postmeta")
for field in fields:
- key = field.find("./{%s}meta_key" % WP_NAMESPACE).text
- value = field.find("./{%s}meta_value" % WP_NAMESPACE).text
+ key = _get_wp_element(field, "meta_key")
+ value = _get_wp_element(field, "meta_value")
+
+ if key is None:
+ continue
- if key == "_wp_attachment_metadata":
- stream = StringIO(value.encode())
+ if key == "_wp_attachment_metadata" and value is not None:
+ stream = BytesIO(value.encode())
try:
data = phpserialize.load(stream)
metadata["attachment_metadata"] = data
except ValueError as e:
- pass
- except Exception as e:
- raise(e)
+ logging.warning(e)
- if key == "_wp_attached_file":
+ elif key == "_wp_attached_file":
metadata["attached_file"] = value
+ else:
+ metadata[key] = value
return metadata
-def _parse_comments(element):
+def _parse_comments(element: ET.Element) -> list[dict[str, str | None]]:
"""
Returns a list with comments.
"""
comments = []
- items = element.findall("./{%s}comment" % WP_NAMESPACE)
+ items = element.findall(f"./{{{WP_NAMESPACE}}}comment")
for item in items:
- comment_id = item.find("./{%s}comment_id" % WP_NAMESPACE).text
- author = item.find("./{%s}comment_author" % WP_NAMESPACE).text
- email = item.find("./{%s}comment_author_email" % WP_NAMESPACE).text
- author_url = item.find("./{%s}comment_author_url" % WP_NAMESPACE).text
- author_ip = item.find("./{%s}comment_author_IP" % WP_NAMESPACE).text
- date = item.find("./{%s}comment_date" % WP_NAMESPACE).text
- date_gmt = item.find("./{%s}comment_date_gmt" % WP_NAMESPACE).text
- content = item.find("./{%s}comment_content" % WP_NAMESPACE).text
- approved = item.find("./{%s}comment_approved" % WP_NAMESPACE).text
- comment_type = item.find("./{%s}comment_type" % WP_NAMESPACE).text
- parent = item.find("./{%s}comment_parent" % WP_NAMESPACE).text
- user_id = item.find("./{%s}comment_user_id" % WP_NAMESPACE).text
+ comment_id = _get_wp_element(item, "comment_id")
+ author = _get_wp_element(item, "comment_author")
+ email = _get_wp_element(item, "comment_author_email")
+ author_url = _get_wp_element(item, "comment_author_url")
+ author_ip = _get_wp_element(item, "comment_author_IP")
+ date = _get_wp_element(item, "comment_date")
+ date_gmt = _get_wp_element(item, "comment_date_gmt")
+ content = _get_wp_element(item, "comment_content")
+ approved = _get_wp_element(item, "comment_approved")
+ comment_type = _get_wp_element(item, "comment_type")
+ parent = _get_wp_element(item, "comment_parent")
+ user_id = _get_wp_element(item, "comment_user_id")
comment = {
"id": comment_id,