From 4f90ace8ba5670bd13e5788d97ba51b13c4e2a0a Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Thu, 9 Sep 2021 17:56:08 -0700 Subject: [PATCH 01/15] Include all postmeta data under post['postmeta'] --- wpparser/parser.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/wpparser/parser.py b/wpparser/parser.py index b800037..361de9f 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -17,6 +17,9 @@ except ImportError: import xml.etree.ElementTree as ET +import phpserialize + + # Namespaces used by ElementTree with parsing wp xml. EXCERPT_NAMESPACE = "http://wordpress.org/export/1.2/excerpt/" @@ -318,8 +321,6 @@ def _parse_posts(element): def _parse_postmeta(element): - import phpserialize - """ Retrive post metadata as a dictionary """ @@ -339,10 +340,12 @@ def _parse_postmeta(element): except ValueError as e: pass except Exception as e: - raise(e) + raise e - if key == "_wp_attached_file": + elif key == "_wp_attached_file": metadata["attached_file"] = value + else: + metadata[key] = value return metadata From cc687e203917c8660320c577faed61dc8107c713 Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Wed, 13 Oct 2021 16:51:36 -0700 Subject: [PATCH 02/15] Include all categories in the export not just tags and category --- wpparser/parser.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/wpparser/parser.py b/wpparser/parser.py index 361de9f..d9ae218 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -278,16 +278,12 @@ def _parse_posts(element): post_password = item.find("./{%s}post_password" % WP_NAMESPACE).text category_items = item.findall("./category") - categories = [] - tags = [] + category_domains = {} for category_item in category_items: - if category_item.attrib["domain"] == "category": - item_list = categories - else: - item_list = tags - - item_list.append(category_item.attrib["nicename"]) + if category_item.attrib["domain"] not in category_domains: + category_domains[category_item.attrib["domain"]] = [] + category_domains[category_item.attrib["domain"]].append({'nicename': category_item.attrib["nicename"], 'text': category_item.text}) post = { "title": title, @@ -306,13 +302,15 @@ def _parse_posts(element): "menu_order": menu_order, "post_type": post_type, "post_name": post_name, - "categories": categories, "is_sticky": is_sticky, "ping_status": ping_status, "post_password": post_password, - "tags": tags, } + # Include all categories with a prefix inorder to avoid collisions + for k, v in category_domains.items(): + post[f'category_{k}'] = v + post["postmeta"] = _parse_postmeta(item) post["comments"] = _parse_comments(item) posts.append(post) From ed1acbdcf53fafe07b4049811ca3c957f21bc88b Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Wed, 13 Oct 2021 16:52:01 -0700 Subject: [PATCH 03/15] whitespace --- wpparser/parser.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wpparser/parser.py b/wpparser/parser.py index d9ae218..ad797c1 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -19,8 +19,6 @@ import phpserialize - - # Namespaces used by ElementTree with parsing wp xml. EXCERPT_NAMESPACE = "http://wordpress.org/export/1.2/excerpt/" CONTENT_NAMESPACE = "http://purl.org/rss/1.0/modules/content/" @@ -161,7 +159,7 @@ def _parse_authors(element): first_name = item.find("./{%s}author_first_name" % WP_NAMESPACE).text last_name = item.find("./{%s}author_last_name" % WP_NAMESPACE).text display_name = item.find( - "./{%s}author_display_name" % WP_NAMESPACE).text + "./{%s}author_display_name" % WP_NAMESPACE).text authors.append({ "login": login, From 5eb7eea1b9c0f935835d966bf609c9832a26583a Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Wed, 13 Oct 2021 16:52:59 -0700 Subject: [PATCH 04/15] Remove Py2 compatibility. --- wpparser/parser.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/wpparser/parser.py b/wpparser/parser.py index ad797c1..1bff70a 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ wpparser --- @@ -7,15 +5,8 @@ Load and parse the wp export file into a readable dictionary. """ -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO - -try: - import xml.etree.cElementTree as ET -except ImportError: - import xml.etree.ElementTree as ET +import xml.etree.ElementTree as ET +from io import BytesIO as StringIO import phpserialize From fdf5268e4f5b9cc4309ec8d857469619113db160 Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Wed, 13 Oct 2021 16:55:23 -0700 Subject: [PATCH 05/15] Convert to f-strings --- wpparser/parser.py | 97 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/wpparser/parser.py b/wpparser/parser.py index 1bff70a..3387651 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -124,8 +124,8 @@ def _parse_blog(element): title = element.find("./title").text tagline = element.find("./description").text language = element.find("./language").text - site_url = element.find("./{%s}base_site_url" % WP_NAMESPACE).text - blog_url = element.find("./{%s}base_blog_url" % WP_NAMESPACE).text + site_url = element.find(f"./{{{WP_NAMESPACE}}}base_site_url").text + blog_url = element.find(f"./{{{WP_NAMESPACE}}}base_blog_url").text return { "title": title, @@ -142,15 +142,14 @@ def _parse_authors(element): """ authors = [] - items = element.findall("./{%s}author" % WP_NAMESPACE) + items = element.findall(f"./{{{WP_NAMESPACE}}}author") for item in items: - login = item.find("./{%s}author_login" % WP_NAMESPACE).text - email = item.find("./{%s}author_email" % WP_NAMESPACE).text - first_name = item.find("./{%s}author_first_name" % WP_NAMESPACE).text - last_name = item.find("./{%s}author_last_name" % WP_NAMESPACE).text - display_name = item.find( - "./{%s}author_display_name" % WP_NAMESPACE).text + login = item.find(f"./{{{WP_NAMESPACE}}}author_login").text + email = item.find(f"./{{{WP_NAMESPACE}}}author_email").text + first_name = item.find(f"./{{{WP_NAMESPACE}}}author_first_name").text + last_name = item.find(f"./{{{WP_NAMESPACE}}}author_last_name").text + display_name = item.find(f"./{{{WP_NAMESPACE}}}author_display_name").text authors.append({ "login": login, @@ -168,13 +167,13 @@ def _parse_categories(element): Returns a list with categories with relations. """ reference = {} - items = element.findall("./{%s}category" % WP_NAMESPACE) + items = element.findall(f"./{{{WP_NAMESPACE}}}category") for item in items: - term_id = item.find("./{%s}term_id" % WP_NAMESPACE).text - nicename = item.find("./{%s}category_nicename" % WP_NAMESPACE).text - name = item.find("./{%s}cat_name" % WP_NAMESPACE).text - parent = item.find("./{%s}category_parent" % WP_NAMESPACE).text + term_id = item.find(f"./{{{WP_NAMESPACE}}}term_id").text + nicename = item.find(f"./{{{WP_NAMESPACE}}}category_nicename").text + name = item.find(f"./{{{WP_NAMESPACE}}}cat_name").text + parent = item.find(f"./{{{WP_NAMESPACE}}}category_parent").text category = { "term_id": term_id, @@ -219,12 +218,12 @@ def _parse_tags(element): """ tags = [] - items = element.findall("./{%s}tag" % WP_NAMESPACE) + items = element.findall(f"./{{{WP_NAMESPACE}}}tag") for item in items: - term_id = item.find("./{%s}term_id" % WP_NAMESPACE).text - slug = item.find("./{%s}tag_slug" % WP_NAMESPACE).text - name = item.find("./{%s}tag_name" % WP_NAMESPACE).text + term_id = item.find(f"./{{{WP_NAMESPACE}}}term_id").text + slug = item.find(f"./{{{WP_NAMESPACE}}}tag_slug").text + name = item.find(f"./{{{WP_NAMESPACE}}}tag_name").text tag = { "term_id": term_id, @@ -249,22 +248,22 @@ def _parse_posts(element): title = item.find("./title").text link = item.find("./link").text pub_date = item.find("./pubDate").text - creator = item.find("./{%s}creator" % DC_NAMESPACE).text + creator = item.find(f"./{{{DC_NAMESPACE}}}creator").text guid = item.find("./guid").text description = item.find("./description").text - content = item.find("./{%s}encoded" % CONTENT_NAMESPACE).text - excerpt = item.find("./{%s}encoded" % EXCERPT_NAMESPACE).text - post_id = item.find("./{%s}post_id" % WP_NAMESPACE).text - post_date = item.find("./{%s}post_date" % WP_NAMESPACE).text - post_date_gmt = item.find("./{%s}post_date_gmt" % WP_NAMESPACE).text - status = item.find("./{%s}status" % WP_NAMESPACE).text - post_parent = item.find("./{%s}post_parent" % WP_NAMESPACE).text - menu_order = item.find("./{%s}menu_order" % WP_NAMESPACE).text - post_type = item.find("./{%s}post_type" % WP_NAMESPACE).text - post_name = item.find("./{%s}post_name" % WP_NAMESPACE).text - is_sticky = item.find("./{%s}is_sticky" % WP_NAMESPACE).text - ping_status = item.find("./{%s}ping_status" % WP_NAMESPACE).text - post_password = item.find("./{%s}post_password" % WP_NAMESPACE).text + content = item.find(f"./{{{CONTENT_NAMESPACE}}}encoded").text + excerpt = item.find(f"./{{{EXCERPT_NAMESPACE}}}encoded").text + post_id = item.find(f"./{{{WP_NAMESPACE}}}post_id").text + post_date = item.find(f"./{{{WP_NAMESPACE}}}post_date").text + post_date_gmt = item.find(f"./{{{WP_NAMESPACE}}}post_date_gmt").text + status = item.find(f"./{{{WP_NAMESPACE}}}status").text + post_parent = item.find(f"./{{{WP_NAMESPACE}}}post_parent").text + menu_order = item.find(f"./{{{WP_NAMESPACE}}}menu_order").text + post_type = item.find(f"./{{{WP_NAMESPACE}}}post_type").text + post_name = item.find(f"./{{{WP_NAMESPACE}}}post_name").text + is_sticky = item.find(f"./{{{WP_NAMESPACE}}}is_sticky").text + ping_status = item.find(f"./{{{WP_NAMESPACE}}}ping_status").text + post_password = item.find(f"./{{{WP_NAMESPACE}}}post_password").text category_items = item.findall("./category") category_domains = {} @@ -309,15 +308,15 @@ def _parse_posts(element): def _parse_postmeta(element): """ - Retrive post metadata as a dictionary + Retrieve post metadata as a dictionary """ metadata = {} - fields = element.findall("./{%s}postmeta" % WP_NAMESPACE) + fields = element.findall(f"./{{{WP_NAMESPACE}}}postmeta") for field in fields: - key = field.find("./{%s}meta_key" % WP_NAMESPACE).text - value = field.find("./{%s}meta_value" % WP_NAMESPACE).text + key = field.find(f"./{{{WP_NAMESPACE}}}meta_key").text + value = field.find(f"./{{{WP_NAMESPACE}}}meta_value").text if key == "_wp_attachment_metadata": stream = StringIO(value.encode()) @@ -343,21 +342,21 @@ def _parse_comments(element): """ comments = [] - items = element.findall("./{%s}comment" % WP_NAMESPACE) + items = element.findall(f"./{{{WP_NAMESPACE}}}comment") for item in items: - comment_id = item.find("./{%s}comment_id" % WP_NAMESPACE).text - author = item.find("./{%s}comment_author" % WP_NAMESPACE).text - email = item.find("./{%s}comment_author_email" % WP_NAMESPACE).text - author_url = item.find("./{%s}comment_author_url" % WP_NAMESPACE).text - author_ip = item.find("./{%s}comment_author_IP" % WP_NAMESPACE).text - date = item.find("./{%s}comment_date" % WP_NAMESPACE).text - date_gmt = item.find("./{%s}comment_date_gmt" % WP_NAMESPACE).text - content = item.find("./{%s}comment_content" % WP_NAMESPACE).text - approved = item.find("./{%s}comment_approved" % WP_NAMESPACE).text - comment_type = item.find("./{%s}comment_type" % WP_NAMESPACE).text - parent = item.find("./{%s}comment_parent" % WP_NAMESPACE).text - user_id = item.find("./{%s}comment_user_id" % WP_NAMESPACE).text + comment_id = item.find(f"./{{{WP_NAMESPACE}}}comment_id").text + author = item.find(f"./{{{WP_NAMESPACE}}}comment_author").text + email = item.find(f"./{{{WP_NAMESPACE}}}comment_author_email").text + author_url = item.find(f"./{{{WP_NAMESPACE}}}comment_author_url").text + author_ip = item.find(f"./{{{WP_NAMESPACE}}}comment_author_IP").text + date = item.find(f"./{{{WP_NAMESPACE}}}comment_date").text + date_gmt = item.find(f"./{{{WP_NAMESPACE}}}comment_date_gmt").text + content = item.find(f"./{{{WP_NAMESPACE}}}comment_content").text + approved = item.find(f"./{{{WP_NAMESPACE}}}comment_approved").text + comment_type = item.find(f"./{{{WP_NAMESPACE}}}comment_type").text + parent = item.find(f"./{{{WP_NAMESPACE}}}comment_parent").text + user_id = item.find(f"./{{{WP_NAMESPACE}}}comment_user_id").text comment = { "id": comment_id, From 45cd8944c3b74e05c8a2088a48d0cba1a7118925 Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Thu, 21 Oct 2021 11:41:56 -0700 Subject: [PATCH 06/15] Fix Typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b68a0b4..e61e8cf 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # wpparser -This library parses Wordpress xml based exports into a simple python dictionary. +This library parses WordPress xml based exports into a simple python dictionary. ## How it works @@ -39,7 +39,7 @@ It returns a well formatted dict, containing the following datatypes: "language": "en-US", "title": "Marteinn / Blog" }, - "authors: [{ + "authors": [{ "login": "admin", "last_name": None, "display_name": "admin", From 97d19a541b60246cf818e54f1e85322ea75d647e Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Thu, 21 Oct 2021 11:42:16 -0700 Subject: [PATCH 07/15] Use more recent pypandoc --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 102ac7e..df93b1c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,2 +1,2 @@ -r tests.txt -pypandoc==0.9.9 +pypandoc>0.9.9 From ca880598d4f5c75519c19905daa229cc0817a88f Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Thu, 21 Oct 2021 11:43:38 -0700 Subject: [PATCH 08/15] Add Type hinting and some functionless code cleanup --- wpparser/parser.py | 126 +++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/wpparser/parser.py b/wpparser/parser.py index 3387651..9994b66 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -4,9 +4,11 @@ Load and parse the wp export file into a readable dictionary. """ - +import logging import xml.etree.ElementTree as ET from io import BytesIO as StringIO +from pathlib import Path +from typing import Dict, List, Optional, Union import phpserialize @@ -18,7 +20,7 @@ WP_NAMESPACE = "http://wordpress.org/export/1.2/" -def parse(path): +def parse(path: Union[str, Path]) -> dict: """ Parses xml and returns a formatted dict. @@ -116,16 +118,20 @@ def parse(path): } -def _parse_blog(element): +def _get_wp_element(element: ET.Element, name: str) -> str: + return element.find(f"./{{{WP_NAMESPACE}}}{name}").text + + +def _parse_blog(element: ET.Element) -> Dict[str, str]: """ - Parse and return genral blog data (title, tagline etc). + Parse and return general blog data (title, tagline etc). """ title = element.find("./title").text tagline = element.find("./description").text language = element.find("./language").text - site_url = element.find(f"./{{{WP_NAMESPACE}}}base_site_url").text - blog_url = element.find(f"./{{{WP_NAMESPACE}}}base_blog_url").text + site_url = _get_wp_element(element, "base_site_url") + blog_url = _get_wp_element(element, "base_blog_url") return { "title": title, @@ -136,20 +142,19 @@ def _parse_blog(element): } -def _parse_authors(element): +def _parse_authors(element: ET.Element) -> List[Dict[str, str]]: """ Returns a well formatted list of users that can be matched against posts. """ authors = [] - items = element.findall(f"./{{{WP_NAMESPACE}}}author") - for item in items: - login = item.find(f"./{{{WP_NAMESPACE}}}author_login").text - email = item.find(f"./{{{WP_NAMESPACE}}}author_email").text - first_name = item.find(f"./{{{WP_NAMESPACE}}}author_first_name").text - last_name = item.find(f"./{{{WP_NAMESPACE}}}author_last_name").text - display_name = item.find(f"./{{{WP_NAMESPACE}}}author_display_name").text + for item in element.findall(f"./{{{WP_NAMESPACE}}}author"): + login = _get_wp_element(item, "author_login") + email = _get_wp_element(item, "author_email") + first_name = _get_wp_element(item, "author_first_name") + last_name = _get_wp_element(item, "author_last_name") + display_name = _get_wp_element(item, "author_display_name") authors.append({ "login": login, @@ -162,18 +167,17 @@ def _parse_authors(element): return authors -def _parse_categories(element): +def _parse_categories(element: ET.Element) -> List[Dict[str, str]]: """ Returns a list with categories with relations. """ reference = {} - items = element.findall(f"./{{{WP_NAMESPACE}}}category") - for item in items: - term_id = item.find(f"./{{{WP_NAMESPACE}}}term_id").text - nicename = item.find(f"./{{{WP_NAMESPACE}}}category_nicename").text - name = item.find(f"./{{{WP_NAMESPACE}}}cat_name").text - parent = item.find(f"./{{{WP_NAMESPACE}}}category_parent").text + for item in element.findall(f"./{{{WP_NAMESPACE}}}category"): + term_id = _get_wp_element(item, "term_id") + nicename = _get_wp_element(item, "category_nicename") + name = _get_wp_element(item, "cat_name") + parent = _get_wp_element(item, "category_parent") category = { "term_id": term_id, @@ -187,7 +191,10 @@ def _parse_categories(element): return _build_category_tree(None, reference=reference) -def _build_category_tree(slug, reference=None, items=None): +def _build_category_tree(slug: Optional[str], + reference: Optional[Dict[str, Dict[str, str]]] = None, + items: Optional[List[Dict[str, str]]] = None + ) -> List[Dict[str, str]]: """ Builds a recursive tree with category relations as children. """ @@ -199,17 +206,15 @@ def _build_category_tree(slug, reference=None, items=None): category = reference[key] if category["parent"] == slug: - children = _build_category_tree(category["nicename"], - reference=reference) - category["children"] = children + category["children"] = _build_category_tree(category["nicename"], reference=reference) items.append(category) return items -def _parse_tags(element): +def _parse_tags(element: ET.Element) -> List[Dict[str, str]]: """ - Retrieves and parses tags into a array/dict. + Retrieves and parses tags into an array/dict. Example: @@ -218,12 +223,11 @@ def _parse_tags(element): """ tags = [] - items = element.findall(f"./{{{WP_NAMESPACE}}}tag") - for item in items: - term_id = item.find(f"./{{{WP_NAMESPACE}}}term_id").text - slug = item.find(f"./{{{WP_NAMESPACE}}}tag_slug").text - name = item.find(f"./{{{WP_NAMESPACE}}}tag_name").text + for item in element.findall(f"./{{{WP_NAMESPACE}}}tag"): + term_id = _get_wp_element(item, "term_id") + slug = _get_wp_element(item, "tag_slug") + name = _get_wp_element(item, "tag_name") tag = { "term_id": term_id, @@ -236,7 +240,7 @@ def _parse_tags(element): return tags -def _parse_posts(element): +def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: """ Returns a list with posts. """ @@ -253,17 +257,17 @@ def _parse_posts(element): description = item.find("./description").text content = item.find(f"./{{{CONTENT_NAMESPACE}}}encoded").text excerpt = item.find(f"./{{{EXCERPT_NAMESPACE}}}encoded").text - post_id = item.find(f"./{{{WP_NAMESPACE}}}post_id").text - post_date = item.find(f"./{{{WP_NAMESPACE}}}post_date").text - post_date_gmt = item.find(f"./{{{WP_NAMESPACE}}}post_date_gmt").text - status = item.find(f"./{{{WP_NAMESPACE}}}status").text - post_parent = item.find(f"./{{{WP_NAMESPACE}}}post_parent").text - menu_order = item.find(f"./{{{WP_NAMESPACE}}}menu_order").text - post_type = item.find(f"./{{{WP_NAMESPACE}}}post_type").text - post_name = item.find(f"./{{{WP_NAMESPACE}}}post_name").text - is_sticky = item.find(f"./{{{WP_NAMESPACE}}}is_sticky").text - ping_status = item.find(f"./{{{WP_NAMESPACE}}}ping_status").text - post_password = item.find(f"./{{{WP_NAMESPACE}}}post_password").text + post_id = _get_wp_element(item, "post_id") + post_date = _get_wp_element(item, "post_date") + post_date_gmt = _get_wp_element(item, "post_date_gmt") + status = _get_wp_element(item, "status") + post_parent = _get_wp_element(item, "post_parent") + menu_order = _get_wp_element(item, "menu_order") + post_type = _get_wp_element(item, "post_type") + post_name = _get_wp_element(item, "post_name") + is_sticky = _get_wp_element(item, "is_sticky") + ping_status = _get_wp_element(item, "ping_status") + post_password = _get_wp_element(item, "post_password") category_items = item.findall("./category") category_domains = {} @@ -306,7 +310,7 @@ def _parse_posts(element): return posts -def _parse_postmeta(element): +def _parse_postmeta(element: ET.Element) -> Dict[str, str]: """ Retrieve post metadata as a dictionary """ @@ -315,8 +319,8 @@ def _parse_postmeta(element): fields = element.findall(f"./{{{WP_NAMESPACE}}}postmeta") for field in fields: - key = field.find(f"./{{{WP_NAMESPACE}}}meta_key").text - value = field.find(f"./{{{WP_NAMESPACE}}}meta_value").text + key = _get_wp_element(field, 'meta_key') + value = _get_wp_element(field, 'meta_value') if key == "_wp_attachment_metadata": stream = StringIO(value.encode()) @@ -324,7 +328,7 @@ def _parse_postmeta(element): data = phpserialize.load(stream) metadata["attachment_metadata"] = data except ValueError as e: - pass + logging.warning(e) except Exception as e: raise e @@ -336,7 +340,7 @@ def _parse_postmeta(element): return metadata -def _parse_comments(element): +def _parse_comments(element: ET.Element) -> List[Dict[str, str]]: """ Returns a list with comments. """ @@ -345,18 +349,18 @@ def _parse_comments(element): items = element.findall(f"./{{{WP_NAMESPACE}}}comment") for item in items: - comment_id = item.find(f"./{{{WP_NAMESPACE}}}comment_id").text - author = item.find(f"./{{{WP_NAMESPACE}}}comment_author").text - email = item.find(f"./{{{WP_NAMESPACE}}}comment_author_email").text - author_url = item.find(f"./{{{WP_NAMESPACE}}}comment_author_url").text - author_ip = item.find(f"./{{{WP_NAMESPACE}}}comment_author_IP").text - date = item.find(f"./{{{WP_NAMESPACE}}}comment_date").text - date_gmt = item.find(f"./{{{WP_NAMESPACE}}}comment_date_gmt").text - content = item.find(f"./{{{WP_NAMESPACE}}}comment_content").text - approved = item.find(f"./{{{WP_NAMESPACE}}}comment_approved").text - comment_type = item.find(f"./{{{WP_NAMESPACE}}}comment_type").text - parent = item.find(f"./{{{WP_NAMESPACE}}}comment_parent").text - user_id = item.find(f"./{{{WP_NAMESPACE}}}comment_user_id").text + comment_id = _get_wp_element(item, 'comment_id') + author = _get_wp_element(item, 'comment_author') + email = _get_wp_element(item, 'comment_author_email') + author_url = _get_wp_element(item, 'comment_author_url') + author_ip = _get_wp_element(item, 'comment_author_IP') + date = _get_wp_element(item, 'comment_date') + date_gmt = _get_wp_element(item, 'comment_date_gmt') + content = _get_wp_element(item, 'comment_content') + approved = _get_wp_element(item, 'comment_approved') + comment_type = _get_wp_element(item, 'comment_type') + parent = _get_wp_element(item, 'comment_parent') + user_id = _get_wp_element(item, 'comment_user_id') comment = { "id": comment_id, From dd651a2b40cabccb4dc329608269158cb1d5abac Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Thu, 21 Oct 2021 11:44:15 -0700 Subject: [PATCH 09/15] Minimum python version is now 3.7. Remove some py2 compatibility stuff --- .travis.yml | 3 ++- runtests.py | 2 -- setup.py | 8 ++++---- tox.ini | 3 ++- wpparser/__init__.py | 3 --- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 579d423..062f320 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,10 @@ language: python python: - "pypy" - - "3.6" - "3.7" - "3.8" + - "3.9" + - "3.10" install: - pip install -r requirements/tests.txt script: diff --git a/runtests.py b/runtests.py index cc43b4e..5be484e 100644 --- a/runtests.py +++ b/runtests.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- - """Tests""" import unittest diff --git a/setup.py b/setup.py index 7205f32..0f0c16f 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import os import re @@ -29,8 +28,8 @@ # Convert markdown to rst try: - from pypandoc import convert - long_description = convert("README.md", "rst") + from pypandoc import convert_file + long_description = convert_file("README.md", "rst") except: long_description = "" @@ -56,8 +55,9 @@ "Natural Language :: English", "License :: OSI Approved :: MIT License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], ) diff --git a/tox.ini b/tox.ini index 04cb9dc..ec12039 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,5 @@ -envlist=py36,py37,py38 +[tox] +envlist=py37,py38,py39,py310 [testenv] extras = test diff --git a/wpparser/__init__.py b/wpparser/__init__.py index 2f03527..afe7392 100644 --- a/wpparser/__init__.py +++ b/wpparser/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ wpparser --- @@ -18,5 +16,4 @@ __license__ = "MIT" __copyright__ = "Copyright 2014-2020 Martin Sandström" - from .parser import parse # NOQA From b0789c74a691bddcbe231271b99ad2574f45a4be Mon Sep 17 00:00:00 2001 From: Ryan Jarvis Date: Mon, 4 Apr 2022 10:47:59 -0700 Subject: [PATCH 10/15] Include modified datetime for Post --- wpparser/parser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wpparser/parser.py b/wpparser/parser.py index 9994b66..b49e6c2 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -260,6 +260,8 @@ def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: post_id = _get_wp_element(item, "post_id") post_date = _get_wp_element(item, "post_date") post_date_gmt = _get_wp_element(item, "post_date_gmt") + post_modified = _get_wp_element(item, "post_modified") + post_modified_gmt = _get_wp_element(item, "post_modified_gmt") status = _get_wp_element(item, "status") post_parent = _get_wp_element(item, "post_parent") menu_order = _get_wp_element(item, "menu_order") @@ -289,6 +291,8 @@ def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: "post_id": post_id, "post_date": post_date, "post_date_gmt": post_date_gmt, + "post_modified": post_modified, + "post_modified_gmt": post_modified_gmt, "status": status, "post_parent": post_parent, "menu_order": menu_order, From 90367fdd07c755bad8c320e9c991089649e77891 Mon Sep 17 00:00:00 2001 From: Ryan Jarvis <828557+Cabalist@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:48:56 -0800 Subject: [PATCH 11/15] Fix build process by updating setup.py --- pyproject.toml | 3 +++ setup.py | 27 +++++---------------------- 2 files changed, 8 insertions(+), 22 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9273fee --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.backends._legacy:_Backend" \ No newline at end of file diff --git a/setup.py b/setup.py index 0f0c16f..af3a61d 100644 --- a/setup.py +++ b/setup.py @@ -1,21 +1,8 @@ #!/usr/bin/env python -import os import re -import sys from setuptools import find_packages, setup -import wpparser - -if sys.argv[-1] == "publish": - os.system("python setup.py sdist upload") - sys.exit() - - -test_requirements = [ - "pytest>=3", -] - requires = [ "phpserialize>=1.3", ] @@ -26,27 +13,23 @@ r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE ).group(1) -# Convert markdown to rst -try: - from pypandoc import convert_file - long_description = convert_file("README.md", "rst") -except: - long_description = "" +with open("README.md", "r") as f: + long_description = f.read() setup( name="wpparser", version=version, - description="Parse wordpress export files into a well formatted python dictionary", # NOQA + description="Parse wordpress export files into a well formatted python dictionary", long_description=long_description, + long_description_content_type="text/markdown", author="Martin Sandström", author_email="martin@marteinn.se", url="https://github.com/marteinn/wpparser", packages=find_packages(), - package_data={"": ["LICENSE", ], "wpparser": ["*.txt"]}, + package_data={"": ["LICENSE"], "wpparser": ["*.txt"]}, package_dir={"wpparser": "wpparser"}, include_package_data=True, install_requires=requires, - tests_require=test_requirements, license="MIT", zip_safe=False, classifiers=[ From d0da34a865e2dcdb20625363a6179b2ad5f9121f Mon Sep 17 00:00:00 2001 From: Ryan Jarvis <828557+Cabalist@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:52:09 -0800 Subject: [PATCH 12/15] Fix build process by updating setup.py --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9273fee..1b68d94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] requires = ["setuptools>=42", "wheel"] -build-backend = "setuptools.backends._legacy:_Backend" \ No newline at end of file +build-backend = "setuptools.build_meta" \ No newline at end of file From d89014b06415efb158786c76b06d930398e341ca Mon Sep 17 00:00:00 2001 From: Ryan Jarvis <828557+Cabalist@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:31:08 -0800 Subject: [PATCH 13/15] Modernize the codebase --- .github/workflows/tests.yml | 36 ++++ .gitignore | 5 +- .travis.yml | 11 -- MANIFEST.in | 1 - Makefile | 5 - README.md | 172 +++++++++--------- pyproject.toml | 57 +++++- requirements/dev.txt | 2 - requirements/install.txt | 1 - requirements/tests.txt | 2 - runtests.py | 31 ---- setup.py | 46 ----- tests/__init__.py | 0 tests/conftest.py | 10 + .../fixtures/blog.wordpress.2014-09-26.xml | 0 tests/test_parser.py | 146 +++++++++++++++ tox.ini | 9 - wpparser/__init__.py | 2 +- wpparser/parser.py | 146 ++++++++------- 19 files changed, 417 insertions(+), 265 deletions(-) create mode 100644 .github/workflows/tests.yml delete mode 100644 .travis.yml delete mode 100644 MANIFEST.in delete mode 100644 Makefile delete mode 100644 requirements/dev.txt delete mode 100644 requirements/install.txt delete mode 100644 requirements/tests.txt delete mode 100644 runtests.py delete mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py rename blog.wordpress.2014-09-26.xml => tests/fixtures/blog.wordpress.2014-09-26.xml (100%) create mode 100644 tests/test_parser.py delete mode 100644 tox.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..6eaf122 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,36 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/ruff-action@v3 + - name: Install dependencies + run: pip install phpserialize>=1.3 setuptools>=42 + - uses: astral-sh/ty-action@v0 + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - name: Install dependencies + run: | + pip install --upgrade pip + pip install pytest phpserialize>=1.3 + - name: Run tests + run: pytest -s tests/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index f5bd85d..525c146 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +__pycache__/ .DS_Store .*.swp tags @@ -8,4 +9,6 @@ build .coverage *.egg-info/ venv* -.tox/ \ No newline at end of file +.tox/ +.pytest_cache/ +.ruff_cache/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 062f320..0000000 --- a/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: python -python: - - "pypy" - - "3.7" - - "3.8" - - "3.9" - - "3.10" -install: - - pip install -r requirements/tests.txt -script: - - python runtests.py diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d7fd551..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include README.md LICENSE requirements/install.txt requirements/tests.txt test.py diff --git a/Makefile b/Makefile deleted file mode 100644 index 43be683..0000000 --- a/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -init: - pip install -r requirements.txt - -test: - py.test test.py -s diff --git a/README.md b/README.md index e61e8cf..34cf60e 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,32 @@ -[![Build Status](https://travis-ci.org/marteinn/wpparser.svg?branch=master)](https://travis-ci.org/marteinn/wpparser) -[![PyPI version](https://badge.fury.io/py/wpparser.svg)](http://badge.fury.io/py/wpparser) - # wpparser -This library parses WordPress xml based exports into a simple python dictionary. - +This library parses WordPress XML exports into a Python dictionary. -## How it works +## Installation -The library uses ElementTree to traverse through the export file. +```bash +pip install wpparser +``` ## Usage - import wpparser - - data = wpparser.parse("./blog.wordpress.2014-09-26.xml") - >>> {"blog": {"tagline": "Tagline",... +```python +import wpparser +data = wpparser.parse("./blog.wordpress.2014-09-26.xml") +``` ## What it returns -It returns a well formatted dict, containing the following datatypes: +A dictionary containing: -- Blog: The general blog information, such as tagline, site url. -- Authors: A list with the different authors. -- Categories: The categories in use, organized as a nested array. -- Tags: A list with the different tags. -- Posts: An array that contains all posts, the post object might also contain the different comments belonging to the post. +- **blog**: General blog information (title, tagline, site url, etc.) +- **authors**: List of authors +- **categories**: Categories organized as a nested tree +- **tags**: List of tags +- **posts**: List of posts, including comments and post metadata -### Example: +### Example ```python { @@ -37,85 +35,83 @@ It returns a well formatted dict, containing the following datatypes: "site_url": "http://marteinn.se/blog", "blog_url": "http://marteinn.se/blog", "language": "en-US", - "title": "Marteinn / Blog" + "title": "Marteinn / Blog", }, - "authors": [{ - "login": "admin", - "last_name": None, - "display_name": "admin", - "email": "martin@marteinn.se", - "first_name": None} + "authors": [ + { + "login": "admin", + "last_name": None, + "display_name": "admin", + "email": "martin@marteinn.se", + "first_name": None, + } + ], + "categories": [ + { + "parent": None, + "term_id": "3", + "name": "Action Script", + "nicename": "action-script", + "children": [ + { + "parent": "action-script", + "term_id": "20", + "name": "Flash related", + "nicename": "flash-related", + "children": [], + } + ], + } + ], + "tags": [{"term_id": "1", "slug": "bash", "name": "Bash"}], + "posts": [ + { + "creator": "admin", + "excerpt": None, + "post_date_gmt": "2014-09-22 20:10:40", + "post_date": "2014-09-22 21:10:40", + "post_type": "post", + "menu_order": "0", + "guid": "http://marteinn.se/blog/?p=828", + "title": "Post Title", + "comments": [ + { + "date_gmt": "2014-09-24 23:08:31", + "parent": "0", + "date": "2014-09-25 00:08:31", + "id": "85929", + "user_id": "0", + "author": "Author", + "author_email": None, + "author_ip": "111.111.111.111", + "approved": "1", + "content": "Comment title", + "author_url": "http://example.com", + "type": "pingback", + } + ], + "content": "Text", + "post_parent": "0", + "post_password": None, + "status": "publish", + "description": None, + "tags": ["tag"], + "ping_status": "open", + "post_id": "1", + "link": "http://www.marteinn.se/blog/slug/", + "pub_date": "Mon, 22 Sep 2014 20:10:40 +0000", + "categories": ["category"], + "is_sticky": "0", + "post_name": "slug", + } ], - "categories": [{ - "parent": None, - "term_id": "3", - "name": "Action Script", - "nicename": "action-script", - "children": [{ - "parent": "action-script", - "term_id": "20", - "name": "Flash related", - "nicename": "flash-related", - "children": [] - }] - }], - "tags": [{ - "term_id": "1", - "slug": "bash", - "name": "Bash" - }], - "posts": [{ - "creator": "admin", - "excerpt": None, - "post_date_gmt": "2014-09-22 20:10:40", - "post_date": "2014-09-22 21:10:40", - "post_type": "post", - "menu_order": "0", - "guid": "http://marteinn.se/blog/?p=828", - "title": "Post Title", - "comments": [{ - "date_gmt": "2014-09-24 23:08:31", - "parent": "0", - "date": "2014-09-25 00:08:31", - "id": "85929", - "user_id": "0", - "author": u"Author", - "author_email": None, - "author_ip": "111.111.111.111", - "approved": "1", - "content": u"Comment title", - "author_url": "http://example.com", - "type": "pingback" - }], - "content": "Text", - "post_parent": "0", - "post_password": None, - "status": "publish", - "description": None, - "tags": ["tag"], - "ping_status": "open", - "post_id": "1", - "link": "http://www.marteinn.se/blog/slug/", - "pub_date": "Mon, 22 Sep 2014 20:10:40 +0000", - "categories": ["category"], - "is_sticky": "0", - "post_name": "slug" - }] } ``` -## Installation - -wpparser can easily be installed through pip. - - $ pip install wpparser - - ## Contributing Want to contribute? Awesome. Just send a pull request. - ## License wpparser is released under the [MIT License](http://www.opensource.org/licenses/MIT). diff --git a/pyproject.toml b/pyproject.toml index 1b68d94..f68d5a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,58 @@ [build-system] requires = ["setuptools>=42", "wheel"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +[project] +name = "wpparser" +dynamic = ["version"] +description = "Parse wordpress export files into a well formatted python dictionary" +readme = "README.md" +license = "MIT" +requires-python = ">=3.10" +authors = [ + { name = "Martin Sandström", email = "martin@marteinn.se" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "phpserialize>=1.3", +] + +[project.urls] +Homepage = "https://github.com/marteinn/wpparser" + +[tool.setuptools.dynamic] +version = { attr = "wpparser.__version__" } + +[tool.ruff] +target-version = "py314" +line-length = 120 + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + "RUF", # ruff-specific rules +] + +[tool.ruff.lint.isort] +known-third-party = ["phpserialize"] + +[tool.ty.environment] +python-version = "3.14" + +[tool.ty.rules] +possibly-unresolved-reference = "warn" +unresolved-reference = "error" +unresolved-import = "warn" diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index df93b1c..0000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,2 +0,0 @@ --r tests.txt -pypandoc>0.9.9 diff --git a/requirements/install.txt b/requirements/install.txt deleted file mode 100644 index f8449aa..0000000 --- a/requirements/install.txt +++ /dev/null @@ -1 +0,0 @@ -phpserialize==1.3 diff --git a/requirements/tests.txt b/requirements/tests.txt deleted file mode 100644 index 539e4ae..0000000 --- a/requirements/tests.txt +++ /dev/null @@ -1,2 +0,0 @@ --r install.txt -pytest>=3 diff --git a/runtests.py b/runtests.py deleted file mode 100644 index 5be484e..0000000 --- a/runtests.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -"""Tests""" - -import unittest -from wpparser import parse - - -class ParseTestCase(unittest.TestCase): - def test_parse(self): - result = parse("./blog.wordpress.2014-09-26.xml") - - assert len(result["posts"]) is 3 - assert result["blog"]["title"] == "Blog" - assert len(result["categories"]) is 1 - assert len(result["tags"]) is 1 - - def test_attachment_metadata(self): - result = parse("./blog.wordpress.2014-09-26.xml") - - post = result["posts"][2] - - assert "postmeta" in post - assert "attached_file" in post["postmeta"] - assert "attachment_metadata" in post["postmeta"] - - attached_file = post["postmeta"]["attached_file"] - assert attached_file == "logo-promo.png" - - -if __name__ == "__main__": - unittest.main() diff --git a/setup.py b/setup.py deleted file mode 100644 index af3a61d..0000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python - -import re -from setuptools import find_packages, setup - -requires = [ - "phpserialize>=1.3", -] - -version = "" -with open("wpparser/__init__.py", "r") as fd: - version = re.search( - r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE - ).group(1) - -with open("README.md", "r") as f: - long_description = f.read() - -setup( - name="wpparser", - version=version, - description="Parse wordpress export files into a well formatted python dictionary", - long_description=long_description, - long_description_content_type="text/markdown", - author="Martin Sandström", - author_email="martin@marteinn.se", - url="https://github.com/marteinn/wpparser", - packages=find_packages(), - package_data={"": ["LICENSE"], "wpparser": ["*.txt"]}, - package_dir={"wpparser": "wpparser"}, - include_package_data=True, - install_requires=requires, - license="MIT", - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Natural Language :: English", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], -) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..87191b1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +from pathlib import Path + +import pytest + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +@pytest.fixture +def sample_export(): + return str(FIXTURES_DIR / "blog.wordpress.2014-09-26.xml") diff --git a/blog.wordpress.2014-09-26.xml b/tests/fixtures/blog.wordpress.2014-09-26.xml similarity index 100% rename from blog.wordpress.2014-09-26.xml rename to tests/fixtures/blog.wordpress.2014-09-26.xml diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..b711bb8 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,146 @@ +import pytest + +from wpparser import parse + + +def test_parse(sample_export): + result = parse(sample_export) + + assert len(result["posts"]) == 3 + assert result["blog"]["title"] == "Blog" + assert len(result["categories"]) == 1 + assert len(result["tags"]) == 1 + + +def test_attachment_metadata(sample_export): + result = parse(sample_export) + + post = result["posts"][2] + + assert "postmeta" in post + assert "attached_file" in post["postmeta"] + assert "attachment_metadata" in post["postmeta"] + + attached_file = post["postmeta"]["attached_file"] + assert attached_file == "logo-promo.png" + + +def test_blog_metadata(sample_export): + result = parse(sample_export) + blog = result["blog"] + + assert blog["title"] == "Blog" + assert blog["tagline"] == "Just another WordPress site" + assert blog["language"] == "en-US" + assert blog["site_url"] == "http://marteinn.se/blog" + assert blog["blog_url"] == "http://marteinn.se/blog" + + +def test_authors(sample_export): + result = parse(sample_export) + authors = result["authors"] + + assert len(authors) == 1 + + author = authors[0] + assert author["login"] == "admin" + assert author["email"] == "martin@marteinn.se" + assert author["display_name"] == "admin" + assert author["first_name"] is None + assert author["last_name"] is None + + +def test_category_structure(sample_export): + result = parse(sample_export) + categories = result["categories"] + + assert len(categories) == 1 + cat = categories[0] + assert cat["term_id"] == "1" + assert cat["nicename"] == "uncategorized" + assert cat["name"] == "Uncategorized" + assert cat["parent"] is None + assert cat["children"] == [] + + +def test_tags(sample_export): + result = parse(sample_export) + tags = result["tags"] + + assert len(tags) == 1 + tag = tags[0] + assert tag["term_id"] == "1" + assert tag["slug"] == "bash" + assert tag["name"] == "Bash" + + +def test_post_fields(sample_export): + result = parse(sample_export) + post = result["posts"][0] + + assert post["title"] == "Hello world!" + assert post["content"] == "Welcome to WordPress. This is your first post. Edit or delete it, then start blogging!" + assert post["post_date"] == "2014-09-26 18:47:05" + assert post["post_date_gmt"] == "2014-09-26 18:47:06" + assert post["status"] == "publish" + assert post["post_type"] == "post" + assert post["post_id"] == "1" + assert post["post_name"] == "hello-world" + assert post["creator"] == "admin" + assert post["ping_status"] == "open" + assert post["menu_order"] == "0" + assert post["is_sticky"] == "0" + + +def test_post_modified_fields(sample_export): + result = parse(sample_export) + post = result["posts"][0] + + assert "post_modified" in post + assert "post_modified_gmt" in post + + +def test_page_post_type(sample_export): + result = parse(sample_export) + page = result["posts"][1] + + assert page["title"] == "Sample Page" + assert page["post_type"] == "page" + + +def test_comments(sample_export): + result = parse(sample_export) + post = result["posts"][0] + comments = post["comments"] + + assert len(comments) == 1 + comment = comments[0] + assert comment["id"] == "1" + assert comment["author"] == "Mr WordPress" + assert comment["author_url"] == "https://wordpress.org/" + assert comment["date"] == "2014-09-26 18:47:05" + assert comment["date_gmt"] == "2014-09-26 18:47:06" + assert comment["approved"] == "1" + assert comment["parent"] == "0" + assert comment["user_id"] == "0" + assert "Hi, this is a comment." in comment["content"] + + +def test_missing_channel_raises_valueerror(tmp_path): + xml_content = '' + xml_file = tmp_path / "invalid.xml" + xml_file.write_text(xml_content) + + with pytest.raises(ValueError, match="missing element"): + parse(str(xml_file)) + + +def test_post_categories_domain(sample_export): + result = parse(sample_export) + post = result["posts"][0] + + assert "category_category" in post + cats = post["category_category"] + assert len(cats) == 1 + assert cats[0]["nicename"] == "uncategorized" + assert cats[0]["text"] == "Uncategorized" diff --git a/tox.ini b/tox.ini deleted file mode 100644 index ec12039..0000000 --- a/tox.ini +++ /dev/null @@ -1,9 +0,0 @@ -[tox] -envlist=py37,py38,py39,py310 - -[testenv] -extras = test -deps = - pytest -commands = - pytest -s runtests.py diff --git a/wpparser/__init__.py b/wpparser/__init__.py index afe7392..f4a1ef4 100644 --- a/wpparser/__init__.py +++ b/wpparser/__init__.py @@ -16,4 +16,4 @@ __license__ = "MIT" __copyright__ = "Copyright 2014-2020 Martin Sandström" -from .parser import parse # NOQA +from .parser import parse as parse diff --git a/wpparser/parser.py b/wpparser/parser.py index b49e6c2..5beb12e 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -4,11 +4,14 @@ Load and parse the wp export file into a readable dictionary. """ + import logging import xml.etree.ElementTree as ET -from io import BytesIO as StringIO -from pathlib import Path -from typing import Dict, List, Optional, Union +from io import BytesIO +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from pathlib import Path import phpserialize @@ -20,7 +23,7 @@ WP_NAMESPACE = "http://wordpress.org/export/1.2/" -def parse(path: Union[str, Path]) -> dict: +def parse(path: str | Path) -> dict: """ Parses xml and returns a formatted dict. @@ -94,6 +97,8 @@ def parse(path: Union[str, Path]) -> dict: "pub_date": "Mon, 22 Sep 2014 20:10:40 +0000", "categories": ["category"], "is_sticky": "0", + "post_modified": "2014-09-22 21:10:40", + "post_modified_gmt": "2014-09-22 20:10:40", "post_name": "slug" }] } @@ -102,6 +107,8 @@ def parse(path: Union[str, Path]) -> dict: doc = ET.parse(path).getroot() channel = doc.find("./channel") + if channel is None: + raise ValueError("Invalid WordPress export: missing element") blog = _parse_blog(channel) authors = _parse_authors(channel) @@ -118,18 +125,23 @@ def parse(path: Union[str, Path]) -> dict: } -def _get_wp_element(element: ET.Element, name: str) -> str: - return element.find(f"./{{{WP_NAMESPACE}}}{name}").text +def _find_text(element: ET.Element, path: str) -> str | None: + el = element.find(path) + return el.text if el is not None else None + + +def _get_wp_element(element: ET.Element, name: str) -> str | None: + return _find_text(element, f"./{{{WP_NAMESPACE}}}{name}") -def _parse_blog(element: ET.Element) -> Dict[str, str]: +def _parse_blog(element: ET.Element) -> dict[str, str | None]: """ Parse and return general blog data (title, tagline etc). """ - title = element.find("./title").text - tagline = element.find("./description").text - language = element.find("./language").text + title = _find_text(element, "./title") + tagline = _find_text(element, "./description") + language = _find_text(element, "./language") site_url = _get_wp_element(element, "base_site_url") blog_url = _get_wp_element(element, "base_blog_url") @@ -142,7 +154,7 @@ def _parse_blog(element: ET.Element) -> Dict[str, str]: } -def _parse_authors(element: ET.Element) -> List[Dict[str, str]]: +def _parse_authors(element: ET.Element) -> list[dict[str, str | None]]: """ Returns a well formatted list of users that can be matched against posts. """ @@ -156,22 +168,24 @@ def _parse_authors(element: ET.Element) -> List[Dict[str, str]]: last_name = _get_wp_element(item, "author_last_name") display_name = _get_wp_element(item, "author_display_name") - authors.append({ - "login": login, - "email": email, - "display_name": display_name, - "first_name": first_name, - "last_name": last_name - }) + authors.append( + { + "login": login, + "email": email, + "display_name": display_name, + "first_name": first_name, + "last_name": last_name, + } + ) return authors -def _parse_categories(element: ET.Element) -> List[Dict[str, str]]: +def _parse_categories(element: ET.Element) -> list[dict[str, Any]]: """ Returns a list with categories with relations. """ - reference = {} + reference: dict[str, dict[str, Any]] = {} for item in element.findall(f"./{{{WP_NAMESPACE}}}category"): term_id = _get_wp_element(item, "term_id") @@ -179,22 +193,19 @@ def _parse_categories(element: ET.Element) -> List[Dict[str, str]]: name = _get_wp_element(item, "cat_name") parent = _get_wp_element(item, "category_parent") - category = { - "term_id": term_id, - "nicename": nicename, - "name": name, - "parent": parent - } + category: dict[str, Any] = {"term_id": term_id, "nicename": nicename, "name": name, "parent": parent} - reference[nicename] = category + if nicename is not None: + reference[nicename] = category return _build_category_tree(None, reference=reference) -def _build_category_tree(slug: Optional[str], - reference: Optional[Dict[str, Dict[str, str]]] = None, - items: Optional[List[Dict[str, str]]] = None - ) -> List[Dict[str, str]]: +def _build_category_tree( + slug: str | None, + reference: dict[str, dict[str, Any]], + items: list[dict[str, Any]] | None = None, +) -> list[dict[str, Any]]: """ Builds a recursive tree with category relations as children. """ @@ -212,7 +223,7 @@ def _build_category_tree(slug: Optional[str], return items -def _parse_tags(element: ET.Element) -> List[Dict[str, str]]: +def _parse_tags(element: ET.Element) -> list[dict[str, str | None]]: """ Retrieves and parses tags into an array/dict. @@ -240,7 +251,7 @@ def _parse_tags(element: ET.Element) -> List[Dict[str, str]]: return tags -def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: +def _parse_posts(element: ET.Element) -> list[dict[str, Any]]: """ Returns a list with posts. """ @@ -249,14 +260,14 @@ def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: items = element.findall("item") for item in items: - title = item.find("./title").text - link = item.find("./link").text - pub_date = item.find("./pubDate").text - creator = item.find(f"./{{{DC_NAMESPACE}}}creator").text - guid = item.find("./guid").text - description = item.find("./description").text - content = item.find(f"./{{{CONTENT_NAMESPACE}}}encoded").text - excerpt = item.find(f"./{{{EXCERPT_NAMESPACE}}}encoded").text + title = _find_text(item, "./title") + link = _find_text(item, "./link") + pub_date = _find_text(item, "./pubDate") + creator = _find_text(item, f"./{{{DC_NAMESPACE}}}creator") + guid = _find_text(item, "./guid") + description = _find_text(item, "./description") + content = _find_text(item, f"./{{{CONTENT_NAMESPACE}}}encoded") + excerpt = _find_text(item, f"./{{{EXCERPT_NAMESPACE}}}encoded") post_id = _get_wp_element(item, "post_id") post_date = _get_wp_element(item, "post_date") post_date_gmt = _get_wp_element(item, "post_date_gmt") @@ -272,14 +283,16 @@ def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: post_password = _get_wp_element(item, "post_password") category_items = item.findall("./category") - category_domains = {} + category_domains: dict[str, list[dict[str, str | None]]] = {} for category_item in category_items: if category_item.attrib["domain"] not in category_domains: category_domains[category_item.attrib["domain"]] = [] - category_domains[category_item.attrib["domain"]].append({'nicename': category_item.attrib["nicename"], 'text': category_item.text}) + category_domains[category_item.attrib["domain"]].append( + {"nicename": category_item.attrib["nicename"], "text": category_item.text} + ) - post = { + post: dict[str, Any] = { "title": title, "link": link, "pub_date": pub_date, @@ -305,7 +318,7 @@ def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: # Include all categories with a prefix inorder to avoid collisions for k, v in category_domains.items(): - post[f'category_{k}'] = v + post[f"category_{k}"] = v post["postmeta"] = _parse_postmeta(item) post["comments"] = _parse_comments(item) @@ -314,27 +327,28 @@ def _parse_posts(element: ET.Element) -> List[Dict[str, str]]: return posts -def _parse_postmeta(element: ET.Element) -> Dict[str, str]: +def _parse_postmeta(element: ET.Element) -> dict[str, Any]: """ Retrieve post metadata as a dictionary """ - metadata = {} + metadata: dict[str, Any] = {} fields = element.findall(f"./{{{WP_NAMESPACE}}}postmeta") for field in fields: - key = _get_wp_element(field, 'meta_key') - value = _get_wp_element(field, 'meta_value') + key = _get_wp_element(field, "meta_key") + value = _get_wp_element(field, "meta_value") + + if key is None: + continue - if key == "_wp_attachment_metadata": - stream = StringIO(value.encode()) + if key == "_wp_attachment_metadata" and value is not None: + stream = BytesIO(value.encode()) try: data = phpserialize.load(stream) metadata["attachment_metadata"] = data except ValueError as e: logging.warning(e) - except Exception as e: - raise e elif key == "_wp_attached_file": metadata["attached_file"] = value @@ -344,7 +358,7 @@ def _parse_postmeta(element: ET.Element) -> Dict[str, str]: return metadata -def _parse_comments(element: ET.Element) -> List[Dict[str, str]]: +def _parse_comments(element: ET.Element) -> list[dict[str, str | None]]: """ Returns a list with comments. """ @@ -353,18 +367,18 @@ def _parse_comments(element: ET.Element) -> List[Dict[str, str]]: items = element.findall(f"./{{{WP_NAMESPACE}}}comment") for item in items: - comment_id = _get_wp_element(item, 'comment_id') - author = _get_wp_element(item, 'comment_author') - email = _get_wp_element(item, 'comment_author_email') - author_url = _get_wp_element(item, 'comment_author_url') - author_ip = _get_wp_element(item, 'comment_author_IP') - date = _get_wp_element(item, 'comment_date') - date_gmt = _get_wp_element(item, 'comment_date_gmt') - content = _get_wp_element(item, 'comment_content') - approved = _get_wp_element(item, 'comment_approved') - comment_type = _get_wp_element(item, 'comment_type') - parent = _get_wp_element(item, 'comment_parent') - user_id = _get_wp_element(item, 'comment_user_id') + comment_id = _get_wp_element(item, "comment_id") + author = _get_wp_element(item, "comment_author") + email = _get_wp_element(item, "comment_author_email") + author_url = _get_wp_element(item, "comment_author_url") + author_ip = _get_wp_element(item, "comment_author_IP") + date = _get_wp_element(item, "comment_date") + date_gmt = _get_wp_element(item, "comment_date_gmt") + content = _get_wp_element(item, "comment_content") + approved = _get_wp_element(item, "comment_approved") + comment_type = _get_wp_element(item, "comment_type") + parent = _get_wp_element(item, "comment_parent") + user_id = _get_wp_element(item, "comment_user_id") comment = { "id": comment_id, From 701efabecfba70eb83fe25823f1695ff46e3f204 Mon Sep 17 00:00:00 2001 From: Ryan Jarvis <828557+Cabalist@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:40:46 -0800 Subject: [PATCH 14/15] Switch to mypy --- .github/workflows/tests.yml | 9 +++++++-- pyproject.toml | 25 +++++++++++++++++++------ wpparser/parser.py | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6eaf122..8240e96 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,9 +12,14 @@ jobs: steps: - uses: actions/checkout@v6 - uses: astral-sh/ruff-action@v3 + - uses: actions/setup-python@v5 + with: + python-version: "3.14" + allow-prereleases: true - name: Install dependencies - run: pip install phpserialize>=1.3 setuptools>=42 - - uses: astral-sh/ty-action@v0 + run: pip install phpserialize>=1.3 setuptools>=42 mypy + - name: Run mypy + run: mypy wpparser/ test: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index f68d5a0..5f417a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,10 +49,23 @@ select = [ [tool.ruff.lint.isort] known-third-party = ["phpserialize"] -[tool.ty.environment] -python-version = "3.14" +[tool.mypy] +python_version = "3.14" +strict = true +warn_unused_configs = true -[tool.ty.rules] -possibly-unresolved-reference = "warn" -unresolved-reference = "error" -unresolved-import = "warn" +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_unreachable = true + +show_error_codes = true +pretty = true + +[[tool.mypy.overrides]] +module = ["phpserialize"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = ["tests.*"] +disallow_untyped_defs = false \ No newline at end of file diff --git a/wpparser/parser.py b/wpparser/parser.py index 5beb12e..3db39a5 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -23,7 +23,7 @@ WP_NAMESPACE = "http://wordpress.org/export/1.2/" -def parse(path: str | Path) -> dict: +def parse(path: str | Path) -> dict[str, Any]: """ Parses xml and returns a formatted dict. From 55eb3336bf63fedd945bfa9ebb4b4f9276c9d01d Mon Sep 17 00:00:00 2001 From: Ryan Jarvis <828557+Cabalist@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:43:12 -0800 Subject: [PATCH 15/15] Fix for older pythons --- wpparser/parser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wpparser/parser.py b/wpparser/parser.py index 3db39a5..58992ca 100644 --- a/wpparser/parser.py +++ b/wpparser/parser.py @@ -5,6 +5,8 @@ Load and parse the wp export file into a readable dictionary. """ +from __future__ import annotations + import logging import xml.etree.ElementTree as ET from io import BytesIO