Diagnose RescueGroups response encoding by adding encoding scheme in log And Repair Mojibake

adoption_sources/rescue_groups.py

@@ -13,6 +13,7 @@
 from typing import Iterator
 
 import requests
+from ftfy import fix_and_explain
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
@@ -39,7 +40,6 @@
 RETRY_TOTAL = 4
 RETRY_BACKOFF_FACTOR = 1
 
-
 def _session_with_retries() -> requests.Session:
     """Build a requests Session that retries transient errors with backoff."""
     retry = Retry(
@@ -197,7 +197,7 @@ def _parse_animal(
             animal_id = animal.get("id", "")
 
             # Extract and clean the name
-            name = self._clean_name(attrs.get("name", "Unknown"))
+            name = self._clean_name(attrs.get("name", "Unknown"), animal_id=animal_id)
 
             # Determine species from the included species relationship
             species_id = (
@@ -217,10 +217,16 @@ def _parse_animal(
             species = SPECIES_SINGULAR[normalized_plural]
 
             # Get breed info
-            breed = attrs.get("breedString", attrs.get("breedPrimary", "Mixed"))
+            breed = self._repair_mojibake(
+                attrs.get("breedString", attrs.get("breedPrimary", "Mixed")),
+                "breed",
+                animal_id,
+            )
 
             # Clean up description (use text version, not HTML)
-            description = self._clean_description(attrs.get("descriptionText", ""))
+            description = self._clean_description(
+                attrs.get("descriptionText", ""), animal_id=animal_id
+            )
 
             # Get adoption_url
             org_id = (
@@ -256,7 +262,12 @@ def _parse_animal(
             image_url = self._get_image_url(attrs)
 
             # Location of the adoption org
-            location = f"{org_attrs.get('city')}, {org_attrs.get('state')}"
+            location = self._repair_mojibake(
+                f"{org_attrs.get('city')}, {org_attrs.get('state')}",
+                "location",
+                org_id or "unknown",
+                "organization",
+            )
 
 
             return AdoptablePet(
@@ -280,26 +291,67 @@ def _parse_animal(
     def _is_placeholder_name(self, name: str) -> bool:
         return name.lower() in PLACEHOLDER_NAMES
 
-    def _clean_name(self, name: str) -> str:
+    def _repair_mojibake(
+        self,
+        text: str,
+        field: str,
+        entity_id: str,
+        entity_type: str = "animal",
+    ) -> str:
+        """Apply ftfy's complete set of repairs to RescueGroups display text.
+
+        This deliberately uses ftfy's default configuration, including its
+        mixed/lossy encoding recovery and general Unicode cleanup. The complete
+        ``ExplainedText`` result is logged whenever ftfy changes a value.
+        """
+        if not text:
+            return text
+
+        result = fix_and_explain(text)
+        repaired = result.text
+        if repaired != text:
+            logger.info(
+                "Fixed RescueGroups %s for %s %s: ftfy_result=%r",
+                field,
+                entity_type,
+                entity_id,
+                result,
+            )
+            return repaired
+        return text
+
+    def _clean_name(self, name: str, animal_id: str = "unknown") -> str:
         """
         Clean up pet name by removing promotional text.
 
         Examples:
             "Doli ***Home for the Holidays 1/2 price!" -> "Doli"
             "Kathy" -> "Kathy"
         """
+        name = self._repair_mojibake(name, "name", animal_id)
+
         # Remove common promotional suffixes
         # Split on common delimiters and take the first part
         cleaned = re.split(r"\s*[\*\-\|]+\s*", name)[0]
         return cleaned.strip()
 
-    def _clean_description(self, description: str) -> str:
+    def _clean_description(
+        self, description: str, animal_id: str = "unknown"
+    ) -> str:
         """Clean up description text."""
         if not description:
             return ""
 
-        # Decode HTML entities
+        # Decode HTML entities first, so mojibake that arrived entity-encoded
+        # (’) is repairable too.
         text = html.unescape(description)
+        # A description may combine paragraphs copied from systems with
+        # different encodings. Treat natural line boundaries independently so
+        # ftfy can assess each paragraph on its own.
+        text = "".join(
+            self._repair_mojibake(line, "description", animal_id)
+            for line in text.splitlines(keepends=True)
+        )
 
         # Remove   and normalize whitespace
         text = text.replace(" ", " ")

requirements.txt

@@ -12,6 +12,7 @@ configparser==3.8.1
 decorator
 EasyProcess==1.1
 emoji==1.7.0
+ftfy==6.3.1
 future==1.0.0
 googleapis-common-protos==1.72.0
 grpcio==1.78.0

social_posters/mastodon.py

@@ -1,3 +1,16 @@
+"""Mastodon posting has two quirks worth keeping in mind.
+
+Mastodon treats every reply in a caption thread as an individual status, so
+the root post and each "comment" have distinct IDs and pages. Fetching a whole
+pet post therefore requires associating the separate reply statuses with their
+root, but their different IDs make that relationship possible to reconstruct.
+
+Text also gets one final, non-blocking mojibake sanity check immediately before
+each status is published. All repairs should already have happened upstream;
+if suspicious encoding remains, we log it for investigation and still post the
+status.
+"""
+
 from __future__ import annotations
 
 import logging
@@ -8,6 +21,7 @@
 from urllib.parse import urlparse
 
 import requests
+from ftfy.badness import is_bad
 from mastodon import Mastodon
 
 from abstractions import AdoptablePet, Post, PostResult, SocialPoster
@@ -193,6 +207,7 @@ def _post_thread(
         replies: list[str],
         media_id: str,
     ) -> Iterator[tuple[str, int | None, dict]]:
+        self._log_suspicious_text(main_caption, "root", None)
         status = session.status_post(
             main_caption,
             media_ids=[media_id],
@@ -202,12 +217,29 @@ def _post_thread(
         root_status_id = status["id"]
 
         for reply_number, reply_text in enumerate(replies, start=1):
+            self._log_suspicious_text(reply_text, "reply", reply_number)
             reply_status = session.status_post(
                 reply_text,
                 in_reply_to_id=root_status_id,
             )
             yield "reply", reply_number, reply_status
 
+    @staticmethod
+    def _log_suspicious_text(
+        text: str,
+        post_kind: str,
+        reply_number: int | None,
+    ) -> None:
+        """Warn about likely encoding damage without preventing publication."""
+        if is_bad(text):
+            logger.warning(
+                "Suspicious text at Mastodon status_post boundary: "
+                "kind=%s reply_number=%s text=%s",
+                post_kind,
+                reply_number,
+                pprint.pformat(text),
+            )
+
     def _format_caption_thread(self, post: Post) -> tuple[str, list[str]]:
         caption_text = post.text.strip()
         tag_suffix = self._format_tag_suffix(post.tags)