From dc3d48caa535ac12109d5130730933d74ea14329 Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Tue, 4 Aug 2026 20:43:07 -0400 Subject: [PATCH 01/13] Diagnose RescueGroups response encoding --- adoption_sources/rescue_groups.py | 23 +++++++- tests/test_rescue_groups.py | 89 ++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 883a64f..6b3d1c3 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -5,6 +5,7 @@ """ import html +import json import logging import pprint import os @@ -157,7 +158,27 @@ def fetch_pets(self) -> Iterator[AdoptablePet]: response = session.post(url, json=payload, headers=headers, timeout=30) response.raise_for_status() - body = response.json() + # JSON is UTF-8 by default. Decode the response bytes explicitly rather + # than relying on an HTTP charset declaration that could be incorrect. + body = json.loads(response.content.decode("utf-8")) + logger.debug( + "RescueGroups response decoding: Content-Type=%r requests_encoding=%r", + response.headers.get("Content-Type"), + response.encoding, + ) + if logger.isEnabledFor(logging.DEBUG): + try: + requests_body = response.json() + except ValueError as exc: + logger.warning( + "RescueGroups response.json() failed while UTF-8 parsing succeeded: %s", + exc, + ) + else: + logger.debug( + "RescueGroups response.json() matches explicit UTF-8 parse: %s", + requests_body == body, + ) data = body.get("data", []) logger.info(f"Received {len(data)} pets from RescueGroups") diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 4642dbe..7857d63 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -3,6 +3,8 @@ from pathlib import Path from unittest.mock import MagicMock, patch +from requests import Response + from adoption_sources.rescue_groups import ( SourceRescueGroups, _build_species_filters, @@ -157,7 +159,7 @@ def test_posts_single_multi_species_request(self, mock_session_factory): mock_session = MagicMock() mock_session_factory.return_value = mock_session mock_response = MagicMock() - mock_response.json.return_value = {"data": [], "included": []} + mock_response.content = b'{"data": [], "included": []}' mock_session.post.return_value = mock_response source = SourceRescueGroups(api_key="dummy") @@ -193,6 +195,91 @@ def test_missing_api_key_raises(self): list(source.fetch_pets()) +class ResponseJsonDecodingTests(unittest.TestCase): + def test_utf8_json_without_a_charset_is_decoded_correctly(self): + """Requests detects UTF-8 for ``application/vnd.api+json`` responses.""" + description = "Adoption center hours: 1:00PM – 6:00PM" + response = Response() + response._content = json.dumps( + {"data": [{"attributes": {"descriptionText": description}}]}, + ensure_ascii=False, + ).encode("utf-8") + + parsed_description = response.json()["data"][0]["attributes"][ + "descriptionText" + ] + + self.assertEqual(parsed_description, description) + + def test_utf8_json_is_mojibaked_when_response_declares_latin_1(self): + """Reproduce the ``–`` -> ``–`` corruption seen in production. + + ``SourceRescueGroups.fetch_pets`` currently calls ``response.json()``. + Requests uses ``response.encoding`` when the server declares one, so an + incorrect Latin-1 declaration decodes otherwise valid UTF-8 JSON into + the same mojibake recorded in the GOOBER post. + """ + description = "Adoption center hours: 1:00PM – 6:00PM" + response = Response() + response.encoding = "iso-8859-1" + response._content = json.dumps( + {"data": [{"attributes": {"descriptionText": description}}]}, + ensure_ascii=False, + ).encode("utf-8") + + parsed_description = response.json()["data"][0]["attributes"][ + "descriptionText" + ] + + self.assertEqual( + parsed_description, + "Adoption center hours: 1:00PM – 6:00PM", + ) + self.assertEqual( + response.content.decode("utf-8"), + json.dumps( + {"data": [{"attributes": {"descriptionText": description}}]}, + ensure_ascii=False, + ), + ) + + +class FetchPetsUtf8DecodingTests(unittest.TestCase): + @patch("adoption_sources.rescue_groups._session_with_retries") + def test_decodes_utf8_response_bytes_despite_wrong_charset(self, mock_session_factory): + """Keep RescueGroups descriptions readable when its charset is wrong.""" + description = "Adoption center hours: 1:00PM – 6:00PM" + body = { + "data": [_make_animal(descriptionText=description)], + "included": [ + { + "type": "orgs", + "id": "org1", + "attributes": _make_org(url="https://example.com/adopt"), + }, + { + "type": "species", + "id": "8", + "attributes": {"plural": "dogs"}, + }, + ], + } + response = Response() + response.status_code = 200 + response.encoding = "iso-8859-1" + response._content = json.dumps(body, ensure_ascii=False).encode("utf-8") + + mock_session = MagicMock() + mock_session.post.return_value = response + mock_session_factory.return_value = mock_session + + pets = list(SourceRescueGroups(api_key="dummy").fetch_pets()) + + self.assertEqual(len(pets), 1) + self.assertEqual(pets[0].description, description) + self.assertNotIn("–", pets[0].description) + + class RealCaptureParsingTests(unittest.TestCase): """Parse the real API capture end-to-end, as a guard against drift between our parsing and what the live API actually returns.""" From 4feee49c35d975f8f601348425965b1a6e61f63c Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:12:32 -0400 Subject: [PATCH 02/13] temporarily remove debug dev --- .github/workflows/dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 7a2082b..3abd51b 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -68,7 +68,7 @@ jobs: APP_ENV: dev run: | #In order to create posts on the test accounts remove the --debugposters debug flag - python ./main.py --debugsources --debugposters + python ./main.py - name: Upload database artifact uses: actions/upload-artifact@v7 From 673a2131f90e3eb545845f6d5364ce5c5e06cecb Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:17:19 -0700 Subject: [PATCH 03/13] delete poc tests and add repair --- adoption_sources/rescue_groups.py | 95 +++++++++++++++----- tests/test_rescue_groups.py | 143 ++++++++++++------------------ 2 files changed, 126 insertions(+), 112 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 6b3d1c3..970c804 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -5,7 +5,6 @@ """ import html -import json import logging import pprint import os @@ -40,6 +39,59 @@ RETRY_TOTAL = 4 RETRY_BACKOFF_FACTOR = 1 +# Common leading characters produced when UTF-8 bytes are decoded as a +# single-byte encoding. C1 control characters are another strong signal. +MOJIBAKE_MARKERS = frozenset(("Â", "Ã", "â", "ï", "ð")) +MOJIBAKE_ENCODINGS = ("cp1252", "latin-1") + + +def _mojibake_score(text: str) -> int: + """Count characters that strongly suggest UTF-8 mojibake.""" + return sum( + 2 if "\x80" <= character <= "\x9f" else 1 + for character in text + if character in MOJIBAKE_MARKERS or "\x80" <= character <= "\x9f" + ) + + +def _repair_mojibake(text: str) -> tuple[str, tuple[str, ...]]: + """Reverse likely UTF-8 mojibake without touching valid Unicode text. + + RescueGroups sometimes returns values that were corrupted before they + reached its API. Repair only whitespace-delimited fragments with strong + mojibake signals, and only accept a reversible decoding that reduces those + signals. Processing fragments separately preserves unrelated characters + such as emoji or non-Latin scripts in the same description. + """ + repaired_encodings: list[str] = [] + fragments = re.split(r"([ \t\r\n\f\v]+)", text) + + for index, fragment in enumerate(fragments): + original_score = _mojibake_score(fragment) + if not original_score: + continue + + best_fragment = fragment + best_score = original_score + best_encoding = None + for encoding in MOJIBAKE_ENCODINGS: + try: + candidate = fragment.encode(encoding).decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + continue + + candidate_score = _mojibake_score(candidate) + if candidate_score < best_score: + best_fragment = candidate + best_score = candidate_score + best_encoding = encoding + + if best_encoding is not None: + fragments[index] = best_fragment + repaired_encodings.append(best_encoding) + + return "".join(fragments), tuple(dict.fromkeys(repaired_encodings)) + def _session_with_retries() -> requests.Session: """Build a requests Session that retries transient errors with backoff.""" @@ -158,27 +210,7 @@ def fetch_pets(self) -> Iterator[AdoptablePet]: response = session.post(url, json=payload, headers=headers, timeout=30) response.raise_for_status() - # JSON is UTF-8 by default. Decode the response bytes explicitly rather - # than relying on an HTTP charset declaration that could be incorrect. - body = json.loads(response.content.decode("utf-8")) - logger.debug( - "RescueGroups response decoding: Content-Type=%r requests_encoding=%r", - response.headers.get("Content-Type"), - response.encoding, - ) - if logger.isEnabledFor(logging.DEBUG): - try: - requests_body = response.json() - except ValueError as exc: - logger.warning( - "RescueGroups response.json() failed while UTF-8 parsing succeeded: %s", - exc, - ) - else: - logger.debug( - "RescueGroups response.json() matches explicit UTF-8 parse: %s", - requests_body == body, - ) + body = response.json() data = body.get("data", []) logger.info(f"Received {len(data)} pets from RescueGroups") @@ -241,7 +273,9 @@ def _parse_animal( breed = attrs.get("breedString", attrs.get("breedPrimary", "Mixed")) # Clean up description (use text version, not HTML) - description = self._clean_description(attrs.get("descriptionText", "")) + description = self._clean_description( + attrs.get("descriptionText", ""), animal_id=animal_id + ) # Get adoption_url org_id = ( @@ -314,7 +348,9 @@ def _clean_name(self, name: str) -> str: cleaned = re.split(r"\s*[\*\-\|]+\s*", name)[0] return cleaned.strip() - def _clean_description(self, description: str) -> str: + def _clean_description( + self, description: str, animal_id: str = "unknown" + ) -> str: """Clean up description text.""" if not description: return "" @@ -322,6 +358,17 @@ def _clean_description(self, description: str) -> str: # Decode HTML entities text = html.unescape(description) + # Repair text that was mojibaked before RescueGroups serialized its + # JSON response. The HTTP response itself is already valid UTF-8. + text, repaired_encodings = _repair_mojibake(text) + if repaired_encodings: + logger.info( + "Repaired mojibake in RescueGroups description for animal %s " + "using %s", + animal_id, + ", ".join(repaired_encodings), + ) + # Remove   and normalize whitespace text = text.replace(" ", " ") text = re.sub(r"\s+", " ", text) diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 7857d63..0ae0825 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -3,8 +3,6 @@ from pathlib import Path from unittest.mock import MagicMock, patch -from requests import Response - from adoption_sources.rescue_groups import ( SourceRescueGroups, _build_species_filters, @@ -153,13 +151,67 @@ def test_real_pet_name_is_not_placeholder(self): self.assertFalse(self.source._is_placeholder_name("Buddy")) +class DescriptionCleaningTests(unittest.TestCase): + def setUp(self): + self.source = SourceRescueGroups(api_key="dummy") + + def test_repairs_latin_1_mojibake_observed_in_api_response(self): + description = "Adoption hours: 1:00PM â\x80\x93 6:00PM" + animal = _make_animal(descriptionText=description) + + with self.assertLogs( + "adoption_sources.rescue_groups", level="INFO" + ) as captured: + pet = self.source._parse_animal( + animal, + {"org1": _make_org(url="https://example.com/adopt")}, + _make_species_by_id(), + ) + + self.assertEqual(pet.description, "Adoption hours: 1:00PM – 6:00PM") + self.assertEqual( + captured.output, + [ + "INFO:adoption_sources.rescue_groups:Repaired mojibake in " + "RescueGroups description for animal 12345 using latin-1" + ], + ) + + def test_repairs_windows_1252_mojibake_after_html_unescape(self): + description = "I’m ready for a home." + + cleaned = self.source._clean_description(description) + + self.assertEqual(cleaned, "I’m ready for a home.") + + def test_preserves_valid_unicode(self): + descriptions = ( + "José loves café visits – and naps.", + "A happy dog 😊", + "猫はとても元気です。", + ) + + for description in descriptions: + with self.subTest(description=description): + self.assertEqual( + self.source._clean_description(description), description + ) + + def test_repairs_mojibake_beside_unrelated_unicode(self): + description = "José says hello 😊 I’m friendly." + + cleaned = self.source._clean_description(description) + + self.assertEqual(cleaned, "José says hello 😊 I’m friendly.") + + class FetchPetsRequestTests(unittest.TestCase): @patch("adoption_sources.rescue_groups._session_with_retries") def test_posts_single_multi_species_request(self, mock_session_factory): mock_session = MagicMock() mock_session_factory.return_value = mock_session mock_response = MagicMock() - mock_response.content = b'{"data": [], "included": []}' + mock_response.json.return_value = {"data": [], "included": []} mock_session.post.return_value = mock_response source = SourceRescueGroups(api_key="dummy") @@ -195,91 +247,6 @@ def test_missing_api_key_raises(self): list(source.fetch_pets()) -class ResponseJsonDecodingTests(unittest.TestCase): - def test_utf8_json_without_a_charset_is_decoded_correctly(self): - """Requests detects UTF-8 for ``application/vnd.api+json`` responses.""" - description = "Adoption center hours: 1:00PM – 6:00PM" - response = Response() - response._content = json.dumps( - {"data": [{"attributes": {"descriptionText": description}}]}, - ensure_ascii=False, - ).encode("utf-8") - - parsed_description = response.json()["data"][0]["attributes"][ - "descriptionText" - ] - - self.assertEqual(parsed_description, description) - - def test_utf8_json_is_mojibaked_when_response_declares_latin_1(self): - """Reproduce the ``–`` -> ``–`` corruption seen in production. - - ``SourceRescueGroups.fetch_pets`` currently calls ``response.json()``. - Requests uses ``response.encoding`` when the server declares one, so an - incorrect Latin-1 declaration decodes otherwise valid UTF-8 JSON into - the same mojibake recorded in the GOOBER post. - """ - description = "Adoption center hours: 1:00PM – 6:00PM" - response = Response() - response.encoding = "iso-8859-1" - response._content = json.dumps( - {"data": [{"attributes": {"descriptionText": description}}]}, - ensure_ascii=False, - ).encode("utf-8") - - parsed_description = response.json()["data"][0]["attributes"][ - "descriptionText" - ] - - self.assertEqual( - parsed_description, - "Adoption center hours: 1:00PM – 6:00PM", - ) - self.assertEqual( - response.content.decode("utf-8"), - json.dumps( - {"data": [{"attributes": {"descriptionText": description}}]}, - ensure_ascii=False, - ), - ) - - -class FetchPetsUtf8DecodingTests(unittest.TestCase): - @patch("adoption_sources.rescue_groups._session_with_retries") - def test_decodes_utf8_response_bytes_despite_wrong_charset(self, mock_session_factory): - """Keep RescueGroups descriptions readable when its charset is wrong.""" - description = "Adoption center hours: 1:00PM – 6:00PM" - body = { - "data": [_make_animal(descriptionText=description)], - "included": [ - { - "type": "orgs", - "id": "org1", - "attributes": _make_org(url="https://example.com/adopt"), - }, - { - "type": "species", - "id": "8", - "attributes": {"plural": "dogs"}, - }, - ], - } - response = Response() - response.status_code = 200 - response.encoding = "iso-8859-1" - response._content = json.dumps(body, ensure_ascii=False).encode("utf-8") - - mock_session = MagicMock() - mock_session.post.return_value = response - mock_session_factory.return_value = mock_session - - pets = list(SourceRescueGroups(api_key="dummy").fetch_pets()) - - self.assertEqual(len(pets), 1) - self.assertEqual(pets[0].description, description) - self.assertNotIn("–", pets[0].description) - - class RealCaptureParsingTests(unittest.TestCase): """Parse the real API capture end-to-end, as a guard against drift between our parsing and what the live API actually returns.""" From 39c7799449c6c69b85fbaca279d4484579384e25 Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:26:57 -0700 Subject: [PATCH 04/13] move out of utils --- adoption_sources/rescue_groups.py | 64 +++---------------------------- requirements.txt | 1 + tests/test_rescue_groups.py | 2 +- 3 files changed, 7 insertions(+), 60 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 970c804..9597008 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -13,6 +13,7 @@ from typing import Iterator import requests +from ftfy import fix_encoding from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry @@ -39,60 +40,6 @@ RETRY_TOTAL = 4 RETRY_BACKOFF_FACTOR = 1 -# Common leading characters produced when UTF-8 bytes are decoded as a -# single-byte encoding. C1 control characters are another strong signal. -MOJIBAKE_MARKERS = frozenset(("Â", "Ã", "â", "ï", "ð")) -MOJIBAKE_ENCODINGS = ("cp1252", "latin-1") - - -def _mojibake_score(text: str) -> int: - """Count characters that strongly suggest UTF-8 mojibake.""" - return sum( - 2 if "\x80" <= character <= "\x9f" else 1 - for character in text - if character in MOJIBAKE_MARKERS or "\x80" <= character <= "\x9f" - ) - - -def _repair_mojibake(text: str) -> tuple[str, tuple[str, ...]]: - """Reverse likely UTF-8 mojibake without touching valid Unicode text. - - RescueGroups sometimes returns values that were corrupted before they - reached its API. Repair only whitespace-delimited fragments with strong - mojibake signals, and only accept a reversible decoding that reduces those - signals. Processing fragments separately preserves unrelated characters - such as emoji or non-Latin scripts in the same description. - """ - repaired_encodings: list[str] = [] - fragments = re.split(r"([ \t\r\n\f\v]+)", text) - - for index, fragment in enumerate(fragments): - original_score = _mojibake_score(fragment) - if not original_score: - continue - - best_fragment = fragment - best_score = original_score - best_encoding = None - for encoding in MOJIBAKE_ENCODINGS: - try: - candidate = fragment.encode(encoding).decode("utf-8") - except (UnicodeEncodeError, UnicodeDecodeError): - continue - - candidate_score = _mojibake_score(candidate) - if candidate_score < best_score: - best_fragment = candidate - best_score = candidate_score - best_encoding = encoding - - if best_encoding is not None: - fragments[index] = best_fragment - repaired_encodings.append(best_encoding) - - return "".join(fragments), tuple(dict.fromkeys(repaired_encodings)) - - def _session_with_retries() -> requests.Session: """Build a requests Session that retries transient errors with backoff.""" retry = Retry( @@ -360,14 +307,13 @@ def _clean_description( # Repair text that was mojibaked before RescueGroups serialized its # JSON response. The HTTP response itself is already valid UTF-8. - text, repaired_encodings = _repair_mojibake(text) - if repaired_encodings: + repaired_text = fix_encoding(text) + if repaired_text != text: logger.info( - "Repaired mojibake in RescueGroups description for animal %s " - "using %s", + "Repaired mojibake in RescueGroups description for animal %s", animal_id, - ", ".join(repaired_encodings), ) + text = repaired_text # Remove   and normalize whitespace text = text.replace(" ", " ") diff --git a/requirements.txt b/requirements.txt index 25519fc..5fcda3c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ configparser==3.8.1 decorator EasyProcess==1.1 emoji==1.7.0 +ftfy==6.3.1 future==1.0.0 googleapis-common-protos==1.72.0 grpcio==1.78.0 diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 0ae0825..7a2e486 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -173,7 +173,7 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): captured.output, [ "INFO:adoption_sources.rescue_groups:Repaired mojibake in " - "RescueGroups description for animal 12345 using latin-1" + "RescueGroups description for animal 12345" ], ) From d9749c20a5d39e4a3e54b33dd3afd48df5fb54ce Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Sun, 9 Aug 2026 16:49:14 -0700 Subject: [PATCH 05/13] more tests for checking if repair would fix bad encoded/decoded words and not disrupt normal words --- adoption_sources/rescue_groups.py | 54 +++-- tests/test_rescue_groups.py | 386 ++++++++++++++++++++++++++++-- 2 files changed, 407 insertions(+), 33 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 9597008..55e0ea1 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -197,7 +197,7 @@ def _parse_animal( animal_id = animal.get("id", "") # Extract and clean the name - name = self._clean_name(attrs.get("name", "Unknown")) + name = self._clean_name(attrs.get("name", "Unknown"), animal_id=animal_id) # Determine species from the included species relationship species_id = ( @@ -217,7 +217,11 @@ def _parse_animal( species = SPECIES_SINGULAR[normalized_plural] # Get breed info - breed = attrs.get("breedString", attrs.get("breedPrimary", "Mixed")) + breed = self._repair_mojibake( + attrs.get("breedString", attrs.get("breedPrimary", "Mixed")), + "breed", + animal_id, + ) # Clean up description (use text version, not HTML) description = self._clean_description( @@ -258,7 +262,11 @@ def _parse_animal( image_url = self._get_image_url(attrs) # Location of the adoption org - location = f"{org_attrs.get('city')}, {org_attrs.get('state')}" + location = self._repair_mojibake( + f"{org_attrs.get('city')}, {org_attrs.get('state')}", + "location", + animal_id, + ) return AdoptablePet( @@ -282,7 +290,27 @@ def _parse_animal( def _is_placeholder_name(self, name: str) -> bool: return name.lower() in PLACEHOLDER_NAMES - def _clean_name(self, name: str) -> str: + def _repair_mojibake(self, text: str, field: str, animal_id: str) -> str: + """Repair text that was mojibaked before RescueGroups serialized it. + + The HTTP response itself is already valid UTF-8 — the corruption + happens upstream of the API, so repairing on our side is the only fix + available to us. ``ftfy`` is conservative: text that isn't recognisable + mojibake is returned untouched. + """ + if not text: + return text + + repaired = fix_encoding(text) + if repaired != text: + logger.info( + "Repaired mojibake in RescueGroups %s for animal %s", + field, + animal_id, + ) + return repaired + + def _clean_name(self, name: str, animal_id: str = "unknown") -> str: """ Clean up pet name by removing promotional text. @@ -290,6 +318,10 @@ def _clean_name(self, name: str) -> str: "Doli ***Home for the Holidays 1/2 price!" -> "Doli" "Kathy" -> "Kathy" """ + # Repair before splitting: ftfy reads the whole string to decide, so a + # mojibaked promotional suffix is extra evidence for fixing the name. + name = self._repair_mojibake(name, "name", animal_id) + # Remove common promotional suffixes # Split on common delimiters and take the first part cleaned = re.split(r"\s*[\*\-\|]+\s*", name)[0] @@ -302,18 +334,10 @@ def _clean_description( if not description: return "" - # Decode HTML entities + # Decode HTML entities first, so mojibake that arrived entity-encoded + # (’) is repairable too. text = html.unescape(description) - - # Repair text that was mojibaked before RescueGroups serialized its - # JSON response. The HTTP response itself is already valid UTF-8. - repaired_text = fix_encoding(text) - if repaired_text != text: - logger.info( - "Repaired mojibake in RescueGroups description for animal %s", - animal_id, - ) - text = repaired_text + text = self._repair_mojibake(text, "description", animal_id) # Remove   and normalize whitespace text = text.replace(" ", " ") diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 7a2e486..448d96e 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -3,6 +3,8 @@ from pathlib import Path from unittest.mock import MagicMock, patch +from requests import Response + from adoption_sources.rescue_groups import ( SourceRescueGroups, _build_species_filters, @@ -151,10 +153,182 @@ def test_real_pet_name_is_not_placeholder(self): self.assertFalse(self.source._is_placeholder_name("Buddy")) -class DescriptionCleaningTests(unittest.TestCase): +def _mojibake(text: str, encoding: str = "cp1252") -> str: + """Corrupt ``text`` the way a mis-decoding upstream system does. + + UTF-8 bytes read back one at a time as a single-byte codepage. ``cp1252`` + leaves five bytes undefined (0x81, 0x8D, 0x8F, 0x90, 0x9D) and real systems + fall through to Latin-1 for those, which is why live descriptions mix tidy + ``’`` runs with raw C1 control characters. Pass ``encoding="latin-1"`` + for the pure Latin-1 flavour we captured in the GOOBER post. + """ + decoded = [] + for byte in text.encode("utf-8"): + chunk = bytes([byte]) + try: + decoded.append(chunk.decode(encoding)) + except UnicodeDecodeError: + decoded.append(chunk.decode("latin-1")) + return "".join(decoded) + + +# A description in the shape shelters actually write, holding one example of +# every mojibake class we have seen or can expect: smart punctuation, Latin-1 +# accents, symbols, and astral-plane emoji. Written as lines because +# ``_clean_description`` collapses whitespace, so the newlines become spaces. +LEGIT_DESCRIPTION_LINES = ( + "Meet Goober! 🐶 He’s a 2-year-old Lab mix and he weighs 52 lbs.", + "Adoption hours are 1:00PM – 6:00PM, Tuesday–Sunday; the fee is $150.", + "His foster, José, calls him “the best boy” — crate-trained, " + "house-trained… and 100% food-motivated.", + "• Neutered: yes • Good with kids: yes • Cats: a slow introduction, " + "and keep the house at 70°F or cooler, please 😊", + "Questions? Email adopt@example.org, or apply at " + "https://example.com/adopt/jos%C3%A9 — se habla español " + "(Doña Müller answers on weekends).", + "Sponsored by PetSmart™ & the Ångström Family Fund.", +) +LEGIT_DESCRIPTION = "\n".join(LEGIT_DESCRIPTION_LINES) +EXPECTED_DESCRIPTION = " ".join(LEGIT_DESCRIPTION_LINES) + +# Each entry is a span of EXPECTED_DESCRIPTION that only survives if that +# mojibake class was repaired. Named so a failure says which class broke. +MOJIBAKE_CLASSES = { + "right single quote U+2019": "He’s a 2-year-old", + "curly double quotes U+201C/U+201D": "“the best boy”", + "en dash U+2013": "1:00PM – 6:00PM", + "em dash U+2014": "— crate-trained", + "ellipsis U+2026": "house-trained… and", + "bullet U+2022": "• Neutered", + "e-acute U+00E9": "José", + "n-tilde U+00F1": "español", + "u-umlaut U+00FC": "Müller", + "A-ring U+00C5": "Ångström", + "degree sign U+00B0": "70°F", + "trademark U+2122": "PetSmart™", + "astral emoji U+1F436": "Goober! 🐶", + "astral emoji U+1F60A": "please 😊", +} + +# Spans the repair has no business touching. Prices, times, emails, and +# percent-encoded URLs are the ones that would quietly break a live post. +UNTOUCHED_SPANS = ( + "2-year-old Lab mix", + "52 lbs", + "1:00PM", + "the fee is $150", + "adopt@example.org", + "https://example.com/adopt/jos%C3%A9", + "PetSmart™ & the", + "100% food-motivated", +) + +# Nothing repaired should still carry a mojibake signature. U+009D is the C1 +# control that Latin-1 leaves behind where cp1252 would have a closing quote. +RESIDUAL_MOJIBAKE_MARKERS = ("â€", "Ã", "Â", "ð", "") + + +class DescriptionMojibakeRepairTests(unittest.TestCase): + """The repair fixes the corruption, across every class we expect.""" + def setUp(self): self.source = SourceRescueGroups(api_key="dummy") + def _assert_fully_repaired(self, corrupted: str) -> None: + cleaned = self.source._clean_description(corrupted) + + for label, span in MOJIBAKE_CLASSES.items(): + with self.subTest(repaired=label): + self.assertIn(span, cleaned) + for span in UNTOUCHED_SPANS: + with self.subTest(untouched=span): + self.assertIn(span, cleaned) + for marker in RESIDUAL_MOJIBAKE_MARKERS: + with self.subTest(residual=marker): + self.assertNotIn(marker, cleaned) + self.assertEqual(cleaned, EXPECTED_DESCRIPTION) + + def test_marker_tables_describe_the_expected_output(self): + """Guard the tables above from drifting out of the description.""" + for label, span in MOJIBAKE_CLASSES.items(): + with self.subTest(repaired=label): + self.assertIn(span, EXPECTED_DESCRIPTION) + for span in UNTOUCHED_SPANS: + with self.subTest(untouched=span): + self.assertIn(span, EXPECTED_DESCRIPTION) + + def test_repairs_a_realistic_description_mangled_as_windows_1252(self): + self._assert_fully_repaired(_mojibake(LEGIT_DESCRIPTION, "cp1252")) + + def test_repairs_a_realistic_description_mangled_as_latin_1(self): + self._assert_fully_repaired(_mojibake(LEGIT_DESCRIPTION, "latin-1")) + + def test_repairs_the_description_on_the_way_out_of_fetch_pets(self): + """End to end: the corruption is already baked into valid UTF-8 JSON. + + This is the upstream shape we diagnosed — RescueGroups serializes text + that was mojibaked before it reached them, so the HTTP response decodes + cleanly and only the characters are wrong. + """ + body = { + "data": [_make_animal(descriptionText=_mojibake(LEGIT_DESCRIPTION))], + "included": [ + { + "type": "orgs", + "id": "org1", + "attributes": _make_org(url="https://example.com/adopt"), + }, + {"type": "species", "id": "8", "attributes": {"plural": "dogs"}}, + ], + } + response = Response() + response.status_code = 200 + response._content = json.dumps(body, ensure_ascii=False).encode("utf-8") + mock_session = MagicMock() + mock_session.post.return_value = response + + with patch( + "adoption_sources.rescue_groups._session_with_retries", + return_value=mock_session, + ): + pets = list(self.source.fetch_pets()) + + self.assertEqual(len(pets), 1) + self.assertEqual(pets[0].description, EXPECTED_DESCRIPTION) + + def test_repairs_only_the_corrupted_span(self): + """Shelters paste one mangled paragraph into otherwise clean text.""" + clean = "Meet Goober! 🐶 His foster José says he’s “the best boy”." + corrupted = _mojibake( + "Adoption hours: 1:00PM – 6:00PM. Doña Müller answers. 😊" + ) + + cleaned = self.source._clean_description(f"{clean} {corrupted}") + + self.assertEqual( + cleaned, + f"{clean} Adoption hours: 1:00PM – 6:00PM. Doña Müller answers. 😊", + ) + + def test_repairs_doubly_encoded_text(self): + self.assertEqual( + self.source._clean_description(_mojibake(_mojibake("José – 😊"))), + "José – 😊", + ) + + def test_repairs_non_breaking_space_before_whitespace_is_collapsed(self): + """``Â\\xa0`` must be repaired before whitespace normalization runs.""" + self.assertEqual( + self.source._clean_description(_mojibake("Goober weighs 52 lbs.")), + "Goober weighs 52 lbs.", + ) + + def test_repairs_mojibake_that_arrives_as_html_entities(self): + self.assertEqual( + self.source._clean_description("I’m ready for a home."), + "I’m ready for a home.", + ) + def test_repairs_latin_1_mojibake_observed_in_api_response(self): description = "Adoption hours: 1:00PM â\x80\x93 6:00PM" animal = _make_animal(descriptionText=description) @@ -177,32 +351,208 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): ], ) - def test_repairs_windows_1252_mojibake_after_html_unescape(self): - description = "I’m ready for a home." - - cleaned = self.source._clean_description(description) - self.assertEqual(cleaned, "I’m ready for a home.") +class DescriptionPreservationTests(unittest.TestCase): + """The repair leaves text alone when there is nothing to repair. + + These are the tests that fail if ``fix_encoding`` ever starts over-reaching: + they all pass against the pre-repair code, so only a regression breaks them. + """ + + # Text that is already correct, including the shapes most likely to be + # mistaken for mojibake: stray Latin-1 letters, percent-encoded URLs, and + # the cp1252 symbols (™ ® °) that mojibake decodes *into*. + CLEAN_DESCRIPTIONS = { + "accented names": "Foster José and Doña Müller say she is très câlin.", + "correct smart punctuation": "He’s “the best boy” — really… 100% good.", + "emoji": "Adopt me! 🐶🐱😊🎉 #AdoptDontShop", + "non-latin scripts": "猫はとても元気です。 강아지 귀여워요! Кот очень милый.", + "currency and symbols": "Fee: $150 / €130 / £110 / ¥1,500 / 50¢ / 70°F / ½ cup", + "percent-encoded url": "Apply at https://example.com/adopt/jos%C3%A9?ref=a%E2%80%93b", + "scandinavian names": "Foster coordinator: Åsa Ångström, Malmö.", + "trademarks": "PetSmart™ · Petco® · ©2026 Example Rescue", + "bare ampersands": "Loves A&W root beer & long walks <3", + "plain ascii": "Buddy is a 2-year-old Lab mix. Hours: 1:00PM - 6:00PM.", + "real a-circumflex": "Château, Ângela, and Râ are real words.", + } - def test_preserves_valid_unicode(self): - descriptions = ( - "José loves café visits – and naps.", - "A happy dog 😊", - "猫はとても元気です。", - ) + def setUp(self): + self.source = SourceRescueGroups(api_key="dummy") - for description in descriptions: - with self.subTest(description=description): + def test_leaves_clean_descriptions_byte_for_byte_identical(self): + for label, description in self.CLEAN_DESCRIPTIONS.items(): + with self.subTest(label): self.assertEqual( self.source._clean_description(description), description ) - def test_repairs_mojibake_beside_unrelated_unicode(self): - description = "José says hello 😊 I’m friendly." + def test_leaves_the_full_realistic_description_untouched_and_unlogged(self): + with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + cleaned = self.source._clean_description(LEGIT_DESCRIPTION) + + self.assertEqual(cleaned, EXPECTED_DESCRIPTION) + + def test_repair_is_idempotent(self): + """Re-cleaning already-repaired text must not mangle it further.""" + repaired = self.source._clean_description(_mojibake(LEGIT_DESCRIPTION)) + + self.assertEqual(self.source._clean_description(repaired), repaired) + + def test_known_false_positive_lone_capital_a_tilde(self): + """Documented limitation, not desired behavior. + + An isolated ``Ã`` followed by a space is byte-identical to mojibaked + ``à``, so ftfy repairs it. Nothing in a real pet description has hit + this; the test exists so we notice if ftfy's heuristics shift. + """ + self.assertEqual( + self.source._clean_description("Letters like à and Ê are rare."), + "Letters like à and Ê are rare.", + ) + + +class PetFieldMojibakeRepairTests(unittest.TestCase): + """Every field we publish gets repaired, not just the description. + + A mojibaked name is the most visible failure of the lot — it lands in the + post title — so these cover name, breed, and location alongside it. + """ + + # Shelters really do use accented names, breeds, and cities. + LEGIT_NAME = "Renée ***Home for the Holidays 1/2 price!" + EXPECTED_NAME = "Renée" + LEGIT_BREED = "Bichon Frisé / Coton de Tuléar Mix" + LEGIT_CITY = "Montréal" + EXPECTED_LOCATION = "Montréal, QC" + LEGIT_TEXT = "She’s a sweetheart – really." - cleaned = self.source._clean_description(description) + def setUp(self): + self.source = SourceRescueGroups(api_key="dummy") + + def _animal(self, corrupt: bool): + transform = _mojibake if corrupt else (lambda text: text) + return _make_animal( + name=transform(self.LEGIT_NAME), + breedString=transform(self.LEGIT_BREED), + descriptionText=transform(self.LEGIT_TEXT), + ) + + def _orgs(self, corrupt: bool): + transform = _mojibake if corrupt else (lambda text: text) + return { + "org1": { + "city": transform(self.LEGIT_CITY), + "state": "QC", + "url": "https://example.com/adopt", + } + } + + def _assert_all_fields_repaired(self, pet) -> None: + self.assertEqual(pet.name, self.EXPECTED_NAME) + self.assertEqual(pet.breed, self.LEGIT_BREED) + self.assertEqual(pet.location, self.EXPECTED_LOCATION) + self.assertEqual(pet.description, self.LEGIT_TEXT) + + def test_repairs_name_breed_and_location_through_fetch_pets(self): + body = { + "data": [self._animal(corrupt=True)], + "included": [ + { + "type": "orgs", + "id": "org1", + "attributes": self._orgs(corrupt=True)["org1"], + }, + {"type": "species", "id": "8", "attributes": {"plural": "dogs"}}, + ], + } + response = Response() + response.status_code = 200 + response._content = json.dumps(body, ensure_ascii=False).encode("utf-8") + mock_session = MagicMock() + mock_session.post.return_value = response + + with patch( + "adoption_sources.rescue_groups._session_with_retries", + return_value=mock_session, + ): + pets = list(self.source.fetch_pets()) + + self.assertEqual(len(pets), 1) + self._assert_all_fields_repaired(pets[0]) + + def test_logs_each_repaired_field_by_name(self): + with self.assertLogs( + "adoption_sources.rescue_groups", level="INFO" + ) as captured: + pet = self.source._parse_animal( + self._animal(corrupt=True), + self._orgs(corrupt=True), + _make_species_by_id(), + ) + + self._assert_all_fields_repaired(pet) + self.assertEqual( + sorted(captured.output), + sorted( + f"INFO:adoption_sources.rescue_groups:Repaired mojibake in " + f"RescueGroups {field} for animal 12345" + for field in ("name", "breed", "description", "location") + ), + ) + + def test_leaves_clean_fields_untouched_and_unlogged(self): + with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + pet = self.source._parse_animal( + self._animal(corrupt=False), + self._orgs(corrupt=False), + _make_species_by_id(), + ) + + self._assert_all_fields_repaired(pet) + + def test_repairs_name_before_stripping_the_promotional_suffix(self): + """The suffix is extra context for ftfy, so repair has to come first.""" + self.assertEqual( + self.source._clean_name(_mojibake("Zoë ***Adoption fee waived!")), + "Zoë", + ) + + def test_repairs_name_and_breed_that_only_differ_in_smart_punctuation(self): + self.assertEqual( + self.source._clean_name(_mojibake("Lucky — the “office dog”")), + "Lucky — the “office dog”", + ) + self.assertEqual( + self.source._repair_mojibake( + _mojibake("Chihuahua – Short Coat"), "breed", "12345" + ), + "Chihuahua – Short Coat", + ) + + def test_empty_and_missing_values_pass_through(self): + for value in ("", None): + with self.subTest(value=value): + self.assertEqual( + self.source._repair_mojibake(value, "breed", "12345"), value + ) - self.assertEqual(cleaned, "José says hello 😊 I’m friendly.") + def test_known_limitation_a_ring_needs_corroborating_mojibake(self): + """Documented limitation, not desired behavior. + + ``Ã…`` is the cp1252 form of ``Å``, but it is also plausible real text, + so ftfy leaves it alone unless the string carries other mojibake to + corroborate it. Scandinavian names are where this bites. + """ + self.assertEqual( + self.source._clean_name(_mojibake("Åsa", "cp1252")), "Ã…sa" + ) + # Latin-1 corruption of the same name has no such ambiguity. + self.assertEqual(self.source._clean_name(_mojibake("Åsa", "latin-1")), "Åsa") + # Neither does cp1252 corruption with a second mojibake sequence. + self.assertEqual( + self.source._clean_name(_mojibake("Åsa the Ångström hound")), + "Åsa the Ångström hound", + ) class FetchPetsRequestTests(unittest.TestCase): From 171b54a872c960042e54e20000f99594a2eba604 Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Sun, 9 Aug 2026 17:11:10 -0700 Subject: [PATCH 06/13] trim tests --- adoption_sources/rescue_groups.py | 6 +- tests/test_rescue_groups.py | 167 ++++++------------------------ 2 files changed, 33 insertions(+), 140 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 55e0ea1..ab19acb 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -318,8 +318,10 @@ def _clean_name(self, name: str, animal_id: str = "unknown") -> str: "Doli ***Home for the Holidays 1/2 price!" -> "Doli" "Kathy" -> "Kathy" """ - # Repair before splitting: ftfy reads the whole string to decide, so a - # mojibaked promotional suffix is extra evidence for fixing the name. + # Repair before splitting. ftfy weighs the whole string when a sequence + # is ambiguous (``Ã…`` is both mojibaked ``Å`` and plausible real text), + # so discarding the promotional suffix first can lose the only evidence + # that tips an accented name toward being repaired. name = self._repair_mojibake(name, "name", animal_id) # Remove common promotional suffixes diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 448d96e..70bf8c5 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -191,110 +191,22 @@ def _mojibake(text: str, encoding: str = "cp1252") -> str: LEGIT_DESCRIPTION = "\n".join(LEGIT_DESCRIPTION_LINES) EXPECTED_DESCRIPTION = " ".join(LEGIT_DESCRIPTION_LINES) -# Each entry is a span of EXPECTED_DESCRIPTION that only survives if that -# mojibake class was repaired. Named so a failure says which class broke. -MOJIBAKE_CLASSES = { - "right single quote U+2019": "He’s a 2-year-old", - "curly double quotes U+201C/U+201D": "“the best boy”", - "en dash U+2013": "1:00PM – 6:00PM", - "em dash U+2014": "— crate-trained", - "ellipsis U+2026": "house-trained… and", - "bullet U+2022": "• Neutered", - "e-acute U+00E9": "José", - "n-tilde U+00F1": "español", - "u-umlaut U+00FC": "Müller", - "A-ring U+00C5": "Ångström", - "degree sign U+00B0": "70°F", - "trademark U+2122": "PetSmart™", - "astral emoji U+1F436": "Goober! 🐶", - "astral emoji U+1F60A": "please 😊", -} - -# Spans the repair has no business touching. Prices, times, emails, and -# percent-encoded URLs are the ones that would quietly break a live post. -UNTOUCHED_SPANS = ( - "2-year-old Lab mix", - "52 lbs", - "1:00PM", - "the fee is $150", - "adopt@example.org", - "https://example.com/adopt/jos%C3%A9", - "PetSmart™ & the", - "100% food-motivated", -) - -# Nothing repaired should still carry a mojibake signature. U+009D is the C1 -# control that Latin-1 leaves behind where cp1252 would have a closing quote. -RESIDUAL_MOJIBAKE_MARKERS = ("â€", "Ã", "Â", "ð", "") - class DescriptionMojibakeRepairTests(unittest.TestCase): - """The repair fixes the corruption, across every class we expect.""" - def setUp(self): self.source = SourceRescueGroups(api_key="dummy") - def _assert_fully_repaired(self, corrupted: str) -> None: - cleaned = self.source._clean_description(corrupted) - - for label, span in MOJIBAKE_CLASSES.items(): - with self.subTest(repaired=label): - self.assertIn(span, cleaned) - for span in UNTOUCHED_SPANS: - with self.subTest(untouched=span): - self.assertIn(span, cleaned) - for marker in RESIDUAL_MOJIBAKE_MARKERS: - with self.subTest(residual=marker): - self.assertNotIn(marker, cleaned) - self.assertEqual(cleaned, EXPECTED_DESCRIPTION) - - def test_marker_tables_describe_the_expected_output(self): - """Guard the tables above from drifting out of the description.""" - for label, span in MOJIBAKE_CLASSES.items(): - with self.subTest(repaired=label): - self.assertIn(span, EXPECTED_DESCRIPTION) - for span in UNTOUCHED_SPANS: - with self.subTest(untouched=span): - self.assertIn(span, EXPECTED_DESCRIPTION) - def test_repairs_a_realistic_description_mangled_as_windows_1252(self): - self._assert_fully_repaired(_mojibake(LEGIT_DESCRIPTION, "cp1252")) + self.assertEqual( + self.source._clean_description(_mojibake(LEGIT_DESCRIPTION, "cp1252")), + EXPECTED_DESCRIPTION, + ) def test_repairs_a_realistic_description_mangled_as_latin_1(self): - self._assert_fully_repaired(_mojibake(LEGIT_DESCRIPTION, "latin-1")) - - def test_repairs_the_description_on_the_way_out_of_fetch_pets(self): - """End to end: the corruption is already baked into valid UTF-8 JSON. - - This is the upstream shape we diagnosed — RescueGroups serializes text - that was mojibaked before it reached them, so the HTTP response decodes - cleanly and only the characters are wrong. - """ - body = { - "data": [_make_animal(descriptionText=_mojibake(LEGIT_DESCRIPTION))], - "included": [ - { - "type": "orgs", - "id": "org1", - "attributes": _make_org(url="https://example.com/adopt"), - }, - {"type": "species", "id": "8", "attributes": {"plural": "dogs"}}, - ], - } - response = Response() - response.status_code = 200 - response._content = json.dumps(body, ensure_ascii=False).encode("utf-8") - mock_session = MagicMock() - mock_session.post.return_value = response - - with patch( - "adoption_sources.rescue_groups._session_with_retries", - return_value=mock_session, - ): - pets = list(self.source.fetch_pets()) - - self.assertEqual(len(pets), 1) - self.assertEqual(pets[0].description, EXPECTED_DESCRIPTION) + self.assertEqual( + self.source._clean_description(_mojibake(LEGIT_DESCRIPTION, "latin-1")), + EXPECTED_DESCRIPTION, + ) def test_repairs_only_the_corrupted_span(self): """Shelters paste one mangled paragraph into otherwise clean text.""" @@ -310,12 +222,6 @@ def test_repairs_only_the_corrupted_span(self): f"{clean} Adoption hours: 1:00PM – 6:00PM. Doña Müller answers. 😊", ) - def test_repairs_doubly_encoded_text(self): - self.assertEqual( - self.source._clean_description(_mojibake(_mojibake("José – 😊"))), - "José – 😊", - ) - def test_repairs_non_breaking_space_before_whitespace_is_collapsed(self): """``Â\\xa0`` must be repaired before whitespace normalization runs.""" self.assertEqual( @@ -343,13 +249,8 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): ) self.assertEqual(pet.description, "Adoption hours: 1:00PM – 6:00PM") - self.assertEqual( - captured.output, - [ - "INFO:adoption_sources.rescue_groups:Repaired mojibake in " - "RescueGroups description for animal 12345" - ], - ) + self.assertEqual(len(captured.records), 1) + self.assertEqual(captured.records[0].args, ("description", "12345")) class DescriptionPreservationTests(unittest.TestCase): @@ -359,21 +260,15 @@ class DescriptionPreservationTests(unittest.TestCase): they all pass against the pre-repair code, so only a regression breaks them. """ - # Text that is already correct, including the shapes most likely to be - # mistaken for mojibake: stray Latin-1 letters, percent-encoded URLs, and - # the cp1252 symbols (™ ® °) that mojibake decodes *into*. + # The shapes most likely to be mistaken for mojibake: real Latin-1 letters + # (â is what mojibake starts with), percent-encoded URLs (a repair would + # break the link), and the cp1252 symbols that mojibake decodes *into*. CLEAN_DESCRIPTIONS = { - "accented names": "Foster José and Doña Müller say she is très câlin.", + "real a-circumflex": "Château, Ângela, and Râ are real words.", "correct smart punctuation": "He’s “the best boy” — really… 100% good.", - "emoji": "Adopt me! 🐶🐱😊🎉 #AdoptDontShop", - "non-latin scripts": "猫はとても元気です。 강아지 귀여워요! Кот очень милый.", - "currency and symbols": "Fee: $150 / €130 / £110 / ¥1,500 / 50¢ / 70°F / ½ cup", "percent-encoded url": "Apply at https://example.com/adopt/jos%C3%A9?ref=a%E2%80%93b", - "scandinavian names": "Foster coordinator: Åsa Ångström, Malmö.", - "trademarks": "PetSmart™ · Petco® · ©2026 Example Rescue", - "bare ampersands": "Loves A&W root beer & long walks <3", - "plain ascii": "Buddy is a 2-year-old Lab mix. Hours: 1:00PM - 6:00PM.", - "real a-circumflex": "Château, Ângela, and Râ are real words.", + "trademarks and degrees": "PetSmart™ · Petco® · 70°F · ©2026 Example Rescue", + "non-latin scripts": "猫はとても元気です。 강아지 귀여워요! Кот очень милый.", } def setUp(self): @@ -392,12 +287,6 @@ def test_leaves_the_full_realistic_description_untouched_and_unlogged(self): self.assertEqual(cleaned, EXPECTED_DESCRIPTION) - def test_repair_is_idempotent(self): - """Re-cleaning already-repaired text must not mangle it further.""" - repaired = self.source._clean_description(_mojibake(LEGIT_DESCRIPTION)) - - self.assertEqual(self.source._clean_description(repaired), repaired) - def test_known_false_positive_lone_capital_a_tilde(self): """Documented limitation, not desired behavior. @@ -412,11 +301,9 @@ def test_known_false_positive_lone_capital_a_tilde(self): class PetFieldMojibakeRepairTests(unittest.TestCase): - """Every field we publish gets repaired, not just the description. - - A mojibaked name is the most visible failure of the lot — it lands in the - post title — so these cover name, breed, and location alongside it. - """ + """A mojibaked name is the most visible failure of the lot — it lands in + the post title — so name, breed, and location are covered alongside the + description they were originally left out of.""" # Shelters really do use accented names, breeds, and cities. LEGIT_NAME = "Renée ***Home for the Holidays 1/2 price!" @@ -492,10 +379,9 @@ def test_logs_each_repaired_field_by_name(self): self._assert_all_fields_repaired(pet) self.assertEqual( - sorted(captured.output), + sorted(record.args for record in captured.records), sorted( - f"INFO:adoption_sources.rescue_groups:Repaired mojibake in " - f"RescueGroups {field} for animal 12345" + (field, "12345") for field in ("name", "breed", "description", "location") ), ) @@ -511,10 +397,15 @@ def test_leaves_clean_fields_untouched_and_unlogged(self): self._assert_all_fields_repaired(pet) def test_repairs_name_before_stripping_the_promotional_suffix(self): - """The suffix is extra context for ftfy, so repair has to come first.""" + """``Ã…`` is ambiguous on its own — ftfy only fixes it when the rest of + the string corroborates. Splitting first throws that evidence away, so + this name comes back as ``Ã…sa`` if the repair moves after the split. + """ self.assertEqual( - self.source._clean_name(_mojibake("Zoë ***Adoption fee waived!")), - "Zoë", + self.source._clean_name( + _mojibake("Åsa ***Ångström's littermate, adopt together!") + ), + "Åsa", ) def test_repairs_name_and_breed_that_only_differ_in_smart_punctuation(self): From 532492d0db69a6f8ff1ccb30ebed0c849f37b989 Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Sun, 9 Aug 2026 23:06:35 -0700 Subject: [PATCH 07/13] correct location id in test --- adoption_sources/rescue_groups.py | 16 ++++++++++++---- tests/test_rescue_groups.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index ab19acb..62869c7 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -265,7 +265,8 @@ def _parse_animal( location = self._repair_mojibake( f"{org_attrs.get('city')}, {org_attrs.get('state')}", "location", - animal_id, + org_id or "unknown", + "organization", ) @@ -290,7 +291,13 @@ def _parse_animal( def _is_placeholder_name(self, name: str) -> bool: return name.lower() in PLACEHOLDER_NAMES - def _repair_mojibake(self, text: str, field: str, animal_id: str) -> str: + def _repair_mojibake( + self, + text: str, + field: str, + entity_id: str, + entity_type: str = "animal", + ) -> str: """Repair text that was mojibaked before RescueGroups serialized it. The HTTP response itself is already valid UTF-8 — the corruption @@ -304,9 +311,10 @@ def _repair_mojibake(self, text: str, field: str, animal_id: str) -> str: repaired = fix_encoding(text) if repaired != text: logger.info( - "Repaired mojibake in RescueGroups %s for animal %s", + "Repaired mojibake in RescueGroups %s for %s %s", field, - animal_id, + entity_type, + entity_id, ) return repaired diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 70bf8c5..1e7f6a7 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -250,7 +250,10 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): self.assertEqual(pet.description, "Adoption hours: 1:00PM – 6:00PM") self.assertEqual(len(captured.records), 1) - self.assertEqual(captured.records[0].args, ("description", "12345")) + self.assertEqual( + captured.records[0].args, + ("description", "animal", "12345"), + ) class DescriptionPreservationTests(unittest.TestCase): @@ -381,8 +384,11 @@ def test_logs_each_repaired_field_by_name(self): self.assertEqual( sorted(record.args for record in captured.records), sorted( - (field, "12345") - for field in ("name", "breed", "description", "location") + [ + (field, "animal", "12345") + for field in ("name", "breed", "description") + ] + + [("location", "organization", "org1")] ), ) From 18a9f3d056721aebd9853e2c4c7869e75bd4c0ec Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:14:19 -0700 Subject: [PATCH 08/13] add regression test with real log data --- tests/test_rescue_groups.py | 161 ++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 1e7f6a7..127ee3b 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -9,6 +9,7 @@ SourceRescueGroups, _build_species_filters, ) +from social_posters.mastodon import PosterMastodon def _make_animal(adoption_url=None, species_id="8", **extra_attrs): @@ -256,6 +257,166 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): ) +class DennisMojibakeMastodonRegressionTests(unittest.TestCase): + """Regression coverage for RescueGroups animal 22658169 (DENNIS). + + The record was captured from the live RescueGroups API on 2026-08-10. + It verifies that mojibaked punctuation is repaired before text reaches + Mastodon's ``status_post`` boundary. + """ + + def test_real_dennis_mojibake_is_repaired_before_mastodon_post(self): + raw_animal = { + "type": "animals", + "id": "22658169", + "attributes": { + "ageGroup": "Senior", + "ageString": "7 Years 3 Months", + "birthDate": "2019-04-06T00:00:00Z", + "breedPrimary": "Domestic Short Hair", + "breedPrimaryId": 35, + "breedString": "Domestic Short Hair (medium coat)", + "coatLength": "Medium", + "name": "DENNIS", + "rescueId": "A299137", + "sex": "Male", + "sizeGroup": "Medium", + "pictureThumbnailUrl": ( + "https://cdn.rescuegroups.org/1975/pictures/" + "animals/22658/22658169/103556832.jpg?width=100" + ), + "descriptionText": ( + "MEET DENNIS!- I am in a foster home, please call the " + "Boston shelter to learn more or come meet me in the " + "shelter on Sundays during adoption hours!" + "Dennis is the sweetest guy looking for his new home! " + "Heâ\x80\x99s diabetic, so he needs a little extra daily " + "care, but he will give you more than enough love to make " + "it worth it! Whether itâ\x80\x99s zooming around with " + "his favorite toys, watching his favorite cat tv, lovingly " + "showing you his belly, or snuggling with you, Dennis is " + "sure to make you smile. He is a silly and quirky guy in " + "the best way, and he will keep you laughing with his " + "antics. Heâ\x80\x99ll let you know heâ\x80\x99s coming " + "to sit on your lap with an activation trill, and " + "heâ\x80\x99d be glad to lifeguard you while " + "youâ\x80\x99re showering to protect you from the scary " + "water. Heâ\x80\x99s also good at setting boundaries and " + "will let you know if needs a break from pets. If " + "youâ\x80\x99re looking for a sweet and funny guy to " + "brighten your home, Dennis may be the cat for you! " + "Dennis is diabetic and his diabetes is being managed " + "with twice daily insulin. To offset the cost of medical " + "care, his adoption fee has waived. Dennis is currently " + "up to date on all vaccinations, has been spayed/neutered, " + "microchipped and seen by our vet team." + "We welcome adopters from NH, RI, CT, and NY however, we " + "are unable to facilitate same day adoptions due to state " + "regulated paperwork requirements." + "For more information on this or any other animal currently " + "residing at the Animal Rescue League of Boston please " + "visit us during our adoption center hours: " + "Wednesdays-Sundays from 1:00PM â\x80\x93 6:00PM, " + "Tuesdays by appointment only from " + "1:00PM â\x80\x93 6:00PM, closed Mondays & Holidays." + "For information about our adoption process click here" + ), + }, + "relationships": { + "breeds": {"data": [{"id": "35", "type": "breeds"}]}, + "locations": { + "data": [{"id": "1000001975", "type": "locations"}] + }, + "orgs": {"data": [{"id": "1975", "type": "orgs"}]}, + "species": {"data": [{"id": "3", "type": "species"}]}, + }, + } + orgs_by_id = { + "1975": { + "city": "Boston", + "state": "MA", + "url": "http://www.arlboston.org", + } + } + species_by_id = {"3": {"plural": "Cats"}} + + raw_description = raw_animal["attributes"]["descriptionText"] + self.assertIn("Heâ\x80\x99s diabetic", raw_description) + self.assertIn("1:00PM â\x80\x93 6:00PM", raw_description) + + source = SourceRescueGroups(api_key="dummy") + with self.assertLogs( + "adoption_sources.rescue_groups", level="INFO" + ) as captured_logs: + pet = source._parse_animal(raw_animal, orgs_by_id, species_by_id) + + self.assertIsNotNone(pet) + assert pet is not None + self.assertIn("He’s diabetic", pet.description) + self.assertIn("Whether it’s zooming", pet.description) + self.assertIn("He’ll let you know", pet.description) + self.assertIn("you’re showering", pet.description) + self.assertIn("1:00PM – 6:00PM", pet.description) + self.assertNotIn("â\x80\x99", pet.description) + self.assertNotIn("â\x80\x93", pet.description) + self.assertIn( + "Repaired mojibake in RescueGroups description for animal 22658169", + "\n".join(captured_logs.output), + ) + self.assertEqual(pet.pet_id, "22658169") + self.assertEqual(pet.name, "DENNIS") + self.assertEqual(pet.species, "cat") + self.assertEqual(pet.breed, "Domestic Short Hair (medium coat)") + self.assertEqual(pet.location, "Boston, MA") + + poster = PosterMastodon.__new__(PosterMastodon) + post = poster.format_post(pet) + self.assertIn("He’s diabetic", post.text) + self.assertIn("Whether it’s zooming", post.text) + self.assertIn("1:00PM – 6:00PM", post.text) + self.assertNotIn("â\x80\x99", post.text) + self.assertNotIn("â\x80\x93", post.text) + + session = MagicMock() + + def fake_status_post(text, **kwargs): + call_number = session.status_post.call_count + return { + "id": f"status-{call_number}", + "url": f"https://mastodon.example/@test/status-{call_number}", + } + + session.status_post.side_effect = fake_status_post + poster._session = session + poster._is_available = True + poster._auth_error = None + poster._upload_media = MagicMock(return_value="media-1") + + result = poster.publish(post) + + self.assertTrue(result.success) + self.assertEqual(result.post_id, "status-1") + poster._upload_media.assert_called_once_with(session, post) + self.assertGreaterEqual(session.status_post.call_count, 1) + + mastodon_payloads = [ + call.args[0] for call in session.status_post.call_args_list + ] + all_text_sent_to_mastodon = "\n".join(mastodon_payloads) + self.assertIn("’", all_text_sent_to_mastodon) + self.assertIn("–", all_text_sent_to_mastodon) + self.assertNotIn("â\x80\x99", all_text_sent_to_mastodon) + self.assertNotIn("â\x80\x93", all_text_sent_to_mastodon) + self.assertNotIn("\x80", all_text_sent_to_mastodon) + self.assertNotIn("\x99", all_text_sent_to_mastodon) + self.assertNotIn("\x93", all_text_sent_to_mastodon) + + root_call = session.status_post.call_args_list[0] + self.assertEqual(root_call.kwargs["media_ids"], ["media-1"]) + for reply_call in session.status_post.call_args_list[1:]: + self.assertEqual(reply_call.kwargs["in_reply_to_id"], "status-1") + + class DescriptionPreservationTests(unittest.TestCase): """The repair leaves text alone when there is nothing to repair. From 7dd3182907d133b824eca419081526a4775eebdd Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:23:07 -0700 Subject: [PATCH 09/13] add comment for mastodon specific watchout list and add post-repair sanity check logging --- social_posters/mastodon.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/social_posters/mastodon.py b/social_posters/mastodon.py index 481f6d0..f953495 100644 --- a/social_posters/mastodon.py +++ b/social_posters/mastodon.py @@ -1,3 +1,16 @@ +"""Mastodon posting has two quirks worth keeping in mind. + +Mastodon treats every reply in a caption thread as an individual status, so +the root post and each "comment" have distinct IDs and pages. Fetching a whole +pet post therefore requires associating the separate reply statuses with their +root, but their different IDs make that relationship possible to reconstruct. + +Text also gets one final, non-blocking mojibake sanity check immediately before +each status is published. All repairs should already have happened upstream; +if suspicious encoding remains, we log it for investigation and still post the +status. +""" + from __future__ import annotations import logging @@ -8,6 +21,7 @@ from urllib.parse import urlparse import requests +from ftfy.badness import is_bad from mastodon import Mastodon from abstractions import AdoptablePet, Post, PostResult, SocialPoster @@ -193,6 +207,7 @@ def _post_thread( replies: list[str], media_id: str, ) -> Iterator[tuple[str, int | None, dict]]: + self._log_suspicious_text(main_caption, "root", None) status = session.status_post( main_caption, media_ids=[media_id], @@ -202,12 +217,29 @@ def _post_thread( root_status_id = status["id"] for reply_number, reply_text in enumerate(replies, start=1): + self._log_suspicious_text(reply_text, "reply", reply_number) reply_status = session.status_post( reply_text, in_reply_to_id=root_status_id, ) yield "reply", reply_number, reply_status + @staticmethod + def _log_suspicious_text( + text: str, + post_kind: str, + reply_number: int | None, + ) -> None: + """Warn about likely encoding damage without preventing publication.""" + if is_bad(text): + logger.warning( + "Suspicious text at Mastodon status_post boundary: " + "kind=%s reply_number=%s text=%s", + post_kind, + reply_number, + pprint.pformat(text), + ) + def _format_caption_thread(self, post: Post) -> tuple[str, list[str]]: caption_text = post.text.strip() tag_suffix = self._format_tag_suffix(post.tags) From 70b7ae619d20298350989d1dd75a86b929b255be Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:11:50 -0700 Subject: [PATCH 10/13] add general mojibake repair reasons based on ftfy's explain functionality --- adoption_sources/rescue_groups.py | 12 ++++++++--- tests/test_rescue_groups.py | 35 +++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 62869c7..0648e93 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -13,7 +13,7 @@ from typing import Iterator import requests -from ftfy import fix_encoding +from ftfy import fix_encoding_and_explain from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry @@ -308,13 +308,19 @@ def _repair_mojibake( if not text: return text - repaired = fix_encoding(text) + result = fix_encoding_and_explain(text) + repaired = result.text if repaired != text: + repair_plan = " -> ".join( + f"{step.action}({step.parameter})" + for step in result.explanation or () + ) logger.info( - "Repaired mojibake in RescueGroups %s for %s %s", + "Repaired mojibake in RescueGroups %s for %s %s: ftfy_plan=%s", field, entity_type, entity_id, + repair_plan or "unspecified", ) return repaired diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 127ee3b..274b52a 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -253,8 +253,37 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): self.assertEqual(len(captured.records), 1) self.assertEqual( captured.records[0].args, - ("description", "animal", "12345"), + ( + "description", + "animal", + "12345", + "encode(latin-1) -> decode(utf-8)", + ), + ) + + def test_logs_every_step_in_a_multi_step_repair_plan(self): + description = "voilà le travail" + + with self.assertLogs( + "adoption_sources.rescue_groups", level="INFO" + ) as captured: + repaired = self.source._clean_description( + description, animal_id="multi-step" + ) + + self.assertEqual(repaired, "voilà le travail") + self.assertEqual( + captured.records[0].args, + ( + "description", + "animal", + "multi-step", + "encode(latin-1) -> transcode(restore_byte_a0) -> " + "decode(utf-8)", + ), ) + self.assertNotIn(description, captured.records[0].getMessage()) + self.assertNotIn(repaired, captured.records[0].getMessage()) class DennisMojibakeMastodonRegressionTests(unittest.TestCase): @@ -543,7 +572,7 @@ def test_logs_each_repaired_field_by_name(self): self._assert_all_fields_repaired(pet) self.assertEqual( - sorted(record.args for record in captured.records), + sorted(record.args[:3] for record in captured.records), sorted( [ (field, "animal", "12345") @@ -552,6 +581,8 @@ def test_logs_each_repaired_field_by_name(self): + [("location", "organization", "org1")] ), ) + for record in captured.records: + self.assertTrue(record.args[3]) def test_leaves_clean_fields_untouched_and_unlogged(self): with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): From af09e0d530635eed153b7d9e0675616b5f7ac4a7 Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:23:57 -0700 Subject: [PATCH 11/13] avoid false positive result from ftfy and use a more conservative method --- adoption_sources/rescue_groups.py | 42 +++++++++++++---- tests/test_rescue_groups.py | 78 +++++++++++++++++++------------ 2 files changed, 83 insertions(+), 37 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 0648e93..61f752d 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -13,7 +13,7 @@ from typing import Iterator import requests -from ftfy import fix_encoding_and_explain +from ftfy import TextFixerConfig, fix_encoding_and_explain from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry @@ -23,6 +23,19 @@ logger = logging.getLogger(__name__) +# Only accept repairs that can be explained as a complete, consistent +# encode/decode round trip. The disabled subordinate fixers reconstruct missing +# bytes or repair isolated spans heuristically, which is useful for recovery but +# can alter legitimate text such as "Ã" or " ". For pet data, preserving valid +# text takes precedence over repairing every damaged string. +MOJIBAKE_CONFIG = TextFixerConfig( + decode_inconsistent_utf8=False, + restore_byte_a0=False, + replace_lossy_sequences=False, + fix_c1_controls=False, +) +SAFE_MOJIBAKE_ACTIONS = frozenset({"encode", "decode"}) + # Some rescues publish entries like "More Dogs Soon!" to point users at their # website; those should never be posted. Add new names here as we encounter them. PLACEHOLDER_NAMES: tuple[str, ...] = ("more dogs soon!", "more cats soon!") @@ -302,18 +315,23 @@ def _repair_mojibake( The HTTP response itself is already valid UTF-8 — the corruption happens upstream of the API, so repairing on our side is the only fix - available to us. ``ftfy`` is conservative: text that isn't recognisable - mojibake is returned untouched. + available to us. Repairs are accepted only when ``ftfy`` can explain + them as a complete encode/decode round trip. More speculative repairs + are left untouched. """ if not text: return text - result = fix_encoding_and_explain(text) + result = fix_encoding_and_explain(text, config=MOJIBAKE_CONFIG) repaired = result.text - if repaired != text: + explanation = result.explanation or () + safe_plan = bool(explanation) and all( + step.action in SAFE_MOJIBAKE_ACTIONS for step in explanation + ) + if repaired != text and safe_plan: repair_plan = " -> ".join( f"{step.action}({step.parameter})" - for step in result.explanation or () + for step in explanation ) logger.info( "Repaired mojibake in RescueGroups %s for %s %s: ftfy_plan=%s", @@ -322,7 +340,8 @@ def _repair_mojibake( entity_id, repair_plan or "unspecified", ) - return repaired + return repaired + return text def _clean_name(self, name: str, animal_id: str = "unknown") -> str: """ @@ -353,7 +372,14 @@ def _clean_description( # Decode HTML entities first, so mojibake that arrived entity-encoded # (’) is repairable too. text = html.unescape(description) - text = self._repair_mojibake(text, "description", animal_id) + # A description may combine paragraphs copied from systems with + # different encodings. Treat natural line boundaries independently so + # one consistently mojibaked paragraph can be repaired without enabling + # ftfy's riskier arbitrary-substring repair. + text = "".join( + self._repair_mojibake(line, "description", animal_id) + for line in text.splitlines(keepends=True) + ) # Remove   and normalize whitespace text = text.replace(" ", " ") diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 274b52a..7be58ac 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -1,6 +1,7 @@ import json import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock, patch from requests import Response @@ -209,14 +210,14 @@ def test_repairs_a_realistic_description_mangled_as_latin_1(self): EXPECTED_DESCRIPTION, ) - def test_repairs_only_the_corrupted_span(self): - """Shelters paste one mangled paragraph into otherwise clean text.""" + def test_repairs_a_corrupted_paragraph_beside_clean_text(self): + """Shelters paste a mangled paragraph into otherwise clean text.""" clean = "Meet Goober! 🐶 His foster José says he’s “the best boy”." corrupted = _mojibake( "Adoption hours: 1:00PM – 6:00PM. Doña Müller answers. 😊" ) - cleaned = self.source._clean_description(f"{clean} {corrupted}") + cleaned = self.source._clean_description(f"{clean}\n{corrupted}") self.assertEqual( cleaned, @@ -261,29 +262,44 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): ), ) - def test_logs_every_step_in_a_multi_step_repair_plan(self): + def test_declines_repair_that_requires_reconstructing_a_missing_byte(self): description = "voilà le travail" - with self.assertLogs( - "adoption_sources.rescue_groups", level="INFO" - ) as captured: + with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): repaired = self.source._clean_description( description, animal_id="multi-step" ) - self.assertEqual(repaired, "voilà le travail") - self.assertEqual( - captured.records[0].args, - ( - "description", - "animal", - "multi-step", - "encode(latin-1) -> transcode(restore_byte_a0) -> " - "decode(utf-8)", - ), + self.assertEqual(repaired, description) + + def test_declines_mixed_encoding_within_one_paragraph(self): + description = ( + "Clean José 🐶. " + + _mojibake("Adoption hours: 1:00PM – 6:00PM. Doña answers.") + ) + + with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + cleaned = self.source._clean_description(description) + + self.assertEqual(cleaned, description) + + @patch("adoption_sources.rescue_groups.fix_encoding_and_explain") + def test_rejects_any_repair_plan_with_a_non_round_trip_step(self, mock_fix): + original = "ambiguous input" + mock_fix.return_value = SimpleNamespace( + text="guessed output", + explanation=[ + SimpleNamespace(action="encode", parameter="latin-1"), + SimpleNamespace(action="transcode", parameter="restore_byte_a0"), + SimpleNamespace(action="decode", parameter="utf-8"), + ], + ) + + repaired = self.source._repair_mojibake( + original, "description", "test-animal" ) - self.assertNotIn(description, captured.records[0].getMessage()) - self.assertNotIn(repaired, captured.records[0].getMessage()) + + self.assertEqual(repaired, original) class DennisMojibakeMastodonRegressionTests(unittest.TestCase): @@ -480,17 +496,21 @@ def test_leaves_the_full_realistic_description_untouched_and_unlogged(self): self.assertEqual(cleaned, EXPECTED_DESCRIPTION) - def test_known_false_positive_lone_capital_a_tilde(self): - """Documented limitation, not desired behavior. + def test_leaves_ambiguous_capital_a_tilde_untouched(self): + description = "Letters like à and Ê are rare." - An isolated ``Ã`` followed by a space is byte-identical to mojibaked - ``à``, so ftfy repairs it. Nothing in a real pet description has hit - this; the test exists so we notice if ftfy's heuristics shift. - """ - self.assertEqual( - self.source._clean_description("Letters like à and Ê are rare."), - "Letters like à and Ê are rare.", - ) + with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + cleaned = self.source._clean_description(description) + + self.assertEqual(cleaned, description) + + def test_leaves_ambiguous_capital_a_circumflex_and_space_untouched(self): + description = " is a letter in Romanian and Vietnamese." + + with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + cleaned = self.source._clean_description(description) + + self.assertEqual(cleaned, description) class PetFieldMojibakeRepairTests(unittest.TestCase): From 4b2eed5aa413d6acf599edd455749b81361f7356 Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:40:44 -0400 Subject: [PATCH 12/13] open up full functionality of ftfy --- adoption_sources/rescue_groups.py | 46 ++------ tests/test_rescue_groups.py | 168 +++++++++++++++++++----------- 2 files changed, 118 insertions(+), 96 deletions(-) diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index 61f752d..ea45882 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -13,7 +13,7 @@ from typing import Iterator import requests -from ftfy import TextFixerConfig, fix_encoding_and_explain +from ftfy import fix_and_explain from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry @@ -23,19 +23,6 @@ logger = logging.getLogger(__name__) -# Only accept repairs that can be explained as a complete, consistent -# encode/decode round trip. The disabled subordinate fixers reconstruct missing -# bytes or repair isolated spans heuristically, which is useful for recovery but -# can alter legitimate text such as "Ã" or " ". For pet data, preserving valid -# text takes precedence over repairing every damaged string. -MOJIBAKE_CONFIG = TextFixerConfig( - decode_inconsistent_utf8=False, - restore_byte_a0=False, - replace_lossy_sequences=False, - fix_c1_controls=False, -) -SAFE_MOJIBAKE_ACTIONS = frozenset({"encode", "decode"}) - # Some rescues publish entries like "More Dogs Soon!" to point users at their # website; those should never be posted. Add new names here as we encounter them. PLACEHOLDER_NAMES: tuple[str, ...] = ("more dogs soon!", "more cats soon!") @@ -311,34 +298,24 @@ def _repair_mojibake( entity_id: str, entity_type: str = "animal", ) -> str: - """Repair text that was mojibaked before RescueGroups serialized it. + """Apply ftfy's complete set of repairs to RescueGroups display text. - The HTTP response itself is already valid UTF-8 — the corruption - happens upstream of the API, so repairing on our side is the only fix - available to us. Repairs are accepted only when ``ftfy`` can explain - them as a complete encode/decode round trip. More speculative repairs - are left untouched. + This deliberately uses ftfy's default configuration, including its + mixed/lossy encoding recovery and general Unicode cleanup. The complete + ``ExplainedText`` result is logged whenever ftfy changes a value. """ if not text: return text - result = fix_encoding_and_explain(text, config=MOJIBAKE_CONFIG) + result = fix_and_explain(text) repaired = result.text - explanation = result.explanation or () - safe_plan = bool(explanation) and all( - step.action in SAFE_MOJIBAKE_ACTIONS for step in explanation - ) - if repaired != text and safe_plan: - repair_plan = " -> ".join( - f"{step.action}({step.parameter})" - for step in explanation - ) + if repaired != text: logger.info( - "Repaired mojibake in RescueGroups %s for %s %s: ftfy_plan=%s", + "Fixed RescueGroups %s for %s %s: ftfy_result=%r", field, entity_type, entity_id, - repair_plan or "unspecified", + result, ) return repaired return text @@ -351,7 +328,7 @@ def _clean_name(self, name: str, animal_id: str = "unknown") -> str: "Doli ***Home for the Holidays 1/2 price!" -> "Doli" "Kathy" -> "Kathy" """ - # Repair before splitting. ftfy weighs the whole string when a sequence + # Fix before splitting. ftfy weighs the whole string when a sequence # is ambiguous (``Ã…`` is both mojibaked ``Å`` and plausible real text), # so discarding the promotional suffix first can lose the only evidence # that tips an accented name toward being repaired. @@ -374,8 +351,7 @@ def _clean_description( text = html.unescape(description) # A description may combine paragraphs copied from systems with # different encodings. Treat natural line boundaries independently so - # one consistently mojibaked paragraph can be repaired without enabling - # ftfy's riskier arbitrary-substring repair. + # ftfy can assess each paragraph on its own. text = "".join( self._repair_mojibake(line, "description", animal_id) for line in text.splitlines(keepends=True) diff --git a/tests/test_rescue_groups.py b/tests/test_rescue_groups.py index 7be58ac..eb1522a 100644 --- a/tests/test_rescue_groups.py +++ b/tests/test_rescue_groups.py @@ -192,6 +192,9 @@ def _mojibake(text: str, encoding: str = "cp1252") -> str: ) LEGIT_DESCRIPTION = "\n".join(LEGIT_DESCRIPTION_LINES) EXPECTED_DESCRIPTION = " ".join(LEGIT_DESCRIPTION_LINES) +EXPECTED_FULLY_FIXED_DESCRIPTION = EXPECTED_DESCRIPTION.translate( + str.maketrans({"’": "'", "“": '"', "”": '"'}) +) class DescriptionMojibakeRepairTests(unittest.TestCase): @@ -201,13 +204,13 @@ def setUp(self): def test_repairs_a_realistic_description_mangled_as_windows_1252(self): self.assertEqual( self.source._clean_description(_mojibake(LEGIT_DESCRIPTION, "cp1252")), - EXPECTED_DESCRIPTION, + EXPECTED_FULLY_FIXED_DESCRIPTION, ) def test_repairs_a_realistic_description_mangled_as_latin_1(self): self.assertEqual( self.source._clean_description(_mojibake(LEGIT_DESCRIPTION, "latin-1")), - EXPECTED_DESCRIPTION, + EXPECTED_FULLY_FIXED_DESCRIPTION, ) def test_repairs_a_corrupted_paragraph_beside_clean_text(self): @@ -221,7 +224,8 @@ def test_repairs_a_corrupted_paragraph_beside_clean_text(self): self.assertEqual( cleaned, - f"{clean} Adoption hours: 1:00PM – 6:00PM. Doña Müller answers. 😊", + "Meet Goober! 🐶 His foster José says he's \"the best boy\". " + "Adoption hours: 1:00PM – 6:00PM. Doña Müller answers. 😊", ) def test_repairs_non_breaking_space_before_whitespace_is_collapsed(self): @@ -234,7 +238,7 @@ def test_repairs_non_breaking_space_before_whitespace_is_collapsed(self): def test_repairs_mojibake_that_arrives_as_html_entities(self): self.assertEqual( self.source._clean_description("I’m ready for a home."), - "I’m ready for a home.", + "I'm ready for a home.", ) def test_repairs_latin_1_mojibake_observed_in_api_response(self): @@ -252,41 +256,45 @@ def test_repairs_latin_1_mojibake_observed_in_api_response(self): self.assertEqual(pet.description, "Adoption hours: 1:00PM – 6:00PM") self.assertEqual(len(captured.records), 1) + record = captured.records[0] + self.assertEqual(record.args[:3], ("description", "animal", "12345")) + self.assertEqual(record.args[3].text, "Adoption hours: 1:00PM – 6:00PM") self.assertEqual( - captured.records[0].args, - ( - "description", - "animal", - "12345", - "encode(latin-1) -> decode(utf-8)", - ), + list(record.args[3].explanation), + [("encode", "latin-1"), ("decode", "utf-8")], ) + self.assertIn(repr(record.args[3]), captured.output[0]) - def test_declines_repair_that_requires_reconstructing_a_missing_byte(self): + def test_repairs_text_that_requires_reconstructing_a_missing_byte(self): description = "voilà le travail" - with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + with self.assertLogs("adoption_sources.rescue_groups", level="INFO"): repaired = self.source._clean_description( description, animal_id="multi-step" ) - self.assertEqual(repaired, description) + self.assertEqual(repaired, "voilà le travail") - def test_declines_mixed_encoding_within_one_paragraph(self): + def test_repairs_mixed_encoding_within_one_paragraph(self): description = ( "Clean José 🐶. " + _mojibake("Adoption hours: 1:00PM – 6:00PM. Doña answers.") ) - with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + with self.assertLogs("adoption_sources.rescue_groups", level="INFO"): cleaned = self.source._clean_description(description) - self.assertEqual(cleaned, description) + self.assertEqual( + cleaned, + "Clean José 🐶. Adoption hours: 1:00PM – 6:00PM. Doña answers.", + ) - @patch("adoption_sources.rescue_groups.fix_encoding_and_explain") - def test_rejects_any_repair_plan_with_a_non_round_trip_step(self, mock_fix): + @patch("adoption_sources.rescue_groups.fix_and_explain") + def test_accepts_non_round_trip_repairs_and_logs_the_complete_result( + self, mock_fix + ): original = "ambiguous input" - mock_fix.return_value = SimpleNamespace( + result = SimpleNamespace( text="guessed output", explanation=[ SimpleNamespace(action="encode", parameter="latin-1"), @@ -294,12 +302,26 @@ def test_rejects_any_repair_plan_with_a_non_round_trip_step(self, mock_fix): SimpleNamespace(action="decode", parameter="utf-8"), ], ) + mock_fix.return_value = result - repaired = self.source._repair_mojibake( - original, "description", "test-animal" - ) + with self.assertLogs( + "adoption_sources.rescue_groups", level="INFO" + ) as captured: + repaired = self.source._repair_mojibake( + original, "description", "test-animal" + ) - self.assertEqual(repaired, original) + self.assertEqual(repaired, "guessed output") + self.assertIs(captured.records[0].args[3], result) + self.assertIn(repr(result), captured.output[0]) + + def test_applies_ftfy_general_text_fixes(self): + with self.assertLogs("adoption_sources.rescue_groups", level="INFO"): + repaired = self.source._repair_mojibake( + "Ｆido’s fine\x00", "name", "12345" + ) + + self.assertEqual(repaired, "Fido's fine") class DennisMojibakeMastodonRegressionTests(unittest.TestCase): @@ -397,17 +419,18 @@ def test_real_dennis_mojibake_is_repaired_before_mastodon_post(self): self.assertIsNotNone(pet) assert pet is not None - self.assertIn("He’s diabetic", pet.description) - self.assertIn("Whether it’s zooming", pet.description) - self.assertIn("He’ll let you know", pet.description) - self.assertIn("you’re showering", pet.description) + self.assertIn("He's diabetic", pet.description) + self.assertIn("Whether it's zooming", pet.description) + self.assertIn("He'll let you know", pet.description) + self.assertIn("you're showering", pet.description) self.assertIn("1:00PM – 6:00PM", pet.description) self.assertNotIn("â\x80\x99", pet.description) self.assertNotIn("â\x80\x93", pet.description) self.assertIn( - "Repaired mojibake in RescueGroups description for animal 22658169", + "Fixed RescueGroups description for animal 22658169", "\n".join(captured_logs.output), ) + self.assertIn("ftfy_result=ExplainedText(text=", captured_logs.output[0]) self.assertEqual(pet.pet_id, "22658169") self.assertEqual(pet.name, "DENNIS") self.assertEqual(pet.species, "cat") @@ -416,8 +439,8 @@ def test_real_dennis_mojibake_is_repaired_before_mastodon_post(self): poster = PosterMastodon.__new__(PosterMastodon) post = poster.format_post(pet) - self.assertIn("He’s diabetic", post.text) - self.assertIn("Whether it’s zooming", post.text) + self.assertIn("He's diabetic", post.text) + self.assertIn("Whether it's zooming", post.text) self.assertIn("1:00PM – 6:00PM", post.text) self.assertNotIn("â\x80\x99", post.text) self.assertNotIn("â\x80\x93", post.text) @@ -448,7 +471,7 @@ def fake_status_post(text, **kwargs): call.args[0] for call in session.status_post.call_args_list ] all_text_sent_to_mastodon = "\n".join(mastodon_payloads) - self.assertIn("’", all_text_sent_to_mastodon) + self.assertIn("He's diabetic", all_text_sent_to_mastodon) self.assertIn("–", all_text_sent_to_mastodon) self.assertNotIn("â\x80\x99", all_text_sent_to_mastodon) self.assertNotIn("â\x80\x93", all_text_sent_to_mastodon) @@ -462,55 +485,66 @@ def fake_status_post(text, **kwargs): self.assertEqual(reply_call.kwargs["in_reply_to_id"], "status-1") -class DescriptionPreservationTests(unittest.TestCase): - """The repair leaves text alone when there is nothing to repair. - - These are the tests that fail if ``fix_encoding`` ever starts over-reaching: - they all pass against the pre-repair code, so only a regression breaks them. - """ +class DescriptionTextFixingTests(unittest.TestCase): + """The full ftfy pipeline fixes text while leaving unrelated content intact.""" # The shapes most likely to be mistaken for mojibake: real Latin-1 letters # (â is what mojibake starts with), percent-encoded URLs (a repair would # break the link), and the cp1252 symbols that mojibake decodes *into*. - CLEAN_DESCRIPTIONS = { - "real a-circumflex": "Château, Ângela, and Râ are real words.", - "correct smart punctuation": "He’s “the best boy” — really… 100% good.", - "percent-encoded url": "Apply at https://example.com/adopt/jos%C3%A9?ref=a%E2%80%93b", - "trademarks and degrees": "PetSmart™ · Petco® · 70°F · ©2026 Example Rescue", - "non-latin scripts": "猫はとても元気です。 강아지 귀여워요! Кот очень милый.", + DESCRIPTION_EXPECTATIONS = { + "real a-circumflex": ( + "Château, Ângela, and Râ are real words.", + "Château, Ângela, and Râ are real words.", + ), + "smart punctuation": ( + "He’s “the best boy” — really… 100% good.", + "He's \"the best boy\" — really… 100% good.", + ), + "percent-encoded url": ( + "Apply at https://example.com/adopt/jos%C3%A9?ref=a%E2%80%93b", + "Apply at https://example.com/adopt/jos%C3%A9?ref=a%E2%80%93b", + ), + "trademarks and degrees": ( + "PetSmart™ · Petco® · 70°F · ©2026 Example Rescue", + "PetSmart™ · Petco® · 70°F · ©2026 Example Rescue", + ), + "non-latin scripts": ( + "猫はとても元気です。 강아지 귀여워요! Кот очень милый.", + "猫はとても元気です。 강아지 귀여워요! Кот очень милый.", + ), } def setUp(self): self.source = SourceRescueGroups(api_key="dummy") - def test_leaves_clean_descriptions_byte_for_byte_identical(self): - for label, description in self.CLEAN_DESCRIPTIONS.items(): + def test_applies_only_relevant_fixes_to_clean_descriptions(self): + for label, (description, expected) in self.DESCRIPTION_EXPECTATIONS.items(): with self.subTest(label): self.assertEqual( - self.source._clean_description(description), description + self.source._clean_description(description), expected ) - def test_leaves_the_full_realistic_description_untouched_and_unlogged(self): - with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + def test_fully_fixes_the_realistic_description_and_logs_changes(self): + with self.assertLogs("adoption_sources.rescue_groups", level="INFO"): cleaned = self.source._clean_description(LEGIT_DESCRIPTION) - self.assertEqual(cleaned, EXPECTED_DESCRIPTION) + self.assertEqual(cleaned, EXPECTED_FULLY_FIXED_DESCRIPTION) - def test_leaves_ambiguous_capital_a_tilde_untouched(self): + def test_aggressively_repairs_ambiguous_capital_a_tilde(self): description = "Letters like à and Ê are rare." - with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + with self.assertLogs("adoption_sources.rescue_groups", level="INFO"): cleaned = self.source._clean_description(description) - self.assertEqual(cleaned, description) + self.assertEqual(cleaned, "Letters like à and Ê are rare.") - def test_leaves_ambiguous_capital_a_circumflex_and_space_untouched(self): + def test_aggressively_repairs_ambiguous_capital_a_circumflex_and_space(self): description = " is a letter in Romanian and Vietnamese." - with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + with self.assertLogs("adoption_sources.rescue_groups", level="INFO"): cleaned = self.source._clean_description(description) - self.assertEqual(cleaned, description) + self.assertEqual(cleaned, " is a letter in Romanian and Vietnamese.") class PetFieldMojibakeRepairTests(unittest.TestCase): @@ -525,6 +559,7 @@ class PetFieldMojibakeRepairTests(unittest.TestCase): LEGIT_CITY = "Montréal" EXPECTED_LOCATION = "Montréal, QC" LEGIT_TEXT = "She’s a sweetheart – really." + EXPECTED_TEXT = "She's a sweetheart – really." def setUp(self): self.source = SourceRescueGroups(api_key="dummy") @@ -551,7 +586,7 @@ def _assert_all_fields_repaired(self, pet) -> None: self.assertEqual(pet.name, self.EXPECTED_NAME) self.assertEqual(pet.breed, self.LEGIT_BREED) self.assertEqual(pet.location, self.EXPECTED_LOCATION) - self.assertEqual(pet.description, self.LEGIT_TEXT) + self.assertEqual(pet.description, self.EXPECTED_TEXT) def test_repairs_name_breed_and_location_through_fetch_pets(self): body = { @@ -602,10 +637,15 @@ def test_logs_each_repaired_field_by_name(self): ), ) for record in captured.records: - self.assertTrue(record.args[3]) + result = record.args[3] + self.assertTrue(result.text) + self.assertTrue(result.explanation) + self.assertIn(repr(result), record.getMessage()) - def test_leaves_clean_fields_untouched_and_unlogged(self): - with self.assertNoLogs("adoption_sources.rescue_groups", level="INFO"): + def test_applies_general_fixes_to_clean_fields(self): + with self.assertLogs( + "adoption_sources.rescue_groups", level="INFO" + ) as captured: pet = self.source._parse_animal( self._animal(corrupt=False), self._orgs(corrupt=False), @@ -613,6 +653,12 @@ def test_leaves_clean_fields_untouched_and_unlogged(self): ) self._assert_all_fields_repaired(pet) + self.assertEqual(len(captured.records), 1) + self.assertEqual(captured.records[0].args[:3], ( + "description", + "animal", + "12345", + )) def test_repairs_name_before_stripping_the_promotional_suffix(self): """``Ã…`` is ambiguous on its own — ftfy only fixes it when the rest of @@ -629,7 +675,7 @@ def test_repairs_name_before_stripping_the_promotional_suffix(self): def test_repairs_name_and_breed_that_only_differ_in_smart_punctuation(self): self.assertEqual( self.source._clean_name(_mojibake("Lucky — the “office dog”")), - "Lucky — the “office dog”", + 'Lucky — the "office dog"', ) self.assertEqual( self.source._repair_mojibake( From b1d4211397c740fbbdc37418df3cd15526a1adef Mon Sep 17 00:00:00 2001 From: patrickZWY <154947644+patrickZWY@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:20:49 -0400 Subject: [PATCH 13/13] return to prev debug flag --- .github/workflows/dev.yml | 2 +- adoption_sources/rescue_groups.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 3abd51b..7a2082b 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -68,7 +68,7 @@ jobs: APP_ENV: dev run: | #In order to create posts on the test accounts remove the --debugposters debug flag - python ./main.py + python ./main.py --debugsources --debugposters - name: Upload database artifact uses: actions/upload-artifact@v7 diff --git a/adoption_sources/rescue_groups.py b/adoption_sources/rescue_groups.py index ea45882..5861982 100644 --- a/adoption_sources/rescue_groups.py +++ b/adoption_sources/rescue_groups.py @@ -328,10 +328,6 @@ def _clean_name(self, name: str, animal_id: str = "unknown") -> str: "Doli ***Home for the Holidays 1/2 price!" -> "Doli" "Kathy" -> "Kathy" """ - # Fix before splitting. ftfy weighs the whole string when a sequence - # is ambiguous (``Ã…`` is both mojibaked ``Å`` and plausible real text), - # so discarding the promotional suffix first can lose the only evidence - # that tips an accented name toward being repaired. name = self._repair_mojibake(name, "name", animal_id) # Remove common promotional suffixes