diff --git a/.gitignore b/.gitignore index 815baf7a..d441bd1f 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ app.*.map.json /android/app/debug /android/app/profile /android/app/release +cache/ diff --git a/db/tests/test_scrape_utils.py b/db/tests/test_scrape_utils.py index e3368b10..a0951a3e 100644 --- a/db/tests/test_scrape_utils.py +++ b/db/tests/test_scrape_utils.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch +import utils.scrape_utils as scrape_utils from utils.scrape_utils import ( _needs_playwright, close_browser, @@ -88,8 +89,9 @@ def test_fetch_url_with_existing_session(tmp_cache_dir): def test_fetch_url_playwright_route(tmp_cache_dir): - url = f'https://{PLAYWRIGHT_REQUIRED_HOSTS[0]}/page' - with patch('utils.scrape_utils._fetch_with_playwright', return_value='pw') as mock_pw: + url = 'https://pw-only.example.com/page' + with patch.object(scrape_utils, 'PLAYWRIGHT_REQUIRED_HOSTS', ['pw-only.example.com']), \ + patch('utils.scrape_utils._fetch_with_playwright', return_value='pw') as mock_pw: result = fetch_url(url) assert result == 'pw' mock_pw.assert_called_once_with(url) diff --git a/db/utils/scrape_utils.py b/db/utils/scrape_utils.py index fa14dbc3..2eb4c601 100644 --- a/db/utils/scrape_utils.py +++ b/db/utils/scrape_utils.py @@ -20,8 +20,10 @@ 'Upgrade-Insecure-Requests': '1' } -# Sites that require Playwright (real browser) due to TLS fingerprinting -PLAYWRIGHT_REQUIRED_HOSTS = ['repo.mariocube.com'] +# Sites that require Playwright (real browser) due to TLS fingerprinting. +# repo.mariocube.com flipped: Cloudflare now challenges headless Chromium +# but passes cloudscraper, so it uses the default path again. +PLAYWRIGHT_REQUIRED_HOSTS = [] # Rate limiting settings MAX_RETRIES = 5