diff --git a/misp_modules/modules/expansion/html_to_markdown.py b/misp_modules/modules/expansion/html_to_markdown.py
index 55d5802b1..67c286832 100755
--- a/misp_modules/modules/expansion/html_to_markdown.py
+++ b/misp_modules/modules/expansion/html_to_markdown.py
@@ -3,7 +3,7 @@
import requests
import ipaddress
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
from markdownify import markdownify
@@ -27,14 +27,7 @@
}
-BLOCKED_RANGES = [
- ipaddress.ip_network("127.0.0.0/8"),
- ipaddress.ip_network("10.0.0.0/8"),
- ipaddress.ip_network("172.16.0.0/12"),
- ipaddress.ip_network("192.168.0.0/16"),
- ipaddress.ip_network("169.254.0.0/16"),
- ipaddress.ip_network("::1/128"),
-]
+MAX_REDIRECTS = 5
def _normalize_ip_address(ip_str: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address:
@@ -43,8 +36,7 @@ def _normalize_ip_address(ip_str: str) -> ipaddress.IPv4Address | ipaddress.IPv6
def _is_ip_blocked(ip_str: str) -> bool:
- ip = _normalize_ip_address(ip_str)
- return any(ip in net for net in BLOCKED_RANGES)
+ return not _normalize_ip_address(ip_str).is_global
def _hostname_resolves_to_blocked_ip(hostname: str) -> bool:
@@ -66,10 +58,15 @@ def is_safe_url(url: str) -> bool:
def fetchHTML(url):
- if not is_safe_url(url):
- raise ValueError(f"Blocked URL: {url}")
- r = requests.get(url, timeout=10)
- return r.text
+ for _ in range(MAX_REDIRECTS + 1):
+ if not is_safe_url(url):
+ raise ValueError(f"Blocked URL: {url}")
+ r = requests.get(url, timeout=10, allow_redirects=False)
+ if r.is_redirect and r.headers.get("location"):
+ url = urljoin(url, r.headers["location"])
+ continue
+ return r.text
+ raise ValueError(f"Too many redirects: {url}")
def stripUselessTags(html):