From 257024e3092acd97b2f51cae8549065d0f7cb070 Mon Sep 17 00:00:00 2001 From: elphabert <245968784+elphabert@users.noreply.github.com> Date: Thu, 27 Nov 2025 04:00:59 +1100 Subject: [PATCH 1/2] Added simple retry policy to all reqeusts to retry 5xx errors --- Downloader.py | 13 +++++++++- parameters.py | 5 +++- streamonitor/bot.py | 25 ++++++++++-------- streamonitor/sites/stripchat.py | 45 ++++++++++++++++++++------------- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/Downloader.py b/Downloader.py index fea95a4d..43e8e79f 100644 --- a/Downloader.py +++ b/Downloader.py @@ -1,14 +1,17 @@ import os import sys +import parameters import streamonitor.config as config +import http.client from streamonitor.managers.httpmanager import HTTPManager from streamonitor.managers.climanager import CLIManager from streamonitor.managers.zmqmanager import ZMQManager from streamonitor.managers.outofspace_detector import OOSDetector from streamonitor.clean_exit import CleanExit +import streamonitor.log import streamonitor.sites # must have - + def is_docker(): path = '/proc/self/cgroup' return ( @@ -24,6 +27,14 @@ def main(): streamers = config.loadStreamers() + if parameters.HTTP_DEBUG + import logging + + http.client.HTTPConnection.debuglevel = 1 + http_log = logging.getLogger("requests.packages.urllib3") + http_log.setLevel(logging.DEBUG) + http_log.addHandler(logging.StreamHandler()) + clean_exit = CleanExit(streamers) oos_detector = OOSDetector(streamers) diff --git a/parameters.py b/parameters.py index 83c24fdf..e4f6470a 100644 --- a/parameters.py +++ b/parameters.py @@ -10,9 +10,12 @@ DOWNLOADS_DIR = env.str("STRMNTR_DOWNLOAD_DIR", "downloads") MIN_FREE_DISK_PERCENT = env.float("STRMNTR_MIN_FREE_SPACE", 5.0) # in % DEBUG = env.bool("STRMNTR_DEBUG", False) +HTTP_DEBUG = env.bool("STRMNTR_HTTP_DEBUG", False) # The camsoda bot ignores this setting in favor of a chrome useragent generated with the fake-useragent library -HTTP_USER_AGENT = env.str("STRMNTR_USER_AGENT", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0") +HTTP_USER_AGENT = env.str( + "STRMNTR_USER_AGENT", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0" +) # Specify the full path to the ffmpeg binary. By default, ffmpeg found on PATH is used. FFMPEG_PATH = env.str("STRMNTR_FFMPEG_PATH", 'ffmpeg') diff --git a/streamonitor/bot.py b/streamonitor/bot.py index e4f2cd2b..e68c7079 100644 --- a/streamonitor/bot.py +++ b/streamonitor/bot.py @@ -8,7 +8,8 @@ from threading import Thread import requests -import requests.cookies +from requests.adapters import HTTPAdapter +from urllib3 import Retry from streamonitor.enums import Status import streamonitor.log as log @@ -32,9 +33,9 @@ class Bot(Thread): sleep_on_ratelimit = 180 long_offline_timeout = 600 - headers = { - "User-Agent": HTTP_USER_AGENT - } + headers = {"User-Agent": HTTP_USER_AGENT} + + retries = Retry(total=10, connect=3, read=3, status_forcelist=range(500, 599), backoff_factor=3, backoff_jitter=3) status_messages = { Status.UNKNOWN: "Unknown error", @@ -46,7 +47,7 @@ class Bot(Thread): Status.NOTEXIST: "Nonexistent user", Status.NOTRUNNING: "Not running", Status.ERROR: "Error on downloading", - Status.RESTRICTED: "Model is restricted, maybe geo-block" + Status.RESTRICTED: "Model is restricted, maybe geo-block", } def __init_subclass__(cls, **kwargs): @@ -63,6 +64,8 @@ def __init__(self, username): self.session = requests.Session() self.session.headers.update(self.headers) + self.session.mount('http://', HTTPAdapter(max_retries=Bot.retries)) + self.session.mount('https://', HTTPAdapter(max_retries=Bot.retries)) self.cookies = None self.cookieUpdater = None self.cookie_update_interval = 0 @@ -178,6 +181,7 @@ def run(self): offline_time = 0 if self.sc == Status.PUBLIC: if self.cookie_update_interval > 0 and self.cookieUpdater is not None: + def update_cookie(): while self.sc == Status.PUBLIC and not self.quitting and self.running: self._sleep(self.cookie_update_interval) @@ -186,6 +190,7 @@ def update_cookie(): self.debug('Updated cookies') else: self.logger.warning('Failed to update cookies') + cookie_update_process = Thread(target=update_cookie) cookie_update_process.start() @@ -253,7 +258,7 @@ def getPlaylistVariants(self, url=None, m3u_data=None): 'url': playlist.uri, 'resolution': resolution, 'frame_rate': stream_info.frame_rate, - 'bandwidth': stream_info.bandwidth + 'bandwidth': stream_info.bandwidth, }) if not variant_m3u8.is_variant and len(sources) >= 1: @@ -308,7 +313,9 @@ def getWantedResolutionPlaylist(self, url): frame_rate = '' if selected_source['frame_rate'] is not None and selected_source['frame_rate'] != 0: frame_rate = f" {selected_source['frame_rate']}fps" - self.logger.info(f"Selected {selected_source['resolution'][0]}x{selected_source['resolution'][1]}{frame_rate} resolution") + self.logger.info( + f"Selected {selected_source['resolution'][0]}x{selected_source['resolution'][1]}{frame_rate} resolution" + ) selected_source_url = selected_source['url'] if selected_source_url.startswith("https://"): return selected_source_url @@ -353,9 +360,7 @@ def export(self): def str2site(site: str): site = site.lower() for sitecls in LOADED_SITES: - if site == sitecls.site.lower() or \ - site == sitecls.siteslug.lower() or \ - site in sitecls.aliases: + if site == sitecls.site.lower() or site == sitecls.siteslug.lower() or site in sitecls.aliases: return sitecls return None diff --git a/streamonitor/sites/stripchat.py b/streamonitor/sites/stripchat.py index fb17b6db..b123a9f4 100644 --- a/streamonitor/sites/stripchat.py +++ b/streamonitor/sites/stripchat.py @@ -6,6 +6,8 @@ import base64 import hashlib +from requests.adapters import HTTPAdapter + from streamonitor.bot import Bot from streamonitor.downloaders.hls import getVideoNativeHLS from streamonitor.enums import Status @@ -37,7 +39,9 @@ def __init__(self, username): @classmethod def getInitialData(cls): - r = requests.get('https://hu.stripchat.com/api/front/v3/config/static', headers=cls.headers) + s = requests.Session() + s.mount('https://', HTTPAdapter(max_retries=cls.retries)) + r = s.get('https://hu.stripchat.com/api/front/v3/config/static', headers=cls.headers) if r.status_code != 200: raise Exception("Failed to fetch static data from StripChat") StripChat._static_data = r.json().get('static') @@ -46,14 +50,14 @@ def getInitialData(cls): mmp_version = StripChat._static_data['featuresV2']['playerModuleExternalLoading']['mmpVersion'] mmp_base = f"{mmp_origin}/v{mmp_version}" - r = requests.get(f"{mmp_base}/main.js", headers=cls.headers) + r = s.get(f"{mmp_base}/main.js", headers=cls.headers) if r.status_code != 200: raise Exception("Failed to fetch main.js from StripChat") StripChat._main_js_data = r.content.decode('utf-8') doppio_js_name = re.findall('require[(]"./(Doppio.*?[.]js)"[)]', StripChat._main_js_data)[0] - r = requests.get(f"{mmp_base}/{doppio_js_name}", headers=cls.headers) + r = s.get(f"{mmp_base}/{doppio_js_name}", headers=cls.headers) if r.status_code != 200: raise Exception("Failed to fetch doppio.js from StripChat") StripChat._doppio_js_data = r.content.decode('utf-8') @@ -66,8 +70,11 @@ def m3u_decoder(cls, content): def _decode(encrypted_b64: str, key: str) -> str: if cls._cached_keys is None: cls._cached_keys = {} - hash_bytes = cls._cached_keys[key] if key in cls._cached_keys \ + hash_bytes = ( + cls._cached_keys[key] + if key in cls._cached_keys else cls._cached_keys.setdefault(key, hashlib.sha256(key.encode("utf-8")).digest()) + ) encrypted_data = base64.b64decode(encrypted_b64 + "==") return bytes(a ^ b for (a, b) in zip(encrypted_data, itertools.cycle(hash_bytes))).decode("utf-8") @@ -78,7 +85,7 @@ def _decode(encrypted_b64: str, key: str) -> str: last_decoded_file = None for line in lines: if line.startswith(_mouflon_file_attr): - last_decoded_file = _decode(line[len(_mouflon_file_attr):], pdkey) + last_decoded_file = _decode(line[len(_mouflon_file_attr) :], pdkey) elif line.endswith(_mouflon_filename) and last_decoded_file: decoded += (line.replace(_mouflon_filename, last_decoded_file)) + '\n' last_decoded_file = None @@ -105,7 +112,7 @@ def _getMouflonFromM3U(m3u8_doc): while _needle in (_doc := m3u8_doc[_start:]): _mouflon_start = _doc.find(_needle) if _mouflon_start > 0: - _mouflon = _doc[_mouflon_start:m3u8_doc.find('\n', _mouflon_start)].strip().split(':') + _mouflon = _doc[_mouflon_start : m3u8_doc.find('\n', _mouflon_start)].strip().split(':') psch = _mouflon[2] pkey = _mouflon[3] pdkey = StripChat.getMouflonDecKey(pkey) @@ -122,28 +129,30 @@ def getVideoUrl(self): def getPlaylistVariants(self, url): url = "https://edge-hls.{host}/hls/{id}{vr}/master/{id}{vr}{auto}.m3u8".format( - host='doppiocdn.' + random.choice(['org', 'com', 'net']), - id=self.lastInfo["streamName"], - vr='_vr' if self.vr else '', - auto='_auto' if not self.vr else '' - ) - result = requests.get(url, headers=self.headers, cookies=self.cookies) + host='doppiocdn.' + random.choice(['org', 'com', 'net']), + id=self.lastInfo["streamName"], + vr='_vr' if self.vr else '', + auto='_auto' if not self.vr else '', + ) + result = self.session.get(url, headers=self.headers, cookies=self.cookies) m3u8_doc = result.content.decode("utf-8") psch, pkey, pdkey = StripChat._getMouflonFromM3U(m3u8_doc) variants = super().getPlaylistVariants(m3u_data=m3u8_doc) - return [variant | {'url': f'{variant["url"]}{"&" if "?" in variant["url"] else "?"}psch={psch}&pkey={pkey}'} - for variant in variants] + return [ + variant | {'url': f'{variant["url"]}{"&" if "?" in variant["url"] else "?"}psch={psch}&pkey={pkey}'} + for variant in variants + ] @staticmethod def uniq(length=16): - chars = ''.join(chr(i) for i in range(ord('a'), ord('z')+1)) - chars += ''.join(chr(i) for i in range(ord('0'), ord('9')+1)) + chars = ''.join(chr(i) for i in range(ord('a'), ord('z') + 1)) + chars += ''.join(chr(i) for i in range(ord('0'), ord('9') + 1)) return ''.join(random.choice(chars) for _ in range(length)) def getStatus(self): - r = requests.get( + r = self.session.get( f'https://stripchat.com/api/front/v2/models/username/{self.username}/cam?uniq={StripChat.uniq()}', - headers=self.headers + headers=self.headers, ) try: From 40f721ab121bb73e7f5862d4856f92d3f013535d Mon Sep 17 00:00:00 2001 From: elphabert <245968784+elphabert@users.noreply.github.com> Date: Thu, 27 Nov 2025 04:17:08 +1100 Subject: [PATCH 2/2] typo --- Downloader.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Downloader.py b/Downloader.py index 43e8e79f..9cbd379e 100644 --- a/Downloader.py +++ b/Downloader.py @@ -14,10 +14,7 @@ def is_docker(): path = '/proc/self/cgroup' - return ( - os.path.exists('/.dockerenv') or - os.path.isfile(path) and any('docker' in line for line in open(path)) - ) + return os.path.exists('/.dockerenv') or os.path.isfile(path) and any('docker' in line for line in open(path)) def main(): @@ -27,7 +24,7 @@ def main(): streamers = config.loadStreamers() - if parameters.HTTP_DEBUG + if parameters.HTTP_DEBUG: import logging http.client.HTTPConnection.debuglevel = 1