From d6af301269b85708d62c6ba7c0309be16ef11a54 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Tue, 25 Nov 2025 16:46:01 +0100 Subject: [PATCH 1/5] [Community] add botlogs --- octobot/community/__init__.py | 1 + octobot/community/models/__init__.py | 5 +++++ octobot/community/models/bot_log.py | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 octobot/community/models/bot_log.py diff --git a/octobot/community/__init__.py b/octobot/community/__init__.py index 783796b8d7..29eaaf8095 100644 --- a/octobot/community/__init__.py +++ b/octobot/community/__init__.py @@ -24,6 +24,7 @@ ) from octobot.community import models from octobot.community.models import ( + BotLogData, CommunityUserAccount, CommunityFields, CommunityTentaclesPackage, diff --git a/octobot/community/models/__init__.py b/octobot/community/models/__init__.py index fa56e3546d..c709652160 100644 --- a/octobot/community/models/__init__.py +++ b/octobot/community/models/__init__.py @@ -18,6 +18,10 @@ from octobot.community.models.community_user_account import ( CommunityUserAccount, ) +from octobot.community.models import bot_log +from octobot.community.models.bot_log import ( + BotLogData, +) from octobot.community.models import community_fields from octobot.community.models.community_fields import ( CommunityFields, @@ -74,6 +78,7 @@ __all__ = [ "CommunityUserAccount", "CommunityFields", + "BotLogData", "CommunityTentaclesPackage", "CommunitySupports", "CommunityDonation", diff --git a/octobot/community/models/bot_log.py b/octobot/community/models/bot_log.py new file mode 100644 index 0000000000..b3e8903123 --- /dev/null +++ b/octobot/community/models/bot_log.py @@ -0,0 +1,26 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import dataclasses +import typing + +import octobot_commons.dataclasses +import octobot.community.supabase_backend.enums as supabase_enums + + +@dataclasses.dataclass +class BotLogData(octobot_commons.dataclasses.FlexibleDataclass): + log_type: supabase_enums.BotLogType + content: typing.Optional[dict] From f3562c6aa35d149d1cf5d3e8ec18a165a066cbbf Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Tue, 25 Nov 2025 18:55:30 +0100 Subject: [PATCH 2/5] [Requirements] add full requirements --- .github/workflows/main.yml | 2 +- Dockerfile | 2 +- MANIFEST.in | 1 + full_requirements.txt | 22 +++++++ .../community/feeds/community_mqtt_feed.py | 27 +++++++- .../clickhouse_historical_backend_client.py | 15 ++++- .../iceberg_historical_backend_client.py | 64 ++++++++++++++++--- requirements.txt | 24 ++----- setup.py | 4 ++ 9 files changed, 129 insertions(+), 32 deletions(-) create mode 100644 full_requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7886fa62cc..dc8a04b3b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -137,7 +137,7 @@ jobs: architecture: ${{ matrix.arch }} - name: Install dependencies - run: pip install --prefer-binary -r dev_requirements.txt -r requirements.txt + run: pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt - name: Build sdist run: python setup.py sdist diff --git a/Dockerfile b/Dockerfile index 868b2e0508..9496ee92a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ENV PATH="/opt/venv/bin:$PATH" COPY . . RUN pip install -U setuptools wheel pip>=20.0.0 \ - && pip install --no-cache-dir --prefer-binary -r requirements.txt \ + && pip install --no-cache-dir --prefer-binary -r requirements.txt -r full_requirements.txt \ && python setup.py install FROM python:3.10-slim-buster diff --git a/MANIFEST.in b/MANIFEST.in index a7c881c076..8986fd3cc0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,5 +4,6 @@ include README.md include LICENSE include CHANGELOG.md include requirements.txt +include full_requirements.txt global-exclude *.c diff --git a/full_requirements.txt b/full_requirements.txt new file mode 100644 index 0000000000..aa4ccfdf9e --- /dev/null +++ b/full_requirements.txt @@ -0,0 +1,22 @@ +# Drakkar-Software full requirements +OctoBot-Commons[full]==1.9.88 +OctoBot-Trading[full]==2.4.232 +OctoBot-Evaluators[full]==1.9.9 +OctoBot-Tentacles-Manager[full]==2.9.19 +OctoBot-Services[full]==1.6.29 +OctoBot-Backtesting[full]==1.9.8 + +## Others +colorlog==6.8.0 + +# Community +gmqtt==0.7.0 +pgpy==0.6.0 +clickhouse-connect==0.8.18 +pyiceberg==0.10.0 +pydantic<2.12 # required for pyiceberg 0.10.0 https://github.com/apache/iceberg-python/issues/2590 +pyarrow==21.0.0 + +# used by ccxt for protobuf "websockets" such as mexc +# lock protobuf to avoid using .rc versions +protobuf==5.29.5 diff --git a/octobot/community/feeds/community_mqtt_feed.py b/octobot/community/feeds/community_mqtt_feed.py index 78662ee70e..bbc9670f33 100644 --- a/octobot/community/feeds/community_mqtt_feed.py +++ b/octobot/community/feeds/community_mqtt_feed.py @@ -15,14 +15,37 @@ # License along with OctoBot. If not, see . import logging import typing -import gmqtt +import octobot_commons.constants as commons_constants +try: + import gmqtt +except ImportError: + if commons_constants.USE_MINIMAL_LIBS: + # mock gmqtt imports + class GmqttImportMock: + class Client: + def __init__(self, *args): + raise ImportError("gmqtt not installed") + class Subscription: + def __init__(self, topic, qos): + raise ImportError("gmqtt not installed") + class constants: + class MQTTv311: + pass + class SubAckReasonCode: + class UNSPECIFIED_ERROR: + value = 1 + class DEFAULT_CONFIG: + pass + class Subscription: + def __init__(self, topic, qos): + pass + gmqtt = GmqttImportMock() import json import asyncio import packaging.version as packaging_version import octobot_commons.enums as commons_enums import octobot_commons.errors as commons_errors -import octobot_commons.constants as commons_constants import octobot.community.errors as errors import octobot.community.feeds.abstract_feed as abstract_feed import octobot.constants as constants diff --git a/octobot/community/history_backend/clickhouse_historical_backend_client.py b/octobot/community/history_backend/clickhouse_historical_backend_client.py index d3e65778f5..e45a6490a1 100644 --- a/octobot/community/history_backend/clickhouse_historical_backend_client.py +++ b/octobot/community/history_backend/clickhouse_historical_backend_client.py @@ -16,7 +16,20 @@ import typing from datetime import datetime, timezone -import clickhouse_connect.driver +import octobot_commons.constants as commons_constants +try: + import clickhouse_connect.driver +except ImportError: + if commons_constants.USE_MINIMAL_LIBS: + # mock clickhouse_connect.driver imports + class ClickhouseConnectImportMock: + async def get_async_client(self, *args): + raise ImportError("clickhouse_connect not installed") + class driver: + class AsyncClient: + async def close(self): + pass + clickhouse_connect = ClickhouseConnectImportMock() import octobot_commons.logging as commons_logging import octobot_commons.enums as commons_enums diff --git a/octobot/community/history_backend/iceberg_historical_backend_client.py b/octobot/community/history_backend/iceberg_historical_backend_client.py index 24741f0ae2..8cbc85b1ce 100644 --- a/octobot/community/history_backend/iceberg_historical_backend_client.py +++ b/octobot/community/history_backend/iceberg_historical_backend_client.py @@ -36,16 +36,60 @@ class PyArrowMock(): Schema = None pyarrow = PyArrowMock() -import pyiceberg.catalog -import pyiceberg.schema -import pyiceberg.types -import pyiceberg.exceptions -import pyiceberg.expressions -import pyiceberg.table -import pyiceberg.table.sorting -import pyiceberg.table.statistics -import pyiceberg.table.update -import pyiceberg.table.refs +try: + import pyiceberg.catalog + import pyiceberg.schema + import pyiceberg.types + import pyiceberg.exceptions + import pyiceberg.expressions + import pyiceberg.table + import pyiceberg.table.sorting + import pyiceberg.table.statistics + import pyiceberg.table.update + import pyiceberg.table.refs +except ImportError as err: + commons_logging.get_logger().info(f"Skipped pyiceberg import: {err}") + class PyIcebergImportMock(): + # type hints mocks only + class catalog: + class Catalog: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + def load_catalog(self, *args, **kwargs): + raise ImportError("pyiceberg not installed") + class table: + class statistics: + class StatisticsFile: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class BlobMetadata: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class Table: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class DataScan: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class exceptions: + class NoSuchTableError: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class schema: + class Schema: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class expressions: + class BooleanExpression: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class Or: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class EqualTo: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + pyiceberg = PyIcebergImportMock() import octobot_commons.enums as commons_enums import octobot.constants as constants diff --git a/requirements.txt b/requirements.txt index 5d5501efe2..627ac673d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,14 @@ # Drakkar-Software requirements -OctoBot-Commons==1.9.86 -OctoBot-Trading==2.4.231 -OctoBot-Evaluators==1.9.7 -OctoBot-Tentacles-Manager==2.9.18 -OctoBot-Services==1.6.26 -OctoBot-Backtesting==1.9.7 +OctoBot-Commons==1.9.88 +OctoBot-Trading==2.4.232 +OctoBot-Evaluators==1.9.9 +OctoBot-Tentacles-Manager==2.9.19 +OctoBot-Services==1.6.29 +OctoBot-Backtesting==1.9.8 Async-Channel==2.2.1 trading-backend==1.2.42 ## Others -colorlog==6.8.0 requests==2.32.5 urllib3 # required by requests, used in imports: make sure it's always available packaging==25.0 @@ -18,13 +17,7 @@ setuptools==79.0.1 # warning: setuptools>=80 breaks easy_install, need to find # see https://community.palantir.com/t/important-update-on-setuptools-pinning-the-version-below-80-0-0/3872 # Community -websockets==15.0.1 # used by supabase, a recent version is required, see https://github.com/supabase/realtime-py/blob/main/pyproject.toml -gmqtt==0.7.0 pgpy==0.6.0 -clickhouse-connect==0.8.18 -pyiceberg==0.10.0 -pydantic<2.12 # required for pyiceberg 0.10.0 https://github.com/apache/iceberg-python/issues/2590 -pyarrow==21.0.0 # Error tracking sentry-sdk==2.35.0 # always make sure sentry_aiohttp_transport.py keep working @@ -33,13 +26,10 @@ sentry-sdk==2.35.0 # always make sure sentry_aiohttp_transport.py keep working supabase==2.18.1 # Supabase client supabase_auth # Supabase authenticated API (required by supabase and enforced to allow direct import) postgrest # Supabase posgres calls (required by supabase and enforced to allow direct import) +websockets==15.0.1 # used by supabase, a recent version is required, see https://github.com/supabase/realtime-py/blob/main/pyproject.toml # async http requests aiohttp==3.12.15 # updating to aiodns==3.2.0 is incompatible (and failing CI) # raises RuntimeError: aiodns needs a SelectorEventLoop on Windows. See more: https://github.com/saghul/aiodns/issues/86 aiodns==3.1.1 # used by aiohttp - -# used by ccxt for protobuf "websockets" such as mexc -# lock protobuf to avoid using .rc versions -protobuf==5.29.5 diff --git a/setup.py b/setup.py index 5c81de1fd9..068c9bb4fe 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ def ignore_git_requirements(requirements): REQUIRED = ignore_git_requirements(open('requirements.txt').readlines()) +FULL_REQUIRED = ignore_git_requirements(open('full_requirements.txt').readlines()) REQUIRES_PYTHON = '>=3.10' setup( @@ -50,6 +51,9 @@ def ignore_git_requirements(requirements): test_suite="tests", zip_safe=False, install_requires=REQUIRED, + extras_require={ + 'full': FULL_REQUIRED, + }, python_requires=REQUIRES_PYTHON, entry_points={ 'console_scripts': [ From 0c9e5570afa145bf12c9bb3b80e3fdbd90bef633 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 26 Nov 2025 14:00:22 +0100 Subject: [PATCH 3/5] [Setup.py] improve bdist_wheel output --- setup.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 068c9bb4fe..8f8307e6db 100644 --- a/setup.py +++ b/setup.py @@ -13,11 +13,27 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . +import sys from setuptools import find_packages from setuptools import setup from octobot import PROJECT_NAME, AUTHOR, VERSION -PACKAGES = find_packages(exclude=["tentacles*", "tests", ]) +is_building_wheel = "bdist_wheel" in sys.argv + +EXCLUDED_PACKAGES = ["tentacles*", "tests"] +DATA_FILES = ["config/*", "strategy_optimizer/optimizer_data_files/*"] +if is_building_wheel: + # exclude data files when building a wheel + DATA_FILES = [] +PACKAGES = [ + pkg for pkg in find_packages(exclude=EXCLUDED_PACKAGES) + if not any( + # manually excluded packages as they can't be excluded from a wheel otherwise + # because those are valid python packages + str(pkg).startswith(excluded_package) + for excluded_package in EXCLUDED_PACKAGES + ) +] # long description from README file with open('README.md', encoding='utf-8') as f: @@ -43,7 +59,7 @@ def ignore_git_requirements(requirements): py_modules=['start'], packages=PACKAGES, package_data={ - "": ["config/*", "strategy_optimizer/optimizer_data_files/*"], + "": DATA_FILES, }, long_description=DESCRIPTION, long_description_content_type='text/markdown', From 09996dbd459533fb530c32a67676a6f79e35060a Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 26 Nov 2025 15:57:09 +0100 Subject: [PATCH 4/5] [Linter] fixes --- .../history_backend/iceberg_historical_backend_client.py | 2 +- standard.rc | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/octobot/community/history_backend/iceberg_historical_backend_client.py b/octobot/community/history_backend/iceberg_historical_backend_client.py index 8cbc85b1ce..3b9900632e 100644 --- a/octobot/community/history_backend/iceberg_historical_backend_client.py +++ b/octobot/community/history_backend/iceberg_historical_backend_client.py @@ -72,7 +72,7 @@ class DataScan: def __init__(self, *args): raise ImportError("pyiceberg not installed") class exceptions: - class NoSuchTableError: + class NoSuchTableError(Exception): def __init__(self, *args): raise ImportError("pyiceberg not installed") class schema: diff --git a/standard.rc b/standard.rc index 9799eadf48..96a7ee5aa8 100644 --- a/standard.rc +++ b/standard.rc @@ -36,10 +36,6 @@ load-plugins= # Pickle collected data for later comparisons. persistent=yes -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no From f29139ae4f62551434f2f13ee0a8c9d504cf661c Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 26 Nov 2025 16:06:03 +0100 Subject: [PATCH 5/5] [CI] fix deps --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dc8a04b3b5..7beb8351f7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ jobs: architecture: ${{ matrix.arch }} - name: Install dependencies - run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt + run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt # - name: Black lint # run: black ${{ secrets.PACKAGE_FOLDER }} --diff --check @@ -70,7 +70,7 @@ jobs: run: exit 1 - name: Install dependencies - run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt + run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt - name: Install tentacles on Unix env: