diff --git a/.github/workflows/slo-2dc.yml b/.github/workflows/slo-2dc.yml new file mode 100644 index 00000000..a8084e51 --- /dev/null +++ b/.github/workflows/slo-2dc.yml @@ -0,0 +1,99 @@ +name: Hosted 2DC SLO + +on: + pull_request: + paths: + - .github/workflows/slo-2dc.yml + - tests/slo/** + - ydb/driver.py + - ydb/resolver.py + - ydb/pool.py + - ydb/aio/driver.py + - ydb/aio/pool.py + workflow_dispatch: + inputs: + server_image: + description: YDB server image to qualify (resolved to a digest) + default: cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb:trunk + required: true + +permissions: + contents: read + +jobs: + bridge: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + concurrency: + group: hosted-2dc-${{ github.ref }} + cancel-in-progress: true + env: + WORKLOAD_MEMORY_LIMIT: 1G + WORKLOAD_CPU_LIMIT: '1.00' + WORKLOAD_CURRENT_IMAGE: py-slo-bridge:current + REQUESTED_IMAGE: ${{ inputs.server_image || 'cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb@sha256:bce9caa6a1a6aac9b513510b46f61c4d4b58c1f88d937bcb54c20dc1bda6c8d4' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: ydb-platform/ydb-slo-action + ref: 54c3c03de1ce8c6a2d8b0828b328bd5db82ef2bb + path: .slo-action + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install the Compose version required by the action + run: | + set -euo pipefail + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL --retry 3 \ + https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \ + -o "$HOME/.docker/cli-plugins/docker-compose" + chmod +x "$HOME/.docker/cli-plugins/docker-compose" + docker compose version + docker compose up --help | grep -F -- --quiet-build + - name: Record runner and resolve server image + run: | + set -euo pipefail + mkdir -p bridge-artifacts + free -h + df -h + docker version + docker compose version + docker pull --platform linux/amd64 "$REQUESTED_IMAGE" + docker image inspect "$REQUESTED_IMAGE" > bridge-artifacts/server-image.json + SERVER_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$REQUESTED_IMAGE") + printf '%s\n' "$SERVER_IMAGE" > bridge-artifacts/server-image.txt + python tests/slo/bridge/prepare.py .slo-action "$SERVER_IMAGE" + - name: Build current Python SDK workload + run: docker build -f tests/slo/Dockerfile -t py-slo-bridge:current . + - name: Run all five Bridge scenarios + id: slo + uses: ./.slo-action/init + timeout-minutes: 20 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workload_name: sync-query + workload_duration: '600' + workload_current_ref: current + workload_current_image: py-slo-bridge:current + workload_current_command: >- + --read-rps 50 --write-rps 10 --read-threads 2 --write-threads 2 + --read-timeout 30000 --write-timeout 30000 + --prefer-primary-pile + bridge_mode: true + disable_compose_profiles: extra-nodes + fail_on_workload_error: true + - name: Verify scenario coverage and workload recovery + if: always() + timeout-minutes: 3 + env: + ACTION_OUTCOME: ${{ steps.slo.outcome }} + run: python tests/slo/bridge/verify.py + - name: Upload qualification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: python-hosted-2dc + path: bridge-artifacts/ + retention-days: 14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 89b1dcb0..68382b47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +* Add opt-in PRIMARY/PROMOTED Bridge pile preference with synchronized-pile fallback for sync and async drivers * Fix automatic refresh of short-lived login tokens by using their JWT expiration time. ## 3.35.0 ## diff --git a/docs/driver.rst b/docs/driver.rst index d5876240..a875d9a7 100644 --- a/docs/driver.rst +++ b/docs/driver.rst @@ -48,6 +48,9 @@ DriverConfig * - ``discovery_request_timeout`` - ``10`` - Timeout in seconds for each discovery request. + * - ``prefer_primary_pile`` + - ``False`` + - Prefer PRIMARY/PROMOTED Bridge endpoints with synchronized-pile fallback; no effect without Bridge metadata. * - ``table_client_settings`` - ``None`` - :class:`~ydb.TableClientSettings` instance to configure the Table service client. @@ -62,6 +65,37 @@ DriverConfig - gRPC-level compression (e.g. ``grpc.Compression.Gzip``). +Preferring the primary Bridge pile +---------------------------------------- + +Set ``prefer_primary_pile`` on the ``DriverConfig`` passed to the sync or async driver. + +.. code-block:: python + + config = ydb.DriverConfig( + endpoint="grpc://discovery.example:2135", + database="/Root/mydb", + prefer_primary_pile=True, + ) + driver = ydb.Driver(config) + +While discovery reports Bridge pile states, the driver tries PRIMARY and PROMOTED +endpoints first, followed by SYNCHRONIZED endpoints, then endpoints in other or +unknown states. This order applies to connection attempts and to unbound requests +for either value of ``use_all_nodes``. Endpoints within a tier have equal priority +regardless of their datacenter. The driver updates priorities when discovery reports a role change. +When a discovery response contains no pile states, the driver resumes its +configured balancing policy for both existing and new connections. +The option controls client-side routing and is disabled by default. + +If the connection cache is full and none of its connections lead to a PRIMARY +or PROMOTED pile, discovery opens a connection to a preferred endpoint before +closing one lower-priority connection. Sessions using that connection can be +interrupted. Once the cache has a preferred connection, additional endpoints +can join as slots become available. Existing sessions keep their node affinity +while their connection is available. + + Creating a Driver ----------------- diff --git a/tests/aio/test_discovery_detect_local_dc.py b/tests/aio/test_discovery_detect_local_dc.py index ad534291..595cfe85 100644 --- a/tests/aio/test_discovery_detect_local_dc.py +++ b/tests/aio/test_discovery_detect_local_dc.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- +import asyncio import pytest from unittest.mock import MagicMock, patch, AsyncMock from ydb import driver, connection +from ydb import _apis, resolver from ydb.aio import pool, _utilities @@ -141,3 +143,207 @@ def mock_init(self, endpoint, driver_config, endpoint_options=None): await discovery.execute_discovery() assert detect_mock.call_count == 0, "detect_local_dc should NOT be called when use_all_nodes=True" + + +class PrimaryConnection: + failed = set() + created = [] + + def __init__(self, endpoint, driver_config, endpoint_options=None): + self.endpoint = endpoint + self.node_id = endpoint_options.node_id + self.callbacks = [] + self.closed = False + self.created.append(self) + + async def connection_ready(self, ready_timeout): + if self.endpoint in self.failed: + raise asyncio.TimeoutError() + + def add_cleanup_callback(self, callback): + self.callbacks.append(callback) + + async def close(self, grace=30): + self.closed = True + for callback in self.callbacks: + callback(self) + + +def primary_result(states, nodes=((1, "A"), (2, "B"))): + message = _apis.ydb_discovery.ListEndpointsResult(self_location="local") + for node_id, pile in nodes: + message.endpoints.add( + address=f"node{node_id}", + port=2135, + node_id=node_id, + location="local" if pile == "A" else "remote", + bridge_pile_name=pile, + ) + for name, state in states.items(): + message.pile_states.add(pile_name=name, state=state) + response = _apis.ydb_discovery.ListEndpointsResponse() + response.operation.ready = True + response.operation.status = 400000 + response.operation.result.Pack(message) + return resolver.DiscoveryResult.from_response(None, response, use_all_nodes=True) + + +def primary_discovery(monkeypatch, result, enabled=True, use_all_nodes=True, failed=()): + PrimaryConnection.failed, PrimaryConnection.created = set(failed), [] + monkeypatch.setattr(pool, "Connection", PrimaryConnection) + config = driver.DriverConfig("grpc://test:2135", "/local", use_all_nodes=use_all_nodes, prefer_primary_pile=enabled) + discovery = pool.Discovery(pool.ConnectionsCache(use_all_nodes), config) + discovery._resolver.resolve = AsyncMock(return_value=result) + return discovery + + +@pytest.mark.asyncio +async def test_primary_role_change_and_affinity(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1} + assert (await discovery._cache.get(connection.EndpointKey("node2:2135", 2))).node_id == 2 + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {2} + assert len(PrimaryConnection.created) == 2 + assert not any(conn.closed for conn in PrimaryConnection.created) + assert (await discovery._cache.get(connection.EndpointKey("node1:2135", 1))).node_id == 1 + await (await discovery._cache.get()).close() + assert (await discovery._cache.get()).node_id == 1 + + +@pytest.mark.asyncio +async def test_primary_off_and_ordinary_cluster_locality(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), enabled=False) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1, 2} + discovery = primary_discovery(monkeypatch, primary_result({}), use_all_nodes=False) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1} + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}) + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 2 + discovery._resolver.resolve.return_value = primary_result({}) + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_all_nodes,detect_local_dc", [(True, True), (False, False), (False, True)]) +async def test_empty_pile_states_restore_configured_balancing(monkeypatch, use_all_nodes, detect_local_dc): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), use_all_nodes=use_all_nodes) + discovery._driver_config.detect_local_dc = detect_local_dc + detector = AsyncMock(return_value="remote") + monkeypatch.setattr(_utilities, "detect_local_dc", detector) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(2)} == {1} + detector.assert_not_awaited() + + ordinary = primary_result({}) + ordinary.self_location = "local" if detect_local_dc else "remote" + discovery._resolver.resolve.return_value = ordinary + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(2)} == ({1, 2} if use_all_nodes else {2}) + assert len(PrimaryConnection.created) == 2 and not any(conn.closed for conn in PrimaryConnection.created) + + +@pytest.mark.asyncio +async def test_primary_promoted_and_unknown_states(monkeypatch): + discovery = primary_discovery( + monkeypatch, + primary_result({"A": 1, "B": 2, "C": 3, "D": 99}, ((1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "missing"))), + ) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1, 2} + await (await discovery._cache.get(connection.EndpointKey("node1:2135", 1))).close() + await (await discovery._cache.get(connection.EndpointKey("node2:2135", 2))).close() + assert (await discovery._cache.get()).node_id == 3 + await (await discovery._cache.get()).close() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {4, 5} + + +@pytest.mark.asyncio +async def test_primary_full_cache_admission_and_failed_connection(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert await discovery.execute_discovery() and discovery._cache.size == 9 + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}, old + ((10, "B"),)) + PrimaryConnection.failed = {"node10:2135"} + assert await discovery.execute_discovery() + assert discovery._cache.size == 9 + assert not any(conn.closed for conn in PrimaryConnection.created if conn.node_id != 10) + PrimaryConnection.failed.clear() + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 10 + assert discovery._cache.size == 9 + created_count = len(PrimaryConnection.created) + assert await discovery.execute_discovery() + assert len(PrimaryConnection.created) == created_count + + +@pytest.mark.asyncio +async def test_full_cache_evicts_only_for_first_preferred_connection(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + nodes = old + ((10, "B"), (11, "B")) + discovery = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert await discovery.execute_discovery() + + discovery._resolver.resolve.return_value = primary_result({"A": 1, "B": 3}, nodes) + assert await discovery.execute_discovery() + assert len(PrimaryConnection.created) == 9 and not any(conn.closed for conn in PrimaryConnection.created) + + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 2}, nodes) + assert await discovery.execute_discovery() + preferred = await discovery._cache.get() + assert preferred.node_id in (10, 11) + assert await discovery.execute_discovery() + assert await discovery._cache.get() is preferred + assert len(PrimaryConnection.created) == 10 + assert sum(conn.closed for conn in PrimaryConnection.created) == 1 + + +@pytest.mark.asyncio +async def test_primary_alias_retirement_keeps_node_affinity(): + cache = pool.ConnectionsCache() + options = connection.EndpointOptions(node_id=1) + old = PrimaryConnection("old:2135", None, options) + new = PrimaryConnection("new:2135", None, options) + cache.add(old) + cache.add(new) + await old.close() + assert await cache.get(connection.EndpointKey(None, 1)) is new + + +@pytest.mark.asyncio +async def test_primary_pile_preference_is_config_only(monkeypatch): + from ydb import topic + from ydb.aio import driver as aio_driver, table, scheme, coordination + + seen = [] + monkeypatch.setattr(pool.ConnectionPool, "__init__", lambda self, config: seen.append(config)) + for module, name in ( + (topic, "TopicClientAsyncIO"), + (table, "TableClient"), + (scheme, "SchemeClient"), + (coordination, "CoordinationClient"), + ): + monkeypatch.setattr(module, name, MagicMock()) + config = aio_driver.DriverConfig("grpc://test:2135", "/local", prefer_primary_pile=True) + aio_driver.Driver(config) + assert seen[-1].prefer_primary_pile + with pytest.raises(TypeError, match="DriverConfig"): + aio_driver.Driver(config, prefer_primary_pile=False) + with pytest.raises(TypeError, match="DriverConfig"): + aio_driver.Driver(endpoint="grpc://test:2135", database="/local", prefer_primary_pile=True) + assert config.prefer_primary_pile + + +@pytest.mark.asyncio +async def test_primary_discovery_cancellation_closes_candidate(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + monkeypatch.setattr(PrimaryConnection, "connection_ready", AsyncMock(side_effect=asyncio.CancelledError)) + with pytest.raises(asyncio.CancelledError): + await discovery.execute_discovery() + assert len(PrimaryConnection.created) == 1 + assert PrimaryConnection.created[0].closed diff --git a/tests/slo/README.md b/tests/slo/README.md index 88d46e72..6821ee48 100644 --- a/tests/slo/README.md +++ b/tests/slo/README.md @@ -1,5 +1,21 @@ # SLO workload +## GitHub-hosted Bridge SLO + +`.github/workflows/slo-2dc.yml` runs the standard five Bridge scenarios from the +pinned SLO Action v2 on `ubuntu-24.04`. The ordinary `sync-query` workload runs for +10 minutes with `--prefer-primary-pile` enabled. + +The runner uses two storage and two database nodes, `erasure=none` and SectorMap disks. +The adapter bounds resources and cold-start readiness time, restricts bootstrap DNS +to active compute nodes, and sets a 30-second gap between fault scenarios to fit the +10-minute workload. The upstream scenario scripts and SDK discovery intervals are unchanged. + +The job requires all five scenarios to complete, successful recent reads and writes, +and no workload failure, OOM or unexpected container restart. Transient operation +errors are recorded. Server image metadata, workload and chaos logs, final container +states and counters are retained in the `python-hosted-2dc` artifact for 14 days. + SLO is the type of test where app based on ydb-sdk is tested against falling YDB cluster nodes, tablets, network (that is possible situations for distributed DBs with hundreds of nodes) diff --git a/tests/slo/bridge/.gitignore b/tests/slo/bridge/.gitignore new file mode 100644 index 00000000..c18dd8d8 --- /dev/null +++ b/tests/slo/bridge/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tests/slo/bridge/prepare.py b/tests/slo/bridge/prepare.py new file mode 100644 index 00000000..b0b14bdb --- /dev/null +++ b/tests/slo/bridge/prepare.py @@ -0,0 +1,73 @@ +"""Configure a pinned Action checkout for a bounded GitHub-hosted run.""" + +import argparse +import json +import subprocess +from pathlib import Path + + +def prepare(action, image): + deploy = action.resolve() / "deploy" + compose = deploy / "compose.bridge.yml" + rendered = subprocess.check_output( + [ + "docker", + "compose", + "-f", + str(compose), + "config", + "--no-interpolate", + "--format", + "json", + ], + text=True, + ) + config = json.loads(rendered) + for service in config["services"].values(): + if "build" in service and service["build"]["context"].endswith("/ydb"): + service["build"].setdefault("args", {})["YDB_IMAGE"] = image + service["mem_limit"] = "2g" + for name in ("workload-current", "workload-baseline"): + config["services"][name]["deploy"]["resources"]["reservations"]["memory"] = "256m" + service = config["services"][name] + hosts = service.get("extra_hosts", []) + if isinstance(hosts, dict): + hosts = [ + f"{host}:{address}" + for host, addresses in hosts.items() + for address in (addresses if isinstance(addresses, list) else [addresses]) + ] + service["extra_hosts"] = [host for host in hosts if not host.startswith(("ydb:", "ydb="))] + [ + "ydb:172.28.0.11", + "ydb:172.28.0.12", + ] + chaos = config["services"]["chaos-monkey"] + environment = chaos["environment"] + if isinstance(environment, list): + environment = {item.partition("=")[0]: item.partition("=")[2] if "=" in item else None for item in environment} + chaos["environment"] = {**environment, "CHAOS_SCENARIO_DELAY": "30"} + for filename in ("config.yaml", "config.bridge.yaml"): + path = deploy / "ydb/rootfs/opt/ydb.tech/ydbd/cfg" / filename + text = path.read_text() + if text.count("cpu_count: 4") != 1 or text.count("hard_limit_bytes: 17179869184") != 1: + raise RuntimeError(f"Unexpected resource configuration: {path}") + path.write_text( + text.replace("cpu_count: 4", "cpu_count: 1").replace( + "hard_limit_bytes: 17179869184", "hard_limit_bytes: 1610612736" + ) + ) + readiness = deploy / "ydb/rootfs/opt/ydb.tech/scripts/ydbd/check-readiness.sh" + text = readiness.read_text() + if text.count("local max_attempts=30") != 2: + raise RuntimeError("Unexpected readiness implementation") + readiness.write_text(text.replace("local max_attempts=30", "local max_attempts=150", 1)) + compose.write_text(json.dumps(config, indent=2) + "\n") + subprocess.run(["docker", "compose", "-f", str(compose), "config", "--quiet"], check=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("action", type=Path) + parser.add_argument("image") + args = parser.parse_args() + prepare(args.action, args.image) diff --git a/tests/slo/bridge/verify.py b/tests/slo/bridge/verify.py new file mode 100644 index 00000000..14257cb4 --- /dev/null +++ b/tests/slo/bridge/verify.py @@ -0,0 +1,124 @@ +"""Verify completion and recovery of the GitHub-hosted Bridge workload.""" + +import json +import math +import os +from pathlib import Path +import subprocess +import urllib.parse +import urllib.request + +SCENARIOS = ( + "01-switchover", + "02-takedown-synchronized", + "03-takedown-primary", + "04-failover-synchronized", + "05-failover-primary", +) + + +def command(*args, check=True, timeout=90, stderr=subprocess.STDOUT): + return subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=stderr, check=check, timeout=timeout).stdout + + +def check_chaos(log, state, restarts): + errors = [] + if state["Status"] != "exited" or state["ExitCode"] != 0 or restarts or "Chaos Monkey finished" not in log: + errors.append("Chaos sequence did not finish without a process restart") + for scenario in SCENARIOS: + if f"Running scenario: {scenario}" not in log: + errors.append(f"Scenario not executed: {scenario}") + if log.count("] Recovered: ") != len(SCENARIOS): + errors.append("Expected five completed chaos recoveries") + return errors + + +def main(): + artifacts = Path("bridge-artifacts") + artifacts.mkdir(exist_ok=True) + result = {"errors": [], "action_outcome": os.environ.get("ACTION_OUTCOME")} + for name in ("ydb-workload-current", "ydb-chaos-monkey"): + (artifacts / f"{name}.log").write_text(command("docker", "logs", name, check=False)) + try: + (artifacts / "healthcheck.json").write_text( + command( + "docker", + "exec", + "ydb-storage-1", + "/opt/ydb.tech/ydb/bin/ydb", + "-e", + "grpc://localhost:2136", + "monitoring", + "healthcheck", + "--format", + "json", + check=False, + timeout=30, + ) + ) + except Exception as error: + (artifacts / "healthcheck-error.txt").write_text(repr(error)) + try: + if result["action_outcome"] != "success": + result["errors"].append("Action did not complete successfully") + chaos_info = json.loads(command("docker", "inspect", "ydb-chaos-monkey"))[0] + chaos = chaos_info["State"] + log = (artifacts / "ydb-chaos-monkey.log").read_text() + result["errors"].extend(check_chaos(log, chaos, chaos_info["RestartCount"])) + result["completed_scenarios"] = len(SCENARIOS) if not result["errors"] else None + ip = command( + "docker", + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + "ydb-prometheus", + ).strip() + + def metric(expression): + url = f"http://{ip}:9090/api/v1/query?" + urllib.parse.urlencode({"query": expression}) + with urllib.request.urlopen(url, timeout=15) as response: + data = json.load(response) + if data["status"] != "success": + raise RuntimeError(data) + return sum(float(item["value"][1]) for item in data["data"]["result"]) + + observations = {} + for operation in ("read", "write"): + selector = f'sdk_operations_success_total{{ref="current",operation_type="{operation}"}}' + count, recent = metric(f"sum({selector})"), metric(f"sum(increase({selector}[120s]))") + observations[operation], observations[f"recent_{operation}"] = count, recent + if not math.isfinite(count) or not math.isfinite(recent) or count <= 0 or recent <= 0: + result["errors"].append(f"No recent successful native {operation} observations") + observations["errors"] = metric('sum(sdk_operations_failure_total{ref="current"})') + result["native_observations"] = observations + containers = json.loads( + command( + "docker", + "inspect", + "ydb-storage-1", + "ydb-storage-2", + "ydb-database-1", + "ydb-database-2", + "ydb-workload-current", + ) + ) + result["container_states"] = { + node["Name"]: {"state": node["State"], "restarts": node["RestartCount"]} for node in containers + } + if any(node["State"]["OOMKilled"] or node["RestartCount"] for node in containers): + result["errors"].append("A container was OOM killed or unexpectedly restarted") + for node in containers: + if node["Name"] == "/ydb-workload-current" and ( + node["State"]["Status"] != "exited" or node["State"]["ExitCode"] != 0 + ): + result["errors"].append(f"Workload process did not exit successfully: {node['Name']}") + except Exception as error: + result["errors"].append(repr(error)) + result["passed"] = not result["errors"] + (artifacts / "result.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/slo/requirements.txt b/tests/slo/requirements.txt index 68f516f1..d9021cc5 100644 --- a/tests/slo/requirements.txt +++ b/tests/slo/requirements.txt @@ -4,6 +4,6 @@ hdrhistogram==0.10.3 # OpenTelemetry (OTLP/HTTP exporter) # NOTE: OpenTelemetry 1.39.1 requires Python >= 3.9. -opentelemetry-api==1.45.1 +opentelemetry-api==1.39.1 opentelemetry-sdk==1.39.1 opentelemetry-exporter-otlp-proto-http==1.39.1 diff --git a/tests/slo/src/options.py b/tests/slo/src/options.py index 200c3c55..9f289df4 100644 --- a/tests/slo/src/options.py +++ b/tests/slo/src/options.py @@ -1,4 +1,10 @@ import argparse +import ydb + + +def get_driver_config(args): + options = {"prefer_primary_pile": True} if getattr(args, "prefer_primary_pile", False) else {} + return ydb.DriverConfig(args.endpoint, database=args.db, grpc_keep_alive_timeout=5000, **options) def add_common_options(parser): @@ -7,6 +13,7 @@ def add_common_options(parser): parser.add_argument("-t", "--table-name", default="key_value", help="Table name") parser.add_argument("--debug", action="store_true", help="Enable debug logging") parser.add_argument("--async", action="store_true", help="Use async mode for operations") + parser.add_argument("--prefer-primary-pile", action="store_true", help="Prefer PRIMARY/PROMOTED Bridge endpoints") def make_table_create_parser(subparsers): diff --git a/tests/slo/src/root_runner.py b/tests/slo/src/root_runner.py index 2c8c2e75..5334a7ff 100644 --- a/tests/slo/src/root_runner.py +++ b/tests/slo/src/root_runner.py @@ -5,6 +5,7 @@ from typing import Dict from core.metrics import WORKLOAD +from options import get_driver_config from runners.topic_runner import TopicRunner from runners.table_runner import TableRunner from runners.base import BaseRunner @@ -45,11 +46,7 @@ def run_command(self, args): def _run_sync_command(self, args, runner_instance, command): """Run command in synchronous mode""" - driver_config = ydb.DriverConfig( - args.endpoint, - database=args.db, - grpc_keep_alive_timeout=5000, - ) + driver_config = get_driver_config(args) with ydb.Driver(driver_config) as driver: driver.wait(timeout=300) @@ -71,11 +68,7 @@ def _run_sync_command(self, args, runner_instance, command): async def _run_async_command(self, args, runner_instance, command): """Run command in asynchronous mode""" - driver_config = ydb.DriverConfig( - args.endpoint, - database=args.db, - grpc_keep_alive_timeout=5000, - ) + driver_config = get_driver_config(args) async with ydb.aio.Driver(driver_config) as driver: await driver.wait(timeout=300) diff --git a/tests/test_discovery_detect_local_dc.py b/tests/test_discovery_detect_local_dc.py index 874ffba7..1552add8 100644 --- a/tests/test_discovery_detect_local_dc.py +++ b/tests/test_discovery_detect_local_dc.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- -from unittest.mock import Mock, MagicMock, patch +from unittest.mock import AsyncMock, Mock, MagicMock, patch + +import pytest from ydb import driver, pool, _utilities, connection +from ydb import _apis, resolver +from ydb.aio import resolver as aio_resolver class MockEndpointInfo: @@ -118,3 +122,265 @@ def test_detect_local_dc_skipped_when_use_all_nodes_true(): discovery.execute_discovery() assert detect_mock.call_count == 0, "detect_local_dc should NOT be called when use_all_nodes=True" + + +class PrimaryConnection: + def __init__(self, endpoint, node_id): + self.endpoint, self.node_id = endpoint, node_id + self.closed = False + self.callbacks = [] + + def add_cleanup_callback(self, callback): + self.callbacks.append(callback) + + def close(self): + self.closed = True + for callback in self.callbacks: + callback(self) + + +def primary_response(states, nodes): + message = _apis.ydb_discovery.ListEndpointsResult(self_location="local") + for node_id, pile in nodes: + message.endpoints.add( + address=f"node{node_id}", + port=2135, + node_id=node_id, + location="local" if pile == "A" else "remote", + bridge_pile_name=pile, + ) + for name, state in states.items(): + message.pile_states.add(pile_name=name, state=state) + response = _apis.ydb_discovery.ListEndpointsResponse() + response.operation.ready = True + response.operation.status = 400000 + response.operation.result.Pack(message) + return response + + +def primary_result(states, nodes=((1, "A"), (2, "B"))): + response = primary_response(states, nodes) + return resolver.DiscoveryResult.from_response(None, response, use_all_nodes=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_all_nodes", [False, True]) +@pytest.mark.parametrize("is_async", [False, True]) +async def test_bridge_discovery_orders_connections_by_pile(monkeypatch, use_all_nodes, is_async): + response = primary_response( + {"A": 3, "B": 3, "C": 2, "D": 1}, ((1, "unknown"), (2, "B"), (3, "A"), (4, "C"), (5, "D")) + ) + config = driver.DriverConfig("grpc://test:2135", "/local", use_all_nodes=use_all_nodes, prefer_primary_pile=True) + module = aio_resolver if is_async else resolver + discovery = module.DiscoveryEndpointsResolver(config) + monkeypatch.setattr(resolver.random, "shuffle", lambda endpoints: endpoints.sort(key=lambda info: info.node_id)) + + def list_endpoints(request, stub, method, wrap_result, *, settings, wrap_args=()): + return wrap_result(None, response, *wrap_args) + + if is_async: + bootstrap = AsyncMock(side_effect=list_endpoints) + monkeypatch.setattr(aio_resolver.conn_impl, "Connection", Mock(return_value=bootstrap)) + result = await discovery.resolve() + else: + bootstrap = Mock(side_effect=list_endpoints) + monkeypatch.setattr(connection.Connection, "ready_factory", Mock(return_value=bootstrap)) + result = discovery.resolve() + + assert [info.node_id for info in result.endpoints] == [4, 5, 2, 3, 1] + + +@pytest.mark.parametrize("states,prefer_primary_pile", [({}, True), ({"A": 3, "B": 1}, False)]) +@pytest.mark.parametrize("use_all_nodes,expected", [(True, [1, 2]), (False, [2, 1])]) +def test_discovery_keeps_ordinary_order_without_bridge_policy( + monkeypatch, states, prefer_primary_pile, use_all_nodes, expected +): + response = primary_response(states, ((1, "B"), (2, "A"))) + monkeypatch.setattr(resolver.random, "shuffle", lambda endpoints: endpoints.sort(key=lambda info: info.node_id)) + result = resolver.DiscoveryResult.from_response(None, response, use_all_nodes, prefer_primary_pile) + assert [info.node_id for info in result.endpoints] == expected + + +def primary_discovery(monkeypatch, result, enabled=True, use_all_nodes=True, failed=()): + created = [] + + def ready(endpoint, config, timeout, endpoint_options): + if endpoint in failed: + return None + conn = PrimaryConnection(endpoint, endpoint_options.node_id) + created.append(conn) + return conn + + monkeypatch.setattr(connection.Connection, "ready_factory", ready) + config = driver.DriverConfig("grpc://test:2135", "/local", use_all_nodes=use_all_nodes, prefer_primary_pile=enabled) + discovery = pool.Discovery(pool.ConnectionsCache(use_all_nodes), config) + discovery._resolver = MagicMock() + discovery._resolver.context_resolve.return_value.__enter__.return_value = result + return discovery, created + + +def test_primary_role_change_updates_existing_connections_and_affinity(monkeypatch): + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1} + assert discovery._cache.get(connection.EndpointKey("node2:2135", 2)).node_id == 2 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({"A": 3, "B": 1}) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {2} + assert len(created) == 2 and not any(conn.closed for conn in created) + assert discovery._cache.get(connection.EndpointKey("node1:2135", 1)).node_id == 1 + discovery._cache.get().close() + assert discovery._cache.get().node_id == 1 + + +def test_primary_option_is_off_by_default_and_ordinary_clusters_keep_locality(monkeypatch): + assert not driver.DriverConfig("grpc://test:2135").prefer_primary_pile + discovery, _ = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), enabled=False) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1, 2} + discovery, created = primary_discovery(monkeypatch, primary_result({}), use_all_nodes=False) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1} + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({"A": 3, "B": 1}) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 2 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({}) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 1 + assert len(created) == 2 + + +@pytest.mark.parametrize("use_all_nodes,detect_local_dc", [(True, True), (False, False), (False, True)]) +def test_empty_pile_states_restore_configured_balancing(monkeypatch, use_all_nodes, detect_local_dc): + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), use_all_nodes=use_all_nodes) + discovery._driver_config.detect_local_dc = detect_local_dc + detector = Mock(return_value="remote") + monkeypatch.setattr(_utilities, "detect_local_dc", detector) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(2)} == {1} + detector.assert_not_called() + + ordinary = primary_result({}) + ordinary.self_location = "local" if detect_local_dc else "remote" + discovery._resolver.context_resolve.return_value.__enter__.return_value = ordinary + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(2)} == ({1, 2} if use_all_nodes else {2}) + assert len(created) == 2 and not any(conn.closed for conn in created) + + +def test_primary_promoted_synchronized_and_unknown_fallback(monkeypatch): + result = primary_result({"A": 1, "B": 2, "C": 3, "D": 99}, ((1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "missing"))) + discovery, _ = primary_discovery(monkeypatch, result) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1, 2} + discovery._cache.get(connection.EndpointKey("node1:2135", 1)).close() + discovery._cache.get(connection.EndpointKey("node2:2135", 2)).close() + assert discovery._cache.get().node_id == 3 + discovery._cache.get().close() + assert {discovery._cache.get().node_id for _ in range(10)} == {4, 5} + + +def test_primary_new_endpoint_enters_full_cache(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert discovery.execute_discovery() and discovery._cache.size == 9 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result( + {"A": 3, "B": 1}, old + ((10, "B"),) + ) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 10 + assert discovery._cache.size == 9 and sum(conn.closed for conn in created) == 1 + assert discovery.execute_discovery() + assert discovery._cache.size == 9 and len(created) == 10 + + +def test_failed_primary_does_not_evict_full_fallback_cache(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery, created = primary_discovery( + monkeypatch, primary_result({"A": 3, "B": 1}, old + ((10, "B"),)), failed={"node10:2135"} + ) + assert discovery.execute_discovery() + assert discovery._cache.size == 9 and not any(conn.closed for conn in created) + assert discovery._cache.get().node_id in range(1, 10) + + +def test_full_cache_evicts_only_for_first_preferred_connection(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + nodes = old + ((10, "B"), (11, "B")) + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert discovery.execute_discovery() + + resolve = discovery._resolver.context_resolve.return_value.__enter__ + resolve.return_value = primary_result({"A": 1, "B": 3}, nodes) + assert discovery.execute_discovery() + assert len(created) == 9 and not any(conn.closed for conn in created) + + resolve.return_value = primary_result({"A": 3, "B": 2}, nodes) + assert discovery.execute_discovery() + preferred = discovery._cache.get() + assert preferred.node_id in (10, 11) + assert discovery.execute_discovery() + assert discovery._cache.get() is preferred + assert len(created) == 10 + assert sum(conn.closed for conn in created) == 1 + + +def test_primary_connection_retirement_preserves_other_address_node_affinity(): + cache = pool.ConnectionsCache() + old, new = PrimaryConnection("old:2135", 1), PrimaryConnection("new:2135", 1) + cache.add(old) + cache.add(new) + old.close() + assert cache.get(connection.EndpointKey(None, 1)) is new + + +def test_primary_pile_preference_is_config_only(monkeypatch): + from ydb import topic, table, scheme, coordination + + seen = [] + monkeypatch.setattr(pool.ConnectionPool, "__init__", lambda self, config: seen.append(config)) + for module, name in ( + (topic, "TopicClient"), + (table, "TableClient"), + (scheme, "SchemeClient"), + (coordination, "CoordinationClient"), + ): + monkeypatch.setattr(module, name, MagicMock()) + config = driver.DriverConfig("grpc://test:2135", "/local", prefer_primary_pile=True) + driver.Driver(config) + assert seen[-1].prefer_primary_pile + with pytest.raises(TypeError, match="DriverConfig"): + driver.Driver(config, prefer_primary_pile=False) + with pytest.raises(TypeError, match="DriverConfig"): + driver.Driver(endpoint="grpc://test:2135", database="/local", prefer_primary_pile=True) + assert config.prefer_primary_pile + + +def test_slo_primary_option_keeps_legacy_baseline_compatible(monkeypatch): + from tests.slo.src import options + + args = options.get_root_parser().parse_args( + ["table-run", "grpc://host:2136", "/Root/testdb", "--prefer-primary-pile"] + ) + assert options.get_driver_config(args).prefer_primary_pile + original = driver.DriverConfig + + def legacy_config(endpoint, database, grpc_keep_alive_timeout): + return original(endpoint, database=database, grpc_keep_alive_timeout=grpc_keep_alive_timeout) + + monkeypatch.setattr(options.ydb, "DriverConfig", legacy_config) + args = options.get_root_parser().parse_args(["table-run", "grpc://host:2136", "/Root/testdb"]) + assert not args.prefer_primary_pile + assert not options.get_driver_config(args).prefer_primary_pile + + +def test_bridge_slo_requires_all_completed_scenarios(): + from tests.slo.bridge.verify import SCENARIOS, check_chaos + + log = "\n".join(f"Running scenario: {name}\n[time] Recovered: {name}" for name in SCENARIOS) + log += "\nChaos Monkey finished" + state = {"Status": "exited", "ExitCode": 0} + assert not check_chaos(log, state, 0) + assert check_chaos(log.replace("Running scenario: 05-failover-primary", "skipped"), state, 0) + assert check_chaos(log.replace("] Recovered: 05-failover-primary", "] incomplete"), state, 0) + assert check_chaos(log, state, 1) diff --git a/ydb/aio/driver.py b/ydb/aio/driver.py index 88221e94..980c9c4f 100644 --- a/ydb/aio/driver.py +++ b/ydb/aio/driver.py @@ -58,6 +58,9 @@ def __init__( credentials: Optional["Credentials"] = None, **kwargs: Any, ) -> None: + if "prefer_primary_pile" in kwargs: + raise TypeError("Set prefer_primary_pile on DriverConfig") + from .. import topic # local import for prevent cycle import error from . import coordination # local import for prevent cycle import error diff --git a/ydb/aio/pool.py b/ydb/aio/pool.py index 15027436..cf367891 100644 --- a/ydb/aio/pool.py +++ b/ydb/aio/pool.py @@ -3,6 +3,7 @@ import asyncio import logging import random +import grpc from typing import Any, Callable, Optional, Tuple, TYPE_CHECKING, cast from ydb import issues @@ -47,15 +48,7 @@ async def get( # async version with different Connection type if preferred_endpoint is not None and preferred_endpoint.endpoint in self.connections: return self.connections[preferred_endpoint.endpoint] # type: ignore[return-value] - for conn_lst in self.conn_lst_order: - try: - endpoint, connection = conn_lst.popitem(last=False) - conn_lst[endpoint] = connection - return connection # type: ignore[return-value] - except KeyError: - continue - - raise issues.ConnectionLost("Couldn't find valid connection") + return self._get_balanced_connection() # type: ignore[return-value] def add(self, connection: Optional[Connection], preferred: bool = False) -> bool: # type: ignore[override] # async Connection type if connection is None: @@ -81,7 +74,8 @@ def complete_discovery(self, error: Optional[Exception]) -> None: self._fast_fail_event.set() def remove(self, connection: Connection) -> None: # type: ignore[override] # async Connection type - self.connections_by_node_id.pop(connection.node_id, None) + if cast(Any, self.connections_by_node_id.get(connection.node_id)) is connection: + self.connections_by_node_id.pop(connection.node_id, None) self.preferred.pop(connection.endpoint, None) self.connections.pop(connection.endpoint, None) self.outdated.pop(connection.endpoint, None) @@ -137,19 +131,21 @@ async def execute_discovery(self) -> bool: if resolve_details is None: return False - resolved_endpoints = set( - endpoint + endpoint_locations = { + endpoint: resolved_endpoint.location for resolved_endpoint in resolve_details.endpoints for endpoint, endpoint_options in resolved_endpoint.endpoints_with_options() - ) + } for cached_endpoint in self._cache.values(): - if cached_endpoint.endpoint not in resolved_endpoints: + if cached_endpoint.endpoint not in endpoint_locations: self._cache.make_outdated(cached_endpoint) local_dc = resolve_details.self_location + priorities = resolve_details.bridge_priorities(True) if self._driver_config.prefer_primary_pile else None + restoring_locality = self._cache.set_bridge_priorities(priorities, endpoint_locations, local_dc) # Detect local DC using TCP latency if enabled and preferred is meaningful - if self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: + if priorities is None and self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: # Use only endpoints that match the SSL requirements for detection ssl_filtered_endpoints = [ endpoint @@ -164,6 +160,8 @@ async def execute_discovery(self) -> bool: ) if detected_location: local_dc = detected_location + if restoring_locality and local_dc != resolve_details.self_location: + self._cache.set_local_dc(endpoint_locations, local_dc) self.logger.info( "Detected local DC via TCP latency: %s (server reported: %s)", local_dc, @@ -200,13 +198,24 @@ async def execute_discovery(self) -> bool: endpoint, endpoint_options, ) in resolved_endpoint.endpoints_with_options(): - if self._cache.size >= self._max_size or self._cache.already_exists(endpoint): + if self._cache.already_exists(endpoint) or not self._cache.connection_slot(endpoint, self._max_size)[0]: continue ready_connection = Connection(endpoint, self._driver_config, endpoint_options=endpoint_options) - await ready_connection.connection_ready(ready_timeout=self._ready_timeout) - - self._cache.add(ready_connection, preferred) + try: + await ready_connection.connection_ready(ready_timeout=self._ready_timeout) + except BaseException as error: + await ready_connection.close(grace=0) + if priorities is not None and isinstance(error, (asyncio.TimeoutError, grpc.RpcError)): + continue + raise + allowed, replacement = self._cache.connection_slot(endpoint, self._max_size) + if allowed: + if replacement is not None: + self._cache.make_outdated(replacement) + self._cache.add(ready_connection, preferred) + else: + await ready_connection.close() await self._cache.cleanup_outdated() return self._cache.size > 0 diff --git a/ydb/aio/resolver.py b/ydb/aio/resolver.py index fb8c4417..81977791 100644 --- a/ydb/aio/resolver.py +++ b/ydb/aio/resolver.py @@ -1,3 +1,4 @@ +from functools import partial from typing import Any, Optional from . import connection as conn_impl @@ -47,7 +48,7 @@ async def resolve(self) -> Optional[DiscoveryResult]: # type: ignore[override] _list_endpoints_request_factory(self._driver_config), _apis.DiscoveryService.Stub, _apis.DiscoveryService.ListEndpoints, - DiscoveryResult.from_response, + partial(DiscoveryResult.from_response, prefer_primary_pile=self._driver_config.prefer_primary_pile), settings=settings_impl.BaseRequestSettings().with_timeout(self._ready_timeout), ) diff --git a/ydb/driver.py b/ydb/driver.py index 98864980..134b7059 100644 --- a/ydb/driver.py +++ b/ydb/driver.py @@ -113,6 +113,7 @@ class DriverConfig(object): "compression", "disable_discovery", "detect_local_dc", + "prefer_primary_pile", "_additional_sdk_headers", ) @@ -141,6 +142,7 @@ def __init__( disable_discovery: bool = False, detect_local_dc: bool = False, *, + prefer_primary_pile: bool = False, _additional_sdk_headers: Tuple[str, ...] = (), ) -> None: """ @@ -167,6 +169,9 @@ def __init__( server-provided self_location. **Note**: This option only affects endpoint selection when use_all_nodes=False.\ When use_all_nodes=True (default), all endpoints are used regardless of detected location. :param _additional_sdk_headers: Reserved for SDK integrations (e.g. dbapi, sqlalchemy). Do not use in application code. + :param prefer_primary_pile: Prefer PRIMARY/PROMOTED Bridge endpoints, with synchronized-pile fallback.\ + Disabled by default and has no effect without Bridge metadata. Overrides locality while in Bridge mode;\ + admitting the first preferred connection to a full cache can interrupt sessions on the evicted connection. """ self.endpoint = endpoint @@ -196,6 +201,7 @@ def __init__( self.compression = compression self.disable_discovery = disable_discovery self.detect_local_dc = detect_local_dc + self.prefer_primary_pile = prefer_primary_pile self._additional_sdk_headers = _additional_sdk_headers def set_database(self, database: str) -> "DriverConfig": @@ -314,6 +320,9 @@ def __init__( :param database: A database path :param credentials: A credentials. If not specified credentials constructed by default. """ + if "prefer_primary_pile" in kwargs: + raise TypeError("Set prefer_primary_pile on DriverConfig") + from . import topic # local import for prevent cycle import error from . import coordination # local import for prevent cycle import error diff --git a/ydb/pool.py b/ydb/pool.py index 2ba1c33a..63581ff5 100644 --- a/ydb/pool.py +++ b/ydb/pool.py @@ -7,7 +7,7 @@ from concurrent import futures import collections import random -from typing import Any, Callable, ContextManager, List, Optional, Set, Tuple, TYPE_CHECKING +from typing import Any, Callable, ContextManager, Dict, List, Optional, Set, Tuple, TYPE_CHECKING from . import connection as connection_impl, issues, resolver, _utilities, tracing from .observability.tracing import SpanName, create_ydb_span @@ -37,6 +37,56 @@ def __init__(self, use_all_nodes: bool = False, tracer: tracing.Tracer = tracing self.use_all_nodes = use_all_nodes self.conn_lst_order = (self.connections,) if self.use_all_nodes else (self.preferred, self.connections) self.fast_fail_subscriptions: Set["futures.Future[None]"] = set() + self._bridge_priorities: Optional[Dict[str, int]] = None + + def set_bridge_priorities( + self, priorities: Optional[Dict[str, int]], endpoint_locations: Dict[str, str], local_dc: str + ) -> bool: + """Return whether cached connections switched back to local-DC routing.""" + with self.lock: + restoring_locality = self._bridge_priorities is not None and priorities is None and not self.use_all_nodes + self._bridge_priorities = priorities + if restoring_locality: + self.set_local_dc(endpoint_locations, local_dc) + return restoring_locality + + def set_local_dc(self, endpoint_locations: Dict[str, str], local_dc: str) -> None: + with self.lock: + self.preferred.clear() + self.preferred.update( + (endpoint, conn) + for endpoint, conn in self.connections.items() + if endpoint_locations.get(endpoint) == local_dc + ) + + def connection_slot(self, endpoint: str, max_size: int) -> Tuple[bool, Optional[Connection]]: + with self.lock: + active = [conn for key, conn in self.connections.items() if key not in self.outdated] + if len(active) < max_size: + return True, None + priorities = self._bridge_priorities + if not active or priorities is None or priorities.get(endpoint) != 0: + return False, None + if any(priorities.get(conn.endpoint) == 0 for conn in active): + return False, None + return True, max(active, key=lambda conn: priorities.get(conn.endpoint, 2)) + + def _get_balanced_connection(self) -> Connection: + if self._bridge_priorities is not None and self.connections: + priorities = self._bridge_priorities + best = min(priorities.get(endpoint, 2) for endpoint in self.connections) + for endpoint, conn in self.connections.items(): + if priorities.get(endpoint, 2) == best: + self.connections.move_to_end(endpoint) + return conn + for conn_lst in self.conn_lst_order: + try: + endpoint, conn = conn_lst.popitem(last=False) + conn_lst[endpoint] = conn + return conn + except KeyError: + continue + raise issues.ConnectionLost("Couldn't find valid connection") def add(self, connection: Optional[Connection], preferred: bool = False) -> bool: if connection is None: @@ -142,20 +192,14 @@ def get(self, preferred_endpoint: Optional[EndpointKey] = None) -> Connection: if preferred_endpoint is not None and preferred_endpoint.endpoint in self.connections: return self.connections[preferred_endpoint.endpoint] - for conn_lst in self.conn_lst_order: - try: - endpoint, connection = conn_lst.popitem(last=False) - conn_lst[endpoint] = connection - tracing.trace(self.tracer, {"found_in_lists": True}) - return connection - except KeyError: - continue - - raise issues.ConnectionLost("Couldn't find valid connection") + connection = self._get_balanced_connection() + tracing.trace(self.tracer, {"found_in_lists": True}) + return connection def remove(self, connection: Connection) -> None: with self.lock: - self.connections_by_node_id.pop(connection.node_id, None) + if self.connections_by_node_id.get(connection.node_id) is connection: + self.connections_by_node_id.pop(connection.node_id, None) self.preferred.pop(connection.endpoint, None) self.connections.pop(connection.endpoint, None) self.outdated.pop(connection.endpoint, None) @@ -228,19 +272,21 @@ def execute_discovery(self) -> bool: if resolve_details is None: return False - resolved_endpoints = set( - endpoint + endpoint_locations = { + endpoint: resolved_endpoint.location for resolved_endpoint in resolve_details.endpoints for endpoint, endpoint_options in resolved_endpoint.endpoints_with_options() - ) + } for cached_endpoint in self._cache.values(): - if cached_endpoint.endpoint not in resolved_endpoints: + if cached_endpoint.endpoint not in endpoint_locations: self._cache.make_outdated(cached_endpoint) local_dc = resolve_details.self_location + priorities = resolve_details.bridge_priorities(True) if self._driver_config.prefer_primary_pile else None + restoring_locality = self._cache.set_bridge_priorities(priorities, endpoint_locations, local_dc) # Detect local DC using TCP latency if enabled and preferred is meaningful - if self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: + if priorities is None and self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: # Use only endpoints that match the SSL requirements for detection ssl_filtered_endpoints = [ endpoint @@ -255,6 +301,8 @@ def execute_discovery(self) -> bool: ) if detected_location: local_dc = detected_location + if restoring_locality and local_dc != resolve_details.self_location: + self._cache.set_local_dc(endpoint_locations, local_dc) self.logger.info( "Detected local DC via TCP latency: %s (server reported: %s)", local_dc, @@ -291,7 +339,10 @@ def execute_discovery(self) -> bool: endpoint, endpoint_options, ) in resolved_endpoint.endpoints_with_options(): - if self._cache.size >= self._max_size or self._cache.already_exists(endpoint): + if ( + self._cache.already_exists(endpoint) + or not self._cache.connection_slot(endpoint, self._max_size)[0] + ): continue ready_connection = connection_impl.Connection.ready_factory( @@ -300,7 +351,14 @@ def execute_discovery(self) -> bool: self._ready_timeout, endpoint_options=endpoint_options, ) - self._cache.add(ready_connection, preferred) + if ready_connection is not None: + allowed, replacement = self._cache.connection_slot(endpoint, self._max_size) + if allowed: + if replacement is not None: + self._cache.make_outdated(replacement) + self._cache.add(ready_connection, preferred) + else: + ready_connection.close() self._cache.cleanup_outdated() diff --git a/ydb/resolver.py b/ydb/resolver.py index 40a04980..ab5eff4c 100644 --- a/ydb/resolver.py +++ b/ydb/resolver.py @@ -7,14 +7,16 @@ import random import itertools import typing -from typing import Any, ContextManager, List, Optional, Iterator +from typing import Any, ContextManager, Dict, List, Optional, Iterator from . import connection as conn_impl, driver, issues, settings as settings_impl, _apis # Workaround for good IDE and universal for runtime if typing.TYPE_CHECKING: from ._grpc.v4.protos import ydb_discovery_pb2 + from ._grpc.v4.protos import ydb_bridge_common_pb2 else: from ._grpc.common.protos import ydb_discovery_pb2 + from ._grpc.common.protos import ydb_bridge_common_pb2 logger = logging.getLogger(__name__) @@ -31,6 +33,7 @@ class EndpointInfo(object): "ipv6_addrs", "ssl_target_name_override", "node_id", + "bridge_pile_name", ) def __init__(self, endpoint_info: ydb_discovery_pb2.EndpointInfo): @@ -43,6 +46,7 @@ def __init__(self, endpoint_info: ydb_discovery_pb2.EndpointInfo): self.ipv6_addrs = tuple(endpoint_info.ip_v6) self.ssl_target_name_override = endpoint_info.ssl_target_name_override self.node_id = endpoint_info.node_id + self.bridge_pile_name = endpoint_info.bridge_pile_name def endpoints_with_options(self) -> typing.Generator[typing.Tuple[str, conn_impl.EndpointOptions], None, None]: ssl_target_name_override = None @@ -95,9 +99,30 @@ def _list_endpoints_request_factory(connection_params: driver.DriverConfig) -> _ class DiscoveryResult(object): - def __init__(self, self_location: str, endpoints: "list[EndpointInfo]"): + def __init__( + self, self_location: str, endpoints: "list[EndpointInfo]", pile_states: Optional[Dict[str, int]] = None + ): self.self_location = self_location self.endpoints = endpoints + self.pile_states = pile_states or {} + + def _pile_priority(self, info: EndpointInfo) -> int: + state = self.pile_states.get(info.bridge_pile_name) + if state in (ydb_bridge_common_pb2.PileState.PRIMARY, ydb_bridge_common_pb2.PileState.PROMOTED): + return 0 + if state == ydb_bridge_common_pb2.PileState.SYNCHRONIZED: + return 1 + return 2 + + def bridge_priorities(self, enabled: bool) -> Optional[Dict[str, int]]: + if not enabled or not self.pile_states: + return None + priorities = {} + for info in self.endpoints: + priority = self._pile_priority(info) + for endpoint, _ in info.endpoints_with_options(): + priorities[endpoint] = priority + return priorities def __str__(self): return "DiscoveryResult " % ( @@ -114,6 +139,7 @@ def from_response( rpc_state: conn_impl._RpcState, response: ydb_discovery_pb2.ListEndpointsResponse, use_all_nodes: bool = False, + prefer_primary_pile: bool = False, ) -> DiscoveryResult: issues._process_response(response.operation) message = _apis.ydb_discovery.ListEndpointsResult() @@ -126,20 +152,14 @@ def from_response( else: unique_different_set.add(EndpointInfo(info)) - result: List[EndpointInfo] = [] - local_endpoints = list(unique_local_set) - different_endpoints = list(unique_different_set) - if use_all_nodes: - result.extend(local_endpoints) - result.extend(different_endpoints) - random.shuffle(result) - else: - random.shuffle(local_endpoints) - random.shuffle(different_endpoints) - result.extend(local_endpoints) - result.extend(different_endpoints) - - return cls(message.self_location, result) + endpoints = list(unique_local_set) + list(unique_different_set) + result = cls(message.self_location, endpoints, {pile.pile_name: pile.state for pile in message.pile_states}) + random.shuffle(endpoints) + if prefer_primary_pile and result.pile_states: + endpoints.sort(key=result._pile_priority) + elif not use_all_nodes: + endpoints.sort(key=lambda info: info.location != message.self_location) + return result class DiscoveryEndpointsResolver(object): @@ -197,7 +217,7 @@ def context_resolve(self) -> Iterator[Optional[DiscoveryResult]]: _apis.DiscoveryService.ListEndpoints, DiscoveryResult.from_response, settings=settings_impl.BaseRequestSettings().with_timeout(self._ready_timeout), - wrap_args=(self._driver_config.use_all_nodes,), + wrap_args=(self._driver_config.use_all_nodes, self._driver_config.prefer_primary_pile), ) self._add_debug_details(