Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/analysis/post_run/logos_messaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,12 +214,12 @@ def settings_reliability(line: str) -> Optional[str]:


def relay_statefulsets(metadata: dict) -> List[Tuple[str, int]]:
"""Statefulsets of the messaging nodes and their sizes. The bootstrap only does discovery."""
"""Statefulsets of the messaging nodes and their sizes. Bootstrap and Store nodes have no API."""

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What it means Bootstrap and Store nodes have no API. They do not have messaging API?

stack = metadata["stack"]
return [
(name, count)
for name, count in zip(stack["stateful_sets"], stack["nodes_per_statefulset"])
if "bootstrap" not in name
if not name.startswith(("bootstrap", "store"))
]


Expand Down
138 changes: 116 additions & 22 deletions src/deployments/experiments/logos_messaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,19 @@
import logging
import random
from datetime import timedelta
from typing import List, Literal
from typing import List, Literal, Optional

import aiohttp
from kubernetes.client import V1Probe
from pydantic import BaseModel, ConfigDict, Field, NonNegativeFloat, NonNegativeInt, PositiveInt
from pydantic import (
BaseModel,
ConfigDict,
Field,
NonNegativeFloat,
NonNegativeInt,
PositiveInt,
model_validator,
)

# Project Imports
from src.deployments.core.configs.container import Image
Expand All @@ -24,7 +32,7 @@
from src.deployments.registry import experiment
from src.deployments.waku.bridge import Bridge
from src.deployments.waku.builders.builders import WakuStatefulSetBuilder
from src.deployments.waku.builders.helpers import find_waku_container_config
from src.deployments.waku.builders.helpers import find_waku_command, find_waku_container_config

logger = logging.getLogger(__name__)

Expand All @@ -41,6 +49,18 @@ class ExpConfig(BaseModel):

num_relay_nodes: PositiveInt = 20
num_bootstrap_nodes: PositiveInt = 3
num_store_nodes: NonNegativeInt = 0
"""Store nodes without the Messaging API."""
store_max_connections: Optional[PositiveInt] = None
"""`--max-connections` on the Store nodes, the regression default when unset."""
num_edge_nodes: NonNegativeInt = 0
"""Messaging API nodes in Edge mode, sent to like the relays."""
reliability: bool = False
"""Confirm each send with a Store node."""
idle_before_edge: NonNegativeFloat = 0
"""Wait between the relays and Store nodes being ready and deploying the Edge nodes."""
delay_cold_start: NonNegativeFloat = 0
"""Wait between all nodes being ready and the first send."""
num_messages: NonNegativeInt = 20
msg_size_kbytes: PositiveInt = 1
delay_after_publish: NonNegativeFloat = 1
Expand All @@ -58,6 +78,14 @@ class ExpConfig(BaseModel):
)
)

@model_validator(mode="after")
def check_store_nodes(self):
if self.reliability and not self.num_store_nodes:
raise ValueError("reliability needs Store nodes")
if self.num_edge_nodes and not self.num_store_nodes:
raise ValueError("Edge nodes need a Store peer to report PartiallyConnected")
Comment on lines +85 to +86

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this still a thing?

return self


class MessagingBridge(Bridge):
interval: Literal["complete", "messages"] = "messages"
Expand All @@ -81,34 +109,75 @@ def build_bootstrap(namespace: str, config: ExpConfig) -> V1Deployable:
)
.with_bootstrap(CMD_TYPE)
.with_env("SERVICE", "zerotesting-bootstrap")
.with_args({"--log-level": config.log_level}, on_duplicate="replace")
.with_args({"--log-level": config.log_level, "--filter": False}, on_duplicate="replace")
.with_binary(LOGOS_DELIVERY_BINARY)
.with_image(config.image)
.build()
)


def build_relays(namespace: str, config: ExpConfig) -> V1Deployable:
args = {
"--log-level": config.log_level,
"--entry-layer": "messaging",
"--reliability": "false",
}
builder = (
def node_builder(
name: str, num_nodes: int, namespace: str, config: ExpConfig, args: dict
) -> WakuStatefulSetBuilder:
return (
WakuStatefulSetBuilder()
.with_waku_config(name="relay-0", namespace=namespace, num_nodes=config.num_relay_nodes)
.with_waku_config(name=name, namespace=namespace, num_nodes=num_nodes)
.with_regression(CMD_TYPE, config.num_bootstrap_nodes)
.with_env("SERVICE", "zerotesting-bootstrap")
.with_args(args, on_duplicate="replace")
.with_binary(LOGOS_DELIVERY_BINARY)
.with_image(config.image)
.with_env("RELAY_MSG_INFO_LOGS", "1")
.with_args({"--log-level": config.log_level, **args}, on_duplicate="replace")
)


def messaging_builder(
name: str, num_nodes: int, namespace: str, config: ExpConfig, args: dict
) -> WakuStatefulSetBuilder:
"""Messaging API nodes that subscribe to the content topic at startup."""
args = {
"--entry-layer": "messaging",
"--reliability": str(config.reliability).lower(),
**args,
}
builder = node_builder(name, num_nodes, namespace, config, args)
container = find_waku_container_config(builder.config)
container.command_config.insert_commands([subscribe_on_start(config.content_topic)], index=0)
return builder


def build_relays(namespace: str, config: ExpConfig) -> V1Deployable:
builder = messaging_builder("relay-0", config.num_relay_nodes, namespace, config, {})
container = find_waku_container_config(builder.config)
container.with_readiness_probe(
messaging_readiness_probe(container.readiness_probe, config.content_topic), overwrite=True
)
return builder.build()
return builder.with_binary(LOGOS_DELIVERY_BINARY).with_image(config.image).build()


def build_edge_nodes(namespace: str, config: ExpConfig) -> V1Deployable:
"""Edge mode nodes. They find their filter, lightpush and Store peers through discv5."""
builder = messaging_builder(
"edge-0", config.num_edge_nodes, namespace, config, {"--mode": "Edge"}
)
container = find_waku_container_config(builder.config)
command = find_waku_command(container.command_config)
# Edge mode only sets flags that are unset, so an explicit --relay would keep relay on.
command.args = [
arg for arg in command.args if not (isinstance(arg, tuple) and arg[0] == "--relay")
]
container.with_readiness_probe(
edge_readiness_probe(container.readiness_probe, config.content_topic), overwrite=True
)
return builder.with_binary(LOGOS_DELIVERY_BINARY).with_image(config.image).build()


def build_store_nodes(namespace: str, config: ExpConfig) -> V1Deployable:
"""Relays without the Messaging API that archive every message in a Postgres sidecar."""
args = (
{"--max-connections": config.store_max_connections} if config.store_max_connections else {}
)
builder = node_builder("store-0", config.num_store_nodes, namespace, config, args).with_store()
builder.config.namespace = namespace
return builder.with_binary(LOGOS_DELIVERY_BINARY).with_image(config.image).build()


def subscribe_on_start(content_topic: str) -> str:
Expand All @@ -120,17 +189,32 @@ def subscribe_on_start(content_topic: str) -> str:
)


def messaging_readiness_probe(mesh_probe: V1Probe, content_topic: str) -> V1Probe:
"""`mesh_probe` (topic mesh >= dLow), after a subscription check that latches once it passes."""
subscribed = (
def subscription_check(content_topic: str) -> str:
"""Shell check that the node is subscribed to the topic. It latches once it passes."""
return (
f"[ -f /tmp/subscribed ] || {{ curl -sf localhost:{REST_PORT}/messaging/v1/subscriptions"
f" | grep -qF '\"{content_topic}\"' && touch /tmp/subscribed; }} || exit 1;\n"
)


def messaging_readiness_probe(mesh_probe: V1Probe, content_topic: str) -> V1Probe:
"""`mesh_probe` (topic mesh >= dLow), after a subscription check that latches once it passes."""
shell, flag, check = mesh_probe._exec.command
mesh_probe._exec.command = [shell, flag, subscribed + check]
mesh_probe._exec.command = [shell, flag, subscription_check(content_topic) + check]
return mesh_probe


def edge_readiness_probe(probe: V1Probe, content_topic: str) -> V1Probe:
"""`probe` timings, the subscription check, then `/health` at least `PartiallyConnected`."""
connected = (
f"curl -sf localhost:{REST_PORT}/health"
f' | grep -qE \'"connectionStatus":"(Partially)?Connected"\'\n'
)
shell, flag, _ = probe._exec.command
probe._exec.command = [shell, flag, subscription_check(content_topic) + connected]
return probe


@experiment(name="logos-messaging")
class LogosMessagingExperiment(BaseExperiment[ExpConfig]):
"""Relay nodes on the Messaging API, sent to over REST and measured from their logs."""
Expand Down Expand Up @@ -171,12 +255,22 @@ async def _run(self):

await self.deploy(deployment=build_bootstrap(self.namespace, self.config))
relays = build_relays(self.namespace, self.config)
await self.deploy(deployment=relays)
stores = (
[build_store_nodes(self.namespace, self.config)] if self.config.num_store_nodes else []
)
await self.deploy(deployment=[relays, *stores])
senders = [relays]
if self.config.num_edge_nodes:
if not self.dry_run:
await asyncio.sleep(self.config.idle_before_edge)
senders.append(build_edge_nodes(self.namespace, self.config))
await self.deploy(deployment=senders[-1])
if self.dry_run:
self.log_event("dry_run_done")
return

pods = [f"{relays.metadata.name}-{index}" for index in range(relays.spec.replicas)]
await asyncio.sleep(self.config.delay_cold_start)
pods = [f"{s.metadata.name}-{index}" for s in senders for index in range(s.spec.replicas)]
requester = requester_address(self.namespace)
await clear_requester_cache(requester)

Expand Down
Loading