diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 43981306..e994dfb4 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -45,6 +45,11 @@ jobs: cd stellar-lend cargo fmt --all -- --check + - name: Check event schema compliance + run: | + cd stellar-lend + python3 scripts/check_event_schemas.py + - name: Run clippy run: | cd stellar-lend diff --git a/docs/event-indexing.md b/docs/event-indexing.md index bb45d63c..a0e363fd 100644 --- a/docs/event-indexing.md +++ b/docs/event-indexing.md @@ -2,6 +2,13 @@ This guide documents the event surfaces that exist in the current StellarLend workspace and how to consume them safely. +The canonical event compatibility rules live in +[`stellar-lend/docs/event-schema.md`](../stellar-lend/docs/event-schema.md), with the +machine-readable schema in +[`stellar-lend/docs/event-schema.v1.json`](../stellar-lend/docs/event-schema.v1.json). +CI runs `stellar-lend/scripts/check_event_schemas.py` to keep `#[contractevent]` definitions, +legacy manual topics, and the schema version constants in sync. + It is written around the code that exists today: - `stellar-lend/contracts/hello-world` is the main protocol contract currently used by the API. @@ -26,6 +33,8 @@ For this repo there are two emission styles: Important decoding notes: +- Decoded rows should include `_schema_version: 1` and `_event_topic` metadata when persisted by + indexers. The prototype `indexing_system` annotates parsed payloads with these fields. - `Option
` means the native asset can appear as `None`. - Amounts are raw integer amounts in the token's smallest unit. - Risk parameters and reserve factors are basis points unless the source struct says otherwise. diff --git a/package.json b/package.json index 49794f0a..ec94cab1 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "deploy:verify": "bash scripts/verify-deployment.sh", "deploy:rollback": "bash scripts/rollback.sh", "verify:contract": "bash scripts/verify-contract.sh", + "contracts:event-schema:check": "python stellar-lend/scripts/check_event_schemas.py", "mutation:api": "npm --workspace api run mutation:test", "mutation:api:check": "npm --workspace api run mutation:check" }, diff --git a/stellar-lend/Cargo.toml b/stellar-lend/Cargo.toml index e1cea76e..5afa9c8b 100644 --- a/stellar-lend/Cargo.toml +++ b/stellar-lend/Cargo.toml @@ -13,7 +13,7 @@ members = [ "contracts/institutional-wallet", "contracts/migration-hub", ] -exclude = ["fuzz"] +exclude = ["fuzz", "indexing_system"] [workspace.dependencies] soroban-sdk = "23.4.1" diff --git a/stellar-lend/contracts/common/src/event_schema.rs b/stellar-lend/contracts/common/src/event_schema.rs new file mode 100644 index 00000000..02415003 --- /dev/null +++ b/stellar-lend/contracts/common/src/event_schema.rs @@ -0,0 +1,27 @@ +//! Shared event schema constants for StellarLend contracts and indexers. +//! +//! Version 1 is intentionally additive: existing event topics and payloads remain stable while +//! new documentation, indexer metadata, and CI checks describe the canonical schema. + +use soroban_sdk::contracttype; + +pub const EVENT_SCHEMA_VERSION: u32 = 1; +pub const EVENT_SCHEMA_VERSION_FIELD: &str = "_schema_version"; +pub const EVENT_TOPIC_FIELD: &str = "_event_topic"; +pub const MAX_SOROBAN_EVENT_TOPICS: u32 = 4; + +#[contracttype] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub enum EventSchemaVersion { + V1 = 1, +} + +#[contracttype] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub enum EventPayloadFormat { + ContractEvent = 1, + SingleValue = 2, + LegacyTuple = 3, +} diff --git a/stellar-lend/contracts/common/src/lib.rs b/stellar-lend/contracts/common/src/lib.rs index b806f230..774469b7 100644 --- a/stellar-lend/contracts/common/src/lib.rs +++ b/stellar-lend/contracts/common/src/lib.rs @@ -1,6 +1,7 @@ #![no_std] #![allow(deprecated)] pub mod cache; +pub mod event_schema; pub mod events; pub mod message_bus; pub mod shared_types; diff --git a/stellar-lend/docs/event-schema.md b/stellar-lend/docs/event-schema.md new file mode 100644 index 00000000..c6155f23 --- /dev/null +++ b/stellar-lend/docs/event-schema.md @@ -0,0 +1,41 @@ +# StellarLend Event Schema v1 + +Schema Version: 1 + +This document defines the event compatibility rules for the StellarLend Soroban contracts. The +machine-readable companion file is `event-schema.v1.json`. + +## Compatibility Rules + +- Existing event topics are append-only. Do not rename or remove a topic without adding a migration note. +- New typed events should use `#[contractevent]` and lower snake case topic names. +- Manual `env.events().publish(...)` calls are allowed only for legacy compatibility and must be listed in the schema JSON. +- Event topics must fit Soroban's four-topic limit, including static topics and `#[topic]` fields. +- Field names must use lower snake case and payload fields should stay append-only. +- Amounts are raw token units. Percentages use basis points where `10000` is `100%`. +- Indexers should persist raw topics and decoded payloads, then annotate decoded rows with `_schema_version` and `_event_topic`. + +## Common Field Semantics + +| Field | Meaning | +| --- | --- | +| `user` | End-user account that initiated or owns a position-changing action. | +| `actor` | Authorized protocol account that performed an administrative action. | +| `caller` | Authenticated account that invoked an administrative or governance entrypoint. | +| `asset` | Soroban token contract address; `None` represents native asset where supported. | +| `amount` | Raw integer amount in the asset's smallest unit. | +| `fee` | Raw integer fee in the asset's smallest unit. | +| `timestamp` | Ledger timestamp in seconds. | +| `*_bps` | Basis points; `10000` is `100%`. | + +## Versioning + +Version 1 is additive. It standardizes documentation, CI validation, and indexer metadata without +changing existing runtime event topics or payload shapes. + +For a future breaking change: + +1. Add a new `event-schema.vN.json` file. +2. Keep the old schema available for historical decoding. +3. Add an explicit migration section to `docs/event-indexing.md`. +4. Update indexers to decode both versions during the migration window. diff --git a/stellar-lend/docs/event-schema.v1.json b/stellar-lend/docs/event-schema.v1.json new file mode 100644 index 00000000..4f4a3fba --- /dev/null +++ b/stellar-lend/docs/event-schema.v1.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "topic_style": "lower_snake_case", + "contracts": [ + { + "name": "hello-world", + "source_globs": ["contracts/hello-world/src/**/*.rs"], + "active": true + }, + { + "name": "lending", + "source_globs": ["contracts/lending/src/**/*.rs"], + "active": true + }, + { + "name": "amm", + "source_globs": ["contracts/amm/src/**/*.rs"], + "active": true + }, + { + "name": "bridge", + "source_globs": ["contracts/bridge/src/**/*.rs"], + "active": true + }, + { + "name": "stablecoin", + "source_globs": ["contracts/stablecoin/src/**/*.rs"], + "active": true + }, + { + "name": "common", + "source_globs": ["contracts/common/src/**/*.rs"], + "active": true + } + ], + "allow_overloaded_topics": [ + { + "contract": "lending", + "topic": "deposit_event", + "reason": "Vault deposits and borrow-collateral deposits predate this schema and are distinguished by payload shape." + } + ], + "legacy_manual_events": [ + { + "contract": "hello-world", + "topic": "bridge", + "format": "legacy_tuple" + }, + { + "contract": "hello-world", + "topic": "mon_hlth", + "format": "legacy_tuple" + }, + { + "contract": "hello-world", + "topic": "mon_init", + "format": "legacy_tuple" + }, + { + "contract": "hello-world", + "topic": "mon_perf", + "format": "legacy_tuple" + }, + { + "contract": "hello-world", + "topic": "mon_sec", + "format": "legacy_tuple" + }, + { + "contract": "hello-world", + "topic": "rep_add", + "format": "legacy_tuple" + }, + { + "contract": "hello-world", + "topic": "rep_del", + "format": "legacy_tuple" + }, + { + "contract": "lending", + "topic": "bad_debt", + "format": "legacy_tuple" + }, + { + "contract": "lending", + "topic": "bad_debt_recovered", + "format": "legacy_tuple" + }, + { + "contract": "lending", + "topic": "yield_compounded", + "format": "legacy_tuple" + }, + { + "contract": "lending", + "topic": "yield_deposit", + "format": "legacy_tuple" + }, + { + "contract": "stablecoin", + "topic": "stablecoin_initialized", + "format": "legacy_tuple" + }, + { + "contract": "stablecoin", + "topic": "shutdown_set", + "format": "legacy_tuple" + }, + { + "contract": "stablecoin", + "topic": "reserve_ratio_set", + "format": "legacy_tuple" + }, + { + "contract": "stablecoin", + "topic": "collateral_deposited", + "format": "legacy_tuple" + }, + { + "contract": "stablecoin", + "topic": "stablecoin_minted", + "format": "legacy_tuple" + }, + { + "contract": "stablecoin", + "topic": "stablecoin_redeemed", + "format": "legacy_tuple" + } + ], + "field_semantics": { + "user": "End-user account that initiated or owns the position-changing operation.", + "actor": "Authorized protocol account that performed an administrative operation.", + "caller": "Authenticated account that invoked an administrative or governance entrypoint.", + "asset": "Soroban token contract address; null/None represents the native asset where supported.", + "amount": "Raw integer amount in the asset's smallest unit.", + "fee": "Raw integer fee amount in the asset's smallest unit.", + "timestamp": "Ledger timestamp in seconds.", + "bps": "Basis points, where 10000 is 100%." + } +} diff --git a/stellar-lend/indexing_system/src/lib.rs b/stellar-lend/indexing_system/src/lib.rs index 9d7a2655..b8099c1f 100644 --- a/stellar-lend/indexing_system/src/lib.rs +++ b/stellar-lend/indexing_system/src/lib.rs @@ -8,6 +8,7 @@ pub mod parallel_indexer; pub mod parser; pub mod query; pub mod repository; +pub mod schema; #[cfg(test)] pub mod tests; @@ -20,10 +21,16 @@ pub use metrics::{IndexingMetrics, MetricsSnapshot}; pub use models::{ CreateEvent, Event, EventQuery, EventStats, EventUpdate, IndexingMetadata, UpdateType, }; -pub use parallel_indexer::{BlockProcessor, BlockRangeResult, BlockRangeTask, ParallelIndexer, StateManager}; +pub use parallel_indexer::{ + BlockProcessor, BlockRangeResult, BlockRangeTask, ParallelIndexer, StateManager, +}; pub use parser::{create_erc20_abi, EventParser}; pub use query::QueryService; pub use repository::EventRepository; +pub use schema::{ + annotate_event_payload, load_standard_event_schema, normalize_event_topic, EventSchemaDocument, + STANDARD_EVENT_SCHEMA_VERSION, +}; pub fn init_tracing() { use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; @@ -77,12 +84,9 @@ pub async fn run_migrations(database_url: &str) -> IndexerResult<()> { /// /// This is the main initialization function that should be called /// to set up the entire indexing system. -pub async fn initialize_system(config: &Config) -> IndexerResult<( - EventRepository, - CacheService, - QueryService, - IndexerService, -)> { +pub async fn initialize_system( + config: &Config, +) -> IndexerResult<(EventRepository, CacheService, QueryService, IndexerService)> { info!("Initializing indexing system..."); // Run database migrations first diff --git a/stellar-lend/indexing_system/src/parser.rs b/stellar-lend/indexing_system/src/parser.rs index b3f9ef97..4622f81e 100644 --- a/stellar-lend/indexing_system/src/parser.rs +++ b/stellar-lend/indexing_system/src/parser.rs @@ -1,6 +1,7 @@ /// Event parser for decoding smart contract events use crate::error::{IndexerError, IndexerResult}; use crate::models::CreateEvent; +use crate::schema::annotate_event_payload; use ethers::abi::{Abi, Event as AbiEvent, RawLog}; use ethers::prelude::*; use serde_json::Value; @@ -100,6 +101,7 @@ impl EventParser { let value = self.token_to_json(¶m.value)?; event_data.insert(param.name, value); } + annotate_event_payload(&mut event_data, &event_def.name); Ok(Some(CreateEvent { contract_address, diff --git a/stellar-lend/indexing_system/src/schema.rs b/stellar-lend/indexing_system/src/schema.rs new file mode 100644 index 00000000..dd064073 --- /dev/null +++ b/stellar-lend/indexing_system/src/schema.rs @@ -0,0 +1,105 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +pub const STANDARD_EVENT_SCHEMA_VERSION: u32 = 1; +pub const EVENT_SCHEMA_VERSION_FIELD: &str = "_schema_version"; +pub const EVENT_TOPIC_FIELD: &str = "_event_topic"; +pub const EVENT_SCHEMA_JSON: &str = include_str!("../../docs/event-schema.v1.json"); + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct EventSchemaDocument { + pub schema_version: u32, + pub topic_style: String, + pub contracts: Vec, + #[serde(default)] + pub allow_overloaded_topics: Vec, + #[serde(default)] + pub legacy_manual_events: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ContractEventSchema { + pub name: String, + pub source_globs: Vec, + pub active: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OverloadedTopic { + pub contract: String, + pub topic: String, + pub reason: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct LegacyManualEvent { + pub contract: String, + pub topic: String, + pub format: String, +} + +pub fn load_standard_event_schema() -> Result { + serde_json::from_str(EVENT_SCHEMA_JSON) +} + +pub fn normalize_event_topic(event_name: &str) -> String { + let mut out = String::new(); + let chars: Vec = event_name.chars().collect(); + + for (idx, ch) in chars.iter().copied().enumerate() { + if ch == '-' || ch == ' ' { + out.push('_'); + continue; + } + + if ch.is_ascii_uppercase() { + let prev = idx.checked_sub(1).and_then(|i| chars.get(i)).copied(); + let next = chars.get(idx + 1).copied(); + let starts_word = prev + .map(|p| p.is_ascii_lowercase() || p.is_ascii_digit()) + .unwrap_or(false) + || (prev.is_some() && next.map(|n| n.is_ascii_lowercase()).unwrap_or(false)); + if starts_word { + out.push('_'); + } + out.push(ch.to_ascii_lowercase()); + } else { + out.push(ch); + } + } + + out +} + +pub fn annotate_event_payload(payload: &mut Map, event_name: &str) { + payload.insert( + EVENT_SCHEMA_VERSION_FIELD.to_string(), + Value::Number(STANDARD_EVENT_SCHEMA_VERSION.into()), + ); + payload.insert( + EVENT_TOPIC_FIELD.to_string(), + Value::String(normalize_event_topic(event_name)), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn loads_embedded_schema_document() { + let schema = load_standard_event_schema().expect("schema json must parse"); + assert_eq!(schema.schema_version, STANDARD_EVENT_SCHEMA_VERSION); + assert!(schema.contracts.iter().any(|c| c.name == "hello-world")); + } + + #[test] + fn normalizes_pascal_case_event_names() { + assert_eq!(normalize_event_topic("DepositEvent"), "deposit_event"); + assert_eq!( + normalize_event_topic("AMMOperationEvent"), + "amm_operation_event" + ); + assert_eq!(normalize_event_topic("bridge-deposit"), "bridge_deposit"); + } +} diff --git a/stellar-lend/scripts/check_event_schemas.py b/stellar-lend/scripts/check_event_schemas.py new file mode 100644 index 00000000..526e1b25 --- /dev/null +++ b/stellar-lend/scripts/check_event_schemas.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Validate StellarLend Soroban event schema conventions. + +The checker is intentionally source-based so it runs quickly in CI and catches drift before +generated contract specs or indexers fall out of sync. +""" + +from __future__ import annotations + +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +CONTRACTS = ROOT / "contracts" +SCHEMA_PATH = ROOT / "docs" / "event-schema.v1.json" +DOC_PATH = ROOT / "docs" / "event-schema.md" +COMMON_SCHEMA_PATH = ROOT / "contracts" / "common" / "src" / "event_schema.rs" +INDEXER_SCHEMA_PATH = ROOT / "indexing_system" / "src" / "schema.rs" + +EVENT_STRUCT_RE = re.compile( + r"(?P(?:#\[[^\]]+\]\s*)*)pub\s+struct\s+(?P[A-Za-z0-9_]+)\s*\{(?P.*?)\n\}", + re.DOTALL, +) +FIELD_RE = re.compile(r"pub\s+(?P[a-zA-Z0-9_]+)\s*:\s*(?P[^,]+),") +TOPICS_RE = re.compile(r"topics\s*=\s*\[(?P[^\]]*)\]") +STRING_RE = re.compile(r'"([^"]+)"') +PUBLISH_RE = re.compile(r"env\.events\(\)\s*\.publish\s*\((?P.*?)\);", re.DOTALL) + + +@dataclass(frozen=True) +class EventDef: + contract: str + path: Path + name: str + topic: str + fields: tuple[str, ...] + topic_field_count: int + static_topic_count: int + + +def snake_case(name: str) -> str: + chars: list[str] = [] + previous_lower_or_digit = False + for ch in name: + if ch in {"-", " "}: + chars.append("_") + previous_lower_or_digit = False + continue + if ch.isupper(): + if previous_lower_or_digit: + chars.append("_") + chars.append(ch.lower()) + previous_lower_or_digit = False + else: + chars.append(ch) + previous_lower_or_digit = ch.islower() or ch.isdigit() + return "".join(chars) + + +def contract_name(path: Path) -> str: + rel = path.relative_to(CONTRACTS) + return rel.parts[0] + + +def is_lower_snake(value: str) -> bool: + return bool(re.fullmatch(r"[a-z][a-z0-9_]*", value)) + + +def explicit_topics(attrs: str) -> list[str]: + match = TOPICS_RE.search(attrs) + if not match: + return [] + return STRING_RE.findall(match.group("topics")) + + +def parse_event_structs() -> list[EventDef]: + events: list[EventDef] = [] + for path in sorted(CONTRACTS.rglob("*.rs")): + text = path.read_text(encoding="utf-8") + for match in EVENT_STRUCT_RE.finditer(text): + attrs = match.group("attrs") + if "contractevent" not in attrs: + continue + name = match.group("name") + body = match.group("body") + topics = explicit_topics(attrs) + fields = tuple(field.group("name") for field in FIELD_RE.finditer(body)) + topic_fields = body.count("#[topic]") + topic = topics[0] if topics else snake_case(name) + events.append( + EventDef( + contract=contract_name(path), + path=path, + name=name, + topic=topic, + fields=fields, + topic_field_count=topic_fields, + static_topic_count=len(topics), + ) + ) + return events + + +def parse_manual_topics() -> set[tuple[str, str]]: + topics: set[tuple[str, str]] = set() + for path in sorted(CONTRACTS.rglob("*.rs")): + text = path.read_text(encoding="utf-8") + if "env.events().publish" not in text: + continue + for match in PUBLISH_RE.finditer(text): + strings = STRING_RE.findall(match.group("args")) + if strings: + topics.add((contract_name(path), strings[0])) + return topics + + +def load_schema() -> dict: + return json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + + +def check_constants(schema_version: int, errors: list[str]) -> None: + common = COMMON_SCHEMA_PATH.read_text(encoding="utf-8") + indexer = INDEXER_SCHEMA_PATH.read_text(encoding="utf-8") + expected_common = f"EVENT_SCHEMA_VERSION: u32 = {schema_version}" + expected_indexer = f"STANDARD_EVENT_SCHEMA_VERSION: u32 = {schema_version}" + if expected_common not in common: + errors.append(f"{COMMON_SCHEMA_PATH}: missing `{expected_common}`") + if expected_indexer not in indexer: + errors.append(f"{INDEXER_SCHEMA_PATH}: missing `{expected_indexer}`") + + +def main() -> int: + schema = load_schema() + schema_version = int(schema["schema_version"]) + allowed_contracts = {entry["name"] for entry in schema["contracts"] if entry.get("active")} + allowed_overloads = { + (entry["contract"], entry["topic"]) for entry in schema.get("allow_overloaded_topics", []) + } + legacy_manual = { + (entry["contract"], entry["topic"]) for entry in schema.get("legacy_manual_events", []) + } + + errors: list[str] = [] + check_constants(schema_version, errors) + + if f"Schema Version: {schema_version}" not in DOC_PATH.read_text(encoding="utf-8"): + errors.append(f"{DOC_PATH}: missing schema version heading") + + events = parse_event_structs() + if not events: + errors.append("No #[contractevent] structs found") + + seen_by_contract_topic: dict[tuple[str, str], EventDef] = {} + for event in events: + rel = event.path.relative_to(ROOT) + if event.contract not in allowed_contracts: + errors.append(f"{rel}: contract `{event.contract}` is not listed in schema JSON") + if not event.name.endswith("Event"): + errors.append(f"{rel}: event struct `{event.name}` must end with `Event`") + if not is_lower_snake(event.topic): + errors.append(f"{rel}: topic `{event.topic}` must be lower_snake_case") + if event.static_topic_count + event.topic_field_count > 4: + errors.append( + f"{rel}: `{event.name}` uses {event.static_topic_count + event.topic_field_count} topics" + ) + for field in event.fields: + if not is_lower_snake(field): + errors.append(f"{rel}: field `{event.name}.{field}` must be lower_snake_case") + + key = (event.contract, event.topic) + previous = seen_by_contract_topic.get(key) + if previous and previous.fields != event.fields and key not in allowed_overloads: + errors.append( + f"{rel}: topic `{event.topic}` is overloaded by `{previous.name}` and `{event.name}`" + ) + seen_by_contract_topic[key] = event + + manual_topics = parse_manual_topics() + for contract, topic in sorted(manual_topics): + if (contract, topic) not in legacy_manual: + errors.append( + f"contracts/{contract}: manual event topic `{topic}` must be listed in legacy_manual_events" + ) + + if errors: + print("Event schema compliance failed:") + for error in errors: + print(f" - {error}") + return 1 + + print( + f"Event schema v{schema_version} OK: {len(events)} typed events, " + f"{len(manual_topics)} legacy manual topics" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())