SDK rpc proxy: a relayed node error does not trip the breaker

apps/web/src/core/sdk-init.ts

@@ -98,7 +98,7 @@ function startRpcProxyReport(): void {
     const r = s.fallbackByReason;
     const line =
       `served=${s.served} fallback=${s.fallback} ` +
-      `(status=${r.status} timeout=${r.timeout} transport=${r.transport} ` +
+      `(status=${r.status} rpcerror=${r.rpcerror ?? 0} timeout=${r.timeout} transport=${r.transport} ` +
       `validate=${r.validate} parse=${r.parse}) skipped=${s.skipped}`;
     if (line === last) return;
     last = line;

apps/web/src/specs/core/sdk-init-proxy.spec.ts

@@ -12,7 +12,7 @@ const stats = {
   served: 0,
   fallback: 0,
   skipped: 0,
-  fallbackByReason: { status: 0, timeout: 0, transport: 0, validate: 0, parse: 0 }
+  fallbackByReason: { status: 0, rpcerror: 0, timeout: 0, transport: 0, validate: 0, parse: 0 }
 };
 const manager = {
   setPrivateApiHost: vi.fn(),
@@ -101,21 +101,22 @@ describe("sdk-init server rpc proxy", () => {
       await vi.advanceTimersByTimeAsync(REPORT_MS);
       expect(log).toHaveBeenCalledTimes(1);
       expect(log).toHaveBeenLastCalledWith(
-        "[rpc-proxy] served=0 fallback=0 (status=0 timeout=0 transport=0 validate=0 parse=0) skipped=0"
+        "[rpc-proxy] served=0 fallback=0 (status=0 rpcerror=0 timeout=0 transport=0 validate=0 parse=0) skipped=0"
       );
 
       // Nothing moved: silence, not a repeat.
       await vi.advanceTimersByTimeAsync(REPORT_MS * 3);
       expect(log).toHaveBeenCalledTimes(1);
 
       stats.served = 41;
-      stats.fallback = 2;
+      stats.fallback = 5;
       stats.fallbackByReason.transport = 2;
+      stats.fallbackByReason.rpcerror = 3;
       stats.skipped = 7;
       await vi.advanceTimersByTimeAsync(REPORT_MS);
       expect(log).toHaveBeenCalledTimes(2);
       expect(log).toHaveBeenLastCalledWith(
-        "[rpc-proxy] served=41 fallback=2 (status=0 timeout=0 transport=2 validate=0 parse=0) skipped=7"
+        "[rpc-proxy] served=41 fallback=5 (status=0 rpcerror=3 timeout=0 transport=2 validate=0 parse=0) skipped=7"
       );
     });
 

packages/sdk/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 2.3.91
+
+### Patch Changes
+
+- SDK rpc proxy: a relayed node error does not trip the breaker (#1629)
+
 ## 2.3.90
 
 ### Patch Changes

packages/sdk/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ecency/sdk",
   "private": false,
-  "version": "2.3.90",
+  "version": "2.3.91",
   "description": "Ecency SDK",
   "repository": {
     "type": "git",

packages/sdk/src/hive-tx/helpers/call.ts

@@ -55,10 +55,17 @@ export const rpcProxyStats = {
   fallback: 0,
   /** Reads that went straight to the nodes because the breaker was open. */
   skipped: 0,
-  fallbackByReason: { status: 0, timeout: 0, transport: 0, validate: 0, parse: 0 } as Record<string, number>
+  fallbackByReason: { status: 0, rpcerror: 0, timeout: 0, transport: 0, validate: 0, parse: 0 } as Record<string, number>
 }
 
-type ProxyMissReason = 'status' | 'timeout' | 'transport' | 'validate' | 'parse'
+/**
+ * `rpcerror` is a 502 tagged `X-Ssr-Cache: RPCERROR`: the proxy reached a node
+ * and relayed the node's own error (a tag or post that does not exist, a bad
+ * argument). The read still falls back so the caller sees the node's answer
+ * unchanged, but the proxy was healthy, so it does not count toward the
+ * breaker; the other reasons do.
+ */
+type ProxyMissReason = 'status' | 'rpcerror' | 'timeout' | 'transport' | 'validate' | 'parse'
 
 class ProxyMiss extends Error {
   constructor(
@@ -128,7 +135,8 @@ async function proxyRpcCall<T>(
       } catch {
         // nothing to release
       }
-      throw new ProxyMiss('status', `proxy answered ${res.status}`)
+      const relayed = res.status === 502 && (res.headers.get('x-ssr-cache') ?? '').toUpperCase() === 'RPCERROR'
+      throw new ProxyMiss(relayed ? 'rpcerror' : 'status', relayed ? 'proxy relayed a node error' : `proxy answered ${res.status}`)
     }
     let result: unknown
     try {
@@ -1465,7 +1473,12 @@ export const callRPC = async <T = any>(
         rpcProxyStats.fallback++
         const reason: string = e instanceof ProxyMiss ? e.reason : 'transport'
         rpcProxyStats.fallbackByReason[reason] = (rpcProxyStats.fallbackByReason[reason] ?? 0) + 1
-        if (++proxyConsecutiveMisses >= proxy.failureThreshold) {
+        if (reason === 'rpcerror') {
+          // A relayed node error is a healthy proxy answer: it closes the
+          // count like a served call. Crawler-made feed URLs produce these in
+          // runs, and counting them opened the breaker on a working proxy.
+          proxyConsecutiveMisses = 0
+        } else if (++proxyConsecutiveMisses >= proxy.failureThreshold) {
           proxyOpenUntil = Date.now() + proxy.cooldownMs
           proxyConsecutiveMisses = 0
         }

packages/sdk/src/hive-tx/helpers/server-rpc-proxy.spec.ts

@@ -231,6 +231,50 @@ describe("server-side RPC proxy", () => {
     expect(urls.filter((u) => u === PROXY)).toHaveLength(3);
   });
 
+  it("a relayed node error falls back but neither opens nor advances the breaker", async () => {
+    setServerRpcProxy({ url: PROXY, headers: {}, timeoutMs: 500, failureThreshold: 2, cooldownMs: 300 });
+    const relayed = () =>
+      new Response(JSON.stringify({ error: "Assert Exception:Tag nosuchtag does not exist" }), {
+        status: 502,
+        headers: { "Content-Type": "application/json", "X-Ssr-Cache": "RPCERROR" }
+      });
+    let answer: () => Response = relayed;
+    const urls: string[] = [];
+    mockFetch(async (input, init) => {
+      urls.push(String(input));
+      if (String(input) === PROXY) return answer();
+      return rpcOk(idOf(init), { from: "node" });
+    });
+    for (let i = 1; i <= 4; i++) {
+      const out = await callRPC("bridge.get_ranked_posts", { sort: "created", tag: `nosuchtag${i}`, limit: 20 });
+      expect(out).toEqual({ from: "node" });
+    }
+    // Four relays in a row, all four reached the proxy: the breaker never opened.
+    expect(urls.filter((u) => u === PROXY)).toHaveLength(4);
+    expect(rpcProxyStats.skipped).toBe(0);
+    expect(rpcProxyStats.fallback).toBe(4);
+    expect(rpcProxyStats.fallbackByReason.rpcerror).toBe(4);
+    expect(rpcProxyStats.fallbackByReason.status).toBe(0);
+
+    // A relay also clears a count a real miss had started: miss, relay, miss
+    // does not reach the threshold of two.
+    answer = () => jsonOk({ error: "down" }, 502);
+    await callRPC("bridge.get_post", { author: "a", permlink: "1" }); // miss 1
+    answer = relayed;
+    await callRPC("bridge.get_post", { author: "a", permlink: "2" }); // relay, count reset
+    answer = () => jsonOk({ error: "down" }, 502);
+    await callRPC("bridge.get_post", { author: "a", permlink: "3" }); // miss 1 again (the relay reset it)
+    await callRPC("bridge.get_post", { author: "a", permlink: "4" }); // miss 2: tried, then opens
+    expect(rpcProxyStats.skipped).toBe(0);
+    expect(urls.filter((u) => u === PROXY)).toHaveLength(8);
+
+    // Bare 502s without the tag are still status misses and still count: the
+    // breaker is open now.
+    await callRPC("bridge.get_post", { author: "a", permlink: "5" }); // skipped
+    expect(rpcProxyStats.skipped).toBe(1);
+    expect(rpcProxyStats.fallbackByReason.status).toBe(3);
+  });
+
   it("a served call closes the breaker count", async () => {
     setServerRpcProxy({ url: PROXY, headers: {}, timeoutMs: 500, failureThreshold: 2, cooldownMs: 300 });
     let fail = true;

packages/wallets/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @ecency/wallets
 
+## 5.0.91
+
+### Patch Changes
+
+- Updated dependencies []:
+  - @ecency/sdk@2.3.91
+
 ## 5.0.90
 
 ### Patch Changes

packages/wallets/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ecency/wallets",
   "private": false,
-  "version": "5.0.90",
+  "version": "5.0.91",
   "description": "Ecency wallets",
   "repository": {
     "type": "git",