From 18ea04e5b9f047d5ecd6833bdc0e90e97b573f08 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Fri, 28 Aug 2026 23:33:05 +0000 Subject: [PATCH 1/4] release: carry the unit files, restore develop CI, write down the edge Three follow-ups from tonight's release and the incident after it. The artifact now packs production/linux, so a release carries its own unit files. Adopting socket activation needed the socket unit and the service drop-in on the host, and they had to be sent from a workstation checkout that happened to be on the right commit. That is the same class of mistake as building the artifact there, which this workflow exists to stop. Universe CI runs on pushes to develop again. Restricting it to main left direct pushes to develop with no CI at all between one pull request and the next, and develop is pushed to directly here. DEPLOYMENT.md gains the step that was missing and an edge section. The release tool at /usr/local/bin/universe-explorer-release is a copy, not a symlink, so it does not travel with a release; it was found running the previous release's gates against a new one while printing a normal-looking log. The edge section records the two path problems that have already caused incidents: favicon.ico and robots.txt being redirected by an inherited apache rewrite, and the tunnel's single-threaded sshd child landing in an unprotected per-session scope under memory pressure. It also records that operator access degrades before user traffic on that host, because a new SSH connection needs uncached IO that proportional weights cannot rescue while an established session does not. Co-Authored-By: Claude Opus 5 --- .github/workflows/universe-ci.yml | 4 +- .../workflows/universe-release-artifact.yml | 10 ++++- docs/operations/DEPLOYMENT.md | 44 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.github/workflows/universe-ci.yml b/.github/workflows/universe-ci.yml index bed9fb53873..a088a9cc629 100644 --- a/.github/workflows/universe-ci.yml +++ b/.github/workflows/universe-ci.yml @@ -2,7 +2,9 @@ name: Universe CI on: push: - branches: [main] + # develop is pushed to directly, so restricting this to main left those + # pushes with no CI at all between one pull request and the next. + branches: [develop, main] pull_request: branches: [develop, main] workflow_dispatch: diff --git a/.github/workflows/universe-release-artifact.yml b/.github/workflows/universe-release-artifact.yml index 573e4f89157..068a6af965f 100644 --- a/.github/workflows/universe-release-artifact.yml +++ b/.github/workflows/universe-release-artifact.yml @@ -94,11 +94,19 @@ jobs: cp -a backend/package-lock.json "$stage/backend/package-lock.json" cp -a frontend/dist/mempool/browser "$stage/frontend/build" cp -a scripts/universe "$stage/scripts/universe" + # The unit files travel with the release rather than being fetched + # from a checkout that happens to be on the right commit. Adopting + # socket activation needed both of these on the host, and taking + # them from a workstation is the same class of mistake as building + # the artifact there. + mkdir -p "$stage/production" + cp -a production/linux "$stage/production/linux" + test -f "$stage/production/linux/universe-explorer-gateway.socket" test -f "$stage/backend/dist/index.js" test -f "$stage/frontend/build/index.html" test -f "$stage/scripts/universe/gateway.mjs" out="$PWD/mempool-$sha.tar.gz" - tar -czf "$out" -C "$stage" backend frontend scripts + tar -czf "$out" -C "$stage" backend frontend scripts production sha256sum "$out" | tee "$out.sha256" printf 'name=mempool-%s\n' "$sha" >> "$GITHUB_OUTPUT" diff --git a/docs/operations/DEPLOYMENT.md b/docs/operations/DEPLOYMENT.md index b5c9fedd29d..6adcd48e780 100644 --- a/docs/operations/DEPLOYMENT.md +++ b/docs/operations/DEPLOYMENT.md @@ -141,6 +141,22 @@ suite can see that. integration tests against a real database, and the visual matrix. 2. Build on the runner fleet and pack `backend/dist`, `frontend/dist`, and `scripts/universe` into one artifact. +2b. Refresh `/usr/local/bin/universe-explorer-release` from the artifact + before using it. It is a copy, not a symlink, so it does not travel with a + release and will otherwise run the previous release's gates against the new + one while printing a log that looks entirely normal. It was found a release + behind, still carrying gates from before the dependency and rollback guards + existed. + + ```bash + tar -xzOf mempool-.tar.gz scripts/universe/release.sh > /usr/local/bin/universe-explorer-release.new + chmod 0755 /usr/local/bin/universe-explorer-release.new + mv -f /usr/local/bin/universe-explorer-release.new /usr/local/bin/universe-explorer-release + ``` + + Taking it from the tarball rather than a checkout keeps the tool and the + release it installs on the same commit. + 3. `universe-explorer-release install ` unpacks it beside the running release under `/opt/universe-explorer/releases/mempool-/` and hard-links the dependency tree from the release in use. A release directory @@ -297,3 +313,31 @@ Every deployment publishes what it is running: the backend reports `gitCommit` on `/api/v1/backend-info`, and the public `/source` page renders the release SHA, the pinned upstream base, the licence, and the source repository link. A build whose SHA is not published must not ship. + +## The edge + +The public origin terminates on the cPanel host and reaches the explorer +gateway over an SSH tunnel, through an nginx upstream with a keep-alive pool. +Two things about that path have already caused incidents and are worth knowing +before diagnosing a third. + +`/favicon.ico` and `/robots.txt` are proxied to apache rather than the tunnel, +and apache inherited a parent rewrite that redirected them elsewhere. A +hijacked `robots.txt` means crawlers never saw this site's policy. Both are now +served from the explorer docroot, with `RewriteEngine On` reset there so the +parent catch-all does not apply. + +The tunnel's server-side sshd child carries every public byte and is +single-threaded. Under memory pressure it lands in a per-session scope with +default resource weights, and a few megabytes of page-in on a saturated device +becomes seconds of stall. Runtime protections were applied during the incident +of 2026-08-28, but a per-session scope dies with the session, so the protection +lapses silently whenever the tunnel reconnects. The durable fix is to pin the +tunnel's server end under a controlled slice. + +Operator access degrades before user traffic does on that host, which is +backwards from what an operator needs: new SSH connections fail during banner +exchange while the established tunnel keeps serving. A new connection needs +fork, exec, PAM and password file reads, all uncached IO, and proportional IO +weight cannot beat a saturated queue. An established session does none of that +per byte. From 86c17cb0d5e3f7e7c1778c1bed6397b7e4f1ddb3 Mon Sep 17 00:00:00 2001 From: Bitcoin Universe Date: Sat, 29 Aug 2026 02:57:02 +0000 Subject: [PATCH 2/4] feat(explorer): add multichain navigation and activity --- .../master-page/master-page.component.html | 34 +- .../master-page/master-page.component.ts | 80 +++- .../search-form/search-form.component.html | 7 +- .../search-form/search-form.component.scss | 18 + .../search-form/search-form.component.ts | 85 +++- .../search-results.component.html | 10 +- frontend/src/app/master-page.module.ts | 10 + .../multichain-explorer.component.html | 154 +++++++ .../multichain-explorer.component.scss | 130 ++++++ .../multichain-explorer.component.ts | 391 ++++++++++++++++++ .../multichain-explorer.module.ts | 66 +++ .../app/universe/saved/saved.component.html | 38 +- .../app/universe/saved/saved.component.scss | 20 + .../src/app/universe/saved/saved.component.ts | 49 ++- .../app/universe/universe-api.service.spec.ts | 27 ++ .../src/app/universe/universe-api.service.ts | 106 +++++ .../universe/universe-chain-routing.spec.ts | 36 ++ .../app/universe/universe-chain-routing.ts | 61 +++ .../universe/universe-local.service.spec.ts | 47 ++- .../app/universe/universe-local.service.ts | 105 +++-- .../universe-websocket.service.spec.ts | 33 ++ .../universe/universe-websocket.service.ts | 152 +++++++ frontend/src/app/universe/universe.types.ts | 92 +++++ frontend/src/styles.scss | 88 ++++ scripts/universe/gateway.mjs | 13 +- scripts/universe/gateway.test.mjs | 7 +- 26 files changed, 1790 insertions(+), 69 deletions(-) create mode 100644 frontend/src/app/universe/multichain-explorer/multichain-explorer.component.html create mode 100644 frontend/src/app/universe/multichain-explorer/multichain-explorer.component.scss create mode 100644 frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts create mode 100644 frontend/src/app/universe/multichain-explorer/multichain-explorer.module.ts create mode 100644 frontend/src/app/universe/universe-chain-routing.spec.ts create mode 100644 frontend/src/app/universe/universe-chain-routing.ts create mode 100644 frontend/src/app/universe/universe-websocket.service.spec.ts create mode 100644 frontend/src/app/universe/universe-websocket.service.ts diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index 40e3760fa63..b0491913b47 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -22,6 +22,22 @@ +
+ +
+ +
+
+ @@ -44,7 +60,7 @@ - + @@ -67,42 +83,42 @@