What needs a decision reaches a person by itself (#167)

.env.example

@@ -67,6 +67,14 @@ EMAIL_REGION=
 # right and fails identically to having no Reply-To at all.
 EMAIL_REPLY_TO=
 
+# Where the instance's operations report goes (#167, docs/operations.md): an
+# ACT NOW message when something needs a decision, and a daily report either
+# way. Read on the instance only, by deploy/ops-check.sh — the application
+# ignores it. Leave empty and the report reaches the journal and nobody else.
+# It carries counts and the instance's own figures, never an address or a line
+# of any log, so a mailbox outside the EU is acceptable for it (§7).
+OPS_EMAIL=
+
 # Application
 APP_URL=
 AUTH_SECRET=

.github/workflows/ci.yml

@@ -296,6 +296,33 @@ jobs:
       - name: The nightly copy skips the preview copies
         run: grep -q "not like 'platform..pr..%'" deploy/backup-db.sh
 
+      # #167. The application's log goes to the host journal, because a log
+      # kept in the container's own directory is deleted with the container —
+      # which is every deploy. And the hourly check that reads it has to reach
+      # the instance, or the report that exists to break the silence is itself
+      # silently missing.
+      - name: The application logs to the journal
+        env:
+          APP_IMAGE: ghcr.io/example/app:validate
+          SITE_ADDRESS: dev.example.invalid
+          S3_ORIGIN: https://bucket.example.invalid
+          S3_UPLOAD_ORIGIN: https://endpoint.example.invalid
+        run: |
+          # Read from the resolved configuration, not grepped: a commented-out
+          # line or one under the wrong service must not pass.
+          : > deploy/.env
+          docker compose --file deploy/compose.yaml config --format json |
+            jq -e '.services.app.logging.driver == "journald" and .services.caddy.logging.driver == "journald"'
+          rm deploy/.env
+      - name: The operations check is shipped and installed
+        run: |
+          # Anchored to the scp line's indentation, so this line cannot satisfy itself.
+          grep -qE '^ +deploy/backup-db.sh deploy/ops-check.sh' .github/workflows/ci.yml
+          grep -q 'ExecStart=/bin/bash /opt/platform-lite/ops-check.sh' deploy/remote-deploy.sh
+          # Defined is not called: the installer has to be invoked on a line of its own.
+          grep -qxE 'install_ops_timer' deploy/remote-deploy.sh
+          bash -n deploy/ops-check.sh
+
   # Smoke on every push and PR (SPEC §6): the database-less project only. No
   # DATABASE_URL_TEST here on purpose — that is what makes this job prove the
   # fail-closed behaviour (401s, redirects to /login, 404s) with no database at
@@ -491,7 +518,7 @@ jobs:
         run: |
           scp -i ~/.ssh/id_deploy \
             deploy/compose.yaml deploy/Caddyfile deploy/remote-deploy.sh \
-            deploy/backup-db.sh \
+            deploy/backup-db.sh deploy/ops-check.sh \
             "$HOST:/opt/platform-lite/"
       - name: Pull the image and restart the container
         if: steps.wired.outputs.wired == 'true'

.github/workflows/watch.yml

@@ -0,0 +1,112 @@
+name: Watch
+
+# #167: whether the site answers, asked from somewhere that is not the
+# instance. The hourly check ON the instance (deploy/ops-check.sh) sees the
+# disk, the containers and the nightly copy — and cannot report the one thing
+# that silences all of it: the instance itself being gone. A box cannot report
+# its own death, so this runs on GitHub's machines.
+#
+# Where it speaks: a GitHub issue labelled `outage`, opened when the site stops
+# answering and closed — with the time it came back — when it answers again.
+# That is no new provider and no new cost while the repository is public, it
+# reaches the owner's phone through GitHub's own notifications, and its STATE
+# is the answer: an open `outage` issue means down, none means up.
+#
+# Every thirty minutes. GitHub runs scheduled workflows late under load and
+# not at all in a repository with no activity for sixty days — both written
+# down in docs/operations.md, because a watcher that has quietly stopped looks
+# exactly like a site that is fine. If the repository goes private (#163),
+# these runs start costing minutes: 48 a day at well under a minute each.
+
+on:
+  schedule:
+    - cron: "7,37 * * * *"
+  workflow_dispatch:
+    inputs:
+      url:
+        description: "An https address to check instead of dev's — a drill, kept apart from the real alarm under the outage-drill label"
+        required: false
+
+# Nothing is checked out and nothing is read from the repository: the one thing
+# this needs is to open and close an issue.
+permissions:
+  issues: write
+
+# Two runs must not both decide to open the issue.
+concurrency:
+  group: watch
+  cancel-in-progress: false
+
+jobs:
+  answers:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    env:
+      URL: ${{ inputs.url || 'https://dev.architektow3d.pl/' }}
+      # A drill has its own label, so trying the alarm against an address that
+      # cannot answer never opens — or closes — dev's real outage issue.
+      LABEL: ${{ inputs.url && 'outage-drill' || 'outage' }}
+    steps:
+      - name: Does it answer?
+        id: probe
+        run: |
+          # https only, and passed to curl as a URL rather than as an argument:
+          # the address can come from a person running this by hand, and a
+          # value starting with a dash, or a file:// one, is not an address.
+          case "$URL" in
+            https://*) ;;
+            *) echo "::error::only https addresses are checked, not: $URL"; exit 1 ;;
+          esac
+          # Three tries thirty seconds apart: one slow answer during a deploy is
+          # not an outage, and an alarm that cries wolf is an alarm nobody
+          # reads. A 4xx is an answer — the server is up and deciding; only a
+          # 5xx, a timeout or no connection at all counts as down.
+          code=000
+          for attempt in 1 2 3; do
+            code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 --proto '=https' --url "$URL" || true)
+            case "$code" in
+              [1-4][0-9][0-9]) break ;;
+            esac
+            [ "$attempt" -eq 3 ] || sleep 30
+          done
+          echo "code=$code" >> "$GITHUB_OUTPUT"
+          echo "$URL answered ${code} (000 is no answer at all)"
+
+      - name: Open or close the outage issue
+        env:
+          CODE: ${{ steps.probe.outputs.code }}
+          OWNER: ${{ github.repository_owner }}
+          # Here and only here: the step above runs curl against an address a
+          # person may have typed, and has no use for a token.
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          open=$(gh issue list --label "$LABEL" --state open --json number --jq '.[0].number // empty')
+          now=$(date -u +%Y-%m-%dT%H:%MZ)
+          case "$CODE" in
+            [1-4][0-9][0-9]) up=true ;;
+            *) up=false ;;
+          esac
+
+          if [ "$up" = false ] && [ -z "$open" ]; then
+            gh label create "$LABEL" --color B60205 \
+              --description "The site is not answering (opened and closed by .github/workflows/watch.yml)" \
+              --force >/dev/null
+            issue=$(gh issue create --label "$LABEL" \
+              --title "$URL is not answering" \
+              --body "$(printf '%s\n' \
+                "Checked from GitHub's machines at $now: three tries thirty seconds apart, last answer \`$CODE\` (\`000\` is no answer at all)." \
+                "" \
+                "What to do: docs/operations.md, \"What each signal means\". This issue closes itself when the site answers again." \
+                "" \
+                "@$OWNER")")
+            # Locked: the thread notifies the owner, so it is not a place for
+            # anyone passing by to post "run this to fix it". Collaborators —
+            # and this workflow — can still write in it.
+            gh issue lock "$issue" || true
+          elif [ "$up" = true ] && [ -n "$open" ]; then
+            gh issue comment "$open" --body "Answering again at $now (\`$CODE\`)." || true
+            gh issue close "$open"
+          else
+            echo "nothing changed: up=$up, open $LABEL issue: ${open:-none}"
+          fi

SPEC.md

@@ -358,6 +358,17 @@ export function ownerKey(
   **The signing key being shared is the thing production must not inherit** — one
   `AUTH_SECRET` across environments plus a copy of the rows is how a session from one
   becomes a session in another; #24 gives production its own.
+- **What needs a decision reaches a person by itself** (#167, `docs/operations.md`): every
+  hour `deploy/ops-check.sh` looks at the instance — disk, containers, whether the site
+  answers, the nightly copy, preview copies nobody owns, the R360 collector's findings,
+  database deadlines, server errors, the mail provider's failures and blocks — and mails
+  `OPS_EMAIL` when something needs acting on, plus a report every morning either way, so an
+  empty inbox means "checked" and a missing report is itself the signal. Whether the site
+  answers from outside is `.github/workflows/watch.yml`, which opens and closes an `outage`
+  issue: a box cannot report its own death. Three tiers and no more. The reports carry counts,
+  never an address or a log line (§7). Application logs live in the host journal, capped,
+  because a log inside the container's directory died with every deploy. No new provider and
+  no new cost: the mail goes through the transactional provider the application already uses.
 - **Every wait on the database has a deadline, and each environment sets its own** (#172):
   the pool answers a caller it cannot give a connection to within five seconds instead of
   queueing them for ever, and the server cuts off a statement that runs past ten seconds or

deploy/compose.yaml

@@ -18,6 +18,19 @@ services:
     # guess passwords without limit (the #8 review's trust contract).
     expose: ["3000"]
     networks: [platform]
+    # #167: into the host's journal, not a file in the container's own
+    # directory. That file is deleted with the container — which is every
+    # deploy — so the log of the hour before a bad deploy was gone by the time
+    # anyone looked for it. The journal outlives the container, is capped by
+    # remote-deploy.sh, and tags every entry with the container's name, which
+    # compose keeps from one deploy to the next.
+    logging:
+      driver: journald
+      # The tag is what rsyslog is told to leave alone (remote-deploy.sh), so
+      # the lines are kept once, in the capped journal, and not again in
+      # /var/log/syslog.
+      options:
+        tag: platform-lite-app
 
   caddy:
     image: caddy:2-alpine
@@ -45,6 +58,11 @@ services:
       S3_UPLOAD_ORIGIN: ${S3_UPLOAD_ORIGIN:?set S3_UPLOAD_ORIGIN to the S3 endpoint origin}
     depends_on: [app]
     networks: [platform]
+    # Same as the app's, for the same reason (#167).
+    logging:
+      driver: journald
+      options:
+        tag: platform-lite-caddy
 
 networks:
   # Created once by the deployment procedure, not by this file, because the