diff --git a/.do/README.md b/.do/README.md deleted file mode 100644 index 1d74b00ad..000000000 --- a/.do/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# `.do/` - -This directory holds the App Spec definition file under [`./specs.ts`](./specs.ts). - -Deployment logic uses `@digitalocean/dots` and can be viewed at [`.github/scripts/redeploy/do/index.ts`](../.github/scripts/redeploy/do/index.ts). - -[View CI/CD documentation for more details](../.github/README.md) diff --git a/.do/specs.ts b/.do/specs.ts deleted file mode 100644 index d90313f96..000000000 --- a/.do/specs.ts +++ /dev/null @@ -1,136 +0,0 @@ -import type { - App_service_spec, - App_spec, - Apps_image_source_spec, - App_variable_definition, -} from "../.github/scripts/node_modules/@digitalocean/dots"; - -const DIGITALOCEAN_BASE_IMAGE: Apps_image_source_spec = { - registry: "tahminator", - registryType: "DOCKER_HUB", - repository: "codebloom", - // override tag - tag: "latest", -}; - -const DIGITALOCEAN_BASE_SERVICE: App_service_spec = { - name: "codebloom", - healthCheck: { - failureThreshold: 9, - httpPath: "/api", - periodSeconds: 10, - successThreshold: 1, - timeoutSeconds: 1, - }, - livenessHealthCheck: { - failureThreshold: 9, - httpPath: "/api", - periodSeconds: 10, - successThreshold: 1, - timeoutSeconds: 1, - }, - httpPort: 8080, - instanceCount: 1, - instanceSizeSlug: "apps-s-1vcpu-1gb-fixed", -}; - -const DIGITALOCEAN_BASE_SPEC: App_spec = { - region: "nyc", - ingress: { - rules: [ - { - component: { - name: "codebloom", - }, - match: { - path: { - prefix: "/", - }, - }, - }, - ], - }, -}; - -export function prodSpec( - envs: App_variable_definition[], - openSearchUsername: string, - openSearchPassword: string, -): App_spec { - return { - ...DIGITALOCEAN_BASE_SPEC, - name: "codebloom-prod", - maintenance: { - enabled: true, - }, - services: [ - { - ...DIGITALOCEAN_BASE_SERVICE, - image: { - ...DIGITALOCEAN_BASE_IMAGE, - tag: "latest", - }, - envs, - logDestinations: [ - { - name: "prod-logs", - openSearch: { - endpoint: "https://opensearch.tahmid.io:443", - basicAuth: { - user: openSearchUsername, - password: openSearchPassword, - }, - indexName: "codebloom.patinanetwork.org", - }, - }, - ], - }, - ], - domains: [ - { - domain: "codebloom.patinanetwork.org", - type: "PRIMARY", - }, - ], - }; -} - -export function stgSpec( - envs: App_variable_definition[], - openSearchUsername: string, - openSearchPassword: string, -): App_spec { - return { - ...DIGITALOCEAN_BASE_SPEC, - name: "codebloom-staging", - services: [ - { - ...DIGITALOCEAN_BASE_SERVICE, - image: { - ...DIGITALOCEAN_BASE_IMAGE, - tag: "staging-latest", - }, - envs, - logDestinations: [ - { - name: "staging-logs", - openSearch: { - endpoint: "https://opensearch.tahmid.io:443", - basicAuth: { - user: openSearchUsername, - password: openSearchPassword, - }, - indexName: "stg.codebloom.patinanetwork.org", - }, - }, - ], - }, - ], - domains: [ - { - domain: "stg.codebloom.patinanetwork.org", - type: "PRIMARY", - }, - ], - }; -} diff --git a/.github/scripts/redeploy/do/apps/create.ts b/.github/scripts/redeploy/do/apps/create.ts deleted file mode 100644 index fc9d65b28..000000000 --- a/.github/scripts/redeploy/do/apps/create.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { App_spec } from "@digitalocean/dots"; -import type { DigitalOceanClient } from "@digitalocean/dots/src/dots/digitalOceanClient"; - -export async function _createAppAndgetAppId( - client: DigitalOceanClient, - projectId: string, - spec: App_spec, -): Promise { - const res = await client.v2.apps.post({ - spec, - projectId, - }); - - if (!res) { - return null; - } - - const { app } = res; - - if (!app) { - return null; - } - - return app.id ?? null; -} diff --git a/.github/scripts/redeploy/do/apps/get.ts b/.github/scripts/redeploy/do/apps/get.ts deleted file mode 100644 index be43a1c05..000000000 --- a/.github/scripts/redeploy/do/apps/get.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { App_spec } from "@digitalocean/dots"; -import type { DigitalOceanClient } from "@digitalocean/dots/src/dots/digitalOceanClient"; - -export async function _getAppId( - client: DigitalOceanClient, - projectId: string, - spec: App_spec, -): Promise { - const appName = spec.name; - - if (!appName) { - throw new Error("App spec name missing, can't find app"); - } - - const res = await client.v2.apps.get({ - queryParameters: { - withProjects: true, - }, - }); - - if (!res) { - return null; - } - - const { apps } = res; - - if (!apps) { - return null; - } - - const foundApp = apps.filter( - (app) => app.projectId == projectId && app.spec?.name == appName, - )[0]; - - if (!foundApp) { - return null; - } - - return foundApp.id ?? null; -} diff --git a/.github/scripts/redeploy/do/index.ts b/.github/scripts/redeploy/do/index.ts deleted file mode 100644 index c39f82f6c..000000000 --- a/.github/scripts/redeploy/do/index.ts +++ /dev/null @@ -1,139 +0,0 @@ -import type { Environment } from "types"; - -import { - DigitalOceanApiKeyAuthenticationProvider, - FetchRequestAdapter, - createDigitalOceanClient, - type App_variable_definition, - type App_response, -} from "@digitalocean/dots"; -import { getEnvVariables } from "load-secrets/env/load"; -import { _createAppAndgetAppId } from "redeploy/do/apps/create"; -import { _getAppId } from "redeploy/do/apps/get"; - -import { prodSpec, stgSpec } from "../../../../.do/specs"; - -function parseCiEnv(ciEnv: Record) { - const user = (() => { - const v = ciEnv["OPENSEARCH_USER"]; - if (!v) { - throw new Error("Missing OPENSEARCH_USER from .env.ci"); - } - return v; - })(); - - const pass = (() => { - const v = ciEnv["OPENSEARCH_PASSWORD"]; - if (!v) { - throw new Error("Missing OPENSEARCH_PASSWORD from .env.ci"); - } - return v; - })(); - - return { user, pass }; -} - -export async function _migrateDo({ - token, - environment, - projectId, - env, -}: { - token: string; - environment: Environment; - projectId: string; - env: Record; -}): Promise { - const authProvider = new DigitalOceanApiKeyAuthenticationProvider(token); - const adapter = new FetchRequestAdapter(authProvider); - const client = createDigitalOceanClient(adapter); - - const envs: App_variable_definition[] = Object.entries(env).map( - ([key, value]) => { - const env: App_variable_definition = { - key, - value, - scope: "RUN_TIME", - type: "SECRET", - }; - return env; - }, - ); - - const { user, pass } = parseCiEnv(await getEnvVariables(["ci"])); - - const spec = - environment === "staging" ? - stgSpec(envs, user, pass) - : prodSpec(envs, user, pass); - - const appId = await (async () => { - const v = await _getAppId(client, projectId, spec); - - if (v) { - return v; - } - - const v2 = await _createAppAndgetAppId(client, projectId, spec); - if (!v2) { - throw new Error( - "App not found and failed to create as well. Please alert Codebloom team.", - ); - } - - return v2; - })(); - - let res: App_response | undefined; - try { - res = await client.v2.apps.byApp_Id(appId).put({ - spec, - updateAllSourceVersions: true, - }); - } catch (e) { - console.error(e); - return process.exit(1); - } - - const pendingDeploymentId = res?.app?.pendingDeployment?.id; - - if (!pendingDeploymentId) { - console.error("Failed to find pending deployment."); - process.exit(1); - } - - console.log("Deployment ID:", pendingDeploymentId); - - let ready = false; - const attempts = 60; - - for (let i = 1; i <= attempts; i++) { - const res = await client.v2.apps - .byApp_Id(appId) - .deployments.byDeployment_id(pendingDeploymentId) - .get(); - - const phase = res?.deployment?.phase ?? "UNKNOWN"; - - console.log("Deployment phase:", phase); - - if (phase === "SUPERSEDED") { - console.error("Deployment was superseded by another deployment."); - process.exit(1); - } - - if (phase === "ACTIVE") { - console.log("Deployment has completed!"); - ready = true; - break; - } - - console.log(`Waiting for deployment to complete... (${i}/${attempts})`); - await Bun.sleep(10000); - } - - if (!ready) { - console.error("Deployment did not reach a valid state within 10 minutes."); - process.exit(1); - } -} diff --git a/.github/scripts/redeploy/index.ts b/.github/scripts/redeploy/index.ts index 10467efb5..d1b93b7f4 100644 --- a/.github/scripts/redeploy/index.ts +++ b/.github/scripts/redeploy/index.ts @@ -1,8 +1,6 @@ import type { Environment } from "types"; -import { getEnvVariables } from "load-secrets/env/load"; import { _migrateDb } from "redeploy/db"; -import { _migrateDo } from "redeploy/do"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; @@ -30,46 +28,10 @@ const { environment, sha } = await yargs(hideBin(process.argv)) .parse(); async function main() { - // should already be called - // await $`git-crypt unlock`; - - const { token, projectId } = parseCiEnv(await getEnvVariables(["ci"])); - const appEnv = await getEnvVariables([environment]); - await _migrateDb({ environment, sha, }); - - await _migrateDo({ - projectId, - token, - environment, - env: appEnv, - }); -} - -function parseCiEnv(ciEnv: Record) { - const token = (() => { - const v = ciEnv["DIGITALOCEAN_PAT"]; - if (!v) { - throw new Error("Missing DIGITALOCEAN_PAT from .env.ci"); - } - return v; - })(); - - const projectId = (() => { - const v = ciEnv["DIGITALOCEAN_PROJECT_ID"]; - if (!v) { - throw new Error("Missing DIGITALOCEAN_PROJECT_ID from .env.ci"); - } - return v; - })(); - - return { - token, - projectId, - }; } main() diff --git a/README.md b/README.md index 1f8475bd4..e85f25f2a 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,6 @@ This repository is a monorepo that contains many different pieces of Codebloom, - Our service gets deployed to two environments (production & staging) in [DigitalOcean App Platform](https://www.digitalocean.com/products/app-platform), which is defined using infrastructure-as-code. We containerize the main application with [Docker](https://www.docker.com/) and monitor metrics & logs with [Prometheus](https://prometheus.io/) & [OpenSearch](https://opensearch.org/) which get fed to [Grafana](https://grafana.com/). - [Goto `.github/scripts/redeploy/` to view Bun Shell redeploy scripts](./.github/scripts/redeploy/) - [Goto `infra/`](./infra/) - - [View DigitalOcean spec](./.do/README.md) - [View detailed documentation](./infra/README.md) - [View observability documentation](./docs/observability/README.md) - Our internal tools (e.g. custom Ink CLI scripts & our internal standup bot) are currently being written in [Rust](https://rust-lang.org/) and [TypeScript](https://typescriptlang.org) diff --git a/docs/observability/README.md b/docs/observability/README.md index aead236ca..4a22ea120 100644 --- a/docs/observability/README.md +++ b/docs/observability/README.md @@ -2,14 +2,17 @@ Grafana -Codebloom uses Spring Boot Actuator and Prometheus to provide operational insights into the running application. We also collect logs via OpenSearch, which is setup through the DigitalOcean App Platform [(view app spec here)](../../.do/specs.ts). Both of these data sources are then fed to a Grafana instance hosted on [monitor.tahmid.io](https://monitor.tahmid.io) +Codebloom uses Spring Boot Actuator and Prometheus to provide operational insights (metrics) into the running application. +Promtail also pulls logs directly from the K8s instance. + +Both of these data sources are then fed to a Grafana instance hosted on [grafana.patinanetwork.org](https://grafana.patinanetwork.org) ## Grafana -All metrics for production & staging can be viewed on [monitor.tahmid.io](https://monitor.tahmid.io). +All metrics for production & staging can be viewed on [grafana.patinanetwork.org](https://grafana.patinanetwork.org). > [!NOTE] -> Please reach out to [@tahminator](https://github.com/tahminator) if you are on the Codebloom dev team & need the credentials to login & view the dashboard +> Please reach out to Henry [@arklian](https://github.com/arklian) if you are on the dev team & need the credentials to login & view the dashboard ## Actuator Endpoints diff --git a/infra/README.md b/infra/README.md index 1e55ef8ee..1542396c8 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,8 +5,8 @@ [`clean-stg-db.SQL`](./clean-stg-db.SQL) is a SQL script used to clean and scramble staging data after copying the production database. -This directory contains the Dockerfile used to build the main Codebloom image, which is then uploaded to [hub.docker.com/r/tahminator/codebloom](https://hub.docker.com/r/tahminator/codebloom). +This directory contains the Dockerfile used to build the main Codebloom image, which is then uploaded to [hub.docker.com/r/patinanetwork/codebloom](https://hub.docker.com/r/patinanetwork/codebloom). -The image is then deployed to the [DigitalOcean App Platform](https://www.digitalocean.com/products/app-platform) see [`.do/`](../.do/) to view the DigitalOcean app spec and more detailed documentation regarding DigitalOcean and deployments. +The image is then deployed to Patina's Kubernetes cluster. [Patina-Network/k8s-manifests](https://github.com/Patina-Network/k8s-manifests) There is a Bun Shell script which helps us manage the workflow for deployments across production and staging, which can be found at [`.github/scripts/redeploy/index.ts`](../.github/scripts/redeploy/index.ts). \ No newline at end of file