diff --git a/.github/workflows/smoke-test-services.yml b/.github/workflows/smoke-test-services.yml new file mode 100644 index 0000000..810e6c8 --- /dev/null +++ b/.github/workflows/smoke-test-services.yml @@ -0,0 +1,148 @@ +name: Smoke Test Services + +on: + push: + branches: + - main + paths: + - 'on-prem/docker-compose.*.yml' + - 'on-prem/templates/**' + - 'on-prem/scripts/**' + - '.env.example' + pull_request: + paths: + - 'on-prem/docker-compose.*.yml' + - 'on-prem/templates/**' + - 'on-prem/scripts/**' + - '.env.example' + workflow_dispatch: + +jobs: + smoke-test: + name: API Smoke Test + runs-on: ubuntu-latest + permissions: + id-token: write # Required for OIDC + contents: read # Required for checkout + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::761136292957:role/GitHubActions-CurrentsDevDocker-ECRPull + aws-region: us-east-1 + + - name: Login to Currents ECR + uses: aws-actions/amazon-ecr-login@v2 + with: + registries: "513558712013" + + - name: Setup environment + working-directory: on-prem + run: ./scripts/setup.sh --env-only --force + + - name: Start infrastructure services + working-directory: on-prem + run: | + echo "Starting Redis and MongoDB..." + docker compose -f docker-compose.full.yml up -d redis mongodb + echo "Waiting for services to initialize..." + + - name: Wait for Redis + working-directory: on-prem + run: | + echo "Waiting for Redis to be ready..." + for i in {1..30}; do + if docker compose -f docker-compose.full.yml exec -T redis redis-cli ping | grep -q PONG; then + echo "✅ Redis is ready" + exit 0 + fi + echo "Attempt $i/30 - Redis not ready yet..." + sleep 2 + done + echo "❌ Redis failed to start" + docker compose -f docker-compose.full.yml logs redis + exit 1 + + - name: Wait for MongoDB + working-directory: on-prem + run: | + echo "Waiting for MongoDB to be healthy..." + for i in {1..60}; do + if docker compose -f docker-compose.full.yml exec -T mongodb mongosh --quiet --eval "db.runCommand('ping').ok" localhost:27017 2>/dev/null | grep -q 1; then + echo "✅ MongoDB is ready" + exit 0 + fi + echo "Attempt $i/60 - MongoDB not ready yet..." + sleep 2 + done + echo "❌ MongoDB failed to start" + docker compose -f docker-compose.full.yml logs mongodb + exit 1 + + - name: Start application services + working-directory: on-prem + run: | + echo "Starting Scheduler and API..." + docker compose -f docker-compose.full.yml up -d scheduler api + echo "Waiting for services to initialize..." + + - name: Wait for API + working-directory: on-prem + run: | + echo "Waiting for API to be ready..." + for i in {1..60}; do + if curl -sf http://localhost:4000/health > /dev/null 2>&1; then + echo "✅ API is ready" + exit 0 + fi + echo "Attempt $i/60 - API not ready yet..." + sleep 2 + done + echo "❌ API failed to start" + docker compose -f docker-compose.full.yml logs api + exit 1 + + - name: Wait for root user + working-directory: on-prem + run: | + source .env + echo "Waiting for root user to be created..." + for i in {1..30}; do + if docker compose -f docker-compose.full.yml exec -T mongodb mongosh \ + -u "$MONGODB_USERNAME" -p "$MONGODB_PASSWORD" --authenticationDatabase admin \ + --quiet --eval "db.getSiblingDB('currents').user.findOne({email: '${ON_PREM_EMAIL:-root@currents.local}'})" 2>/dev/null | grep -q "_id"; then + echo "✅ Root user exists" + exit 0 + fi + echo "Attempt $i/30 - Root user not created yet..." + sleep 2 + done + echo "❌ Root user was not created" + docker compose -f docker-compose.full.yml logs api scheduler + exit 1 + + - name: Seed database + id: seed + working-directory: on-prem + run: | + echo "Seeding database with test data..." + # Capture the KEY=VALUE output from seed script + eval $(./scripts/smoke-test/seed-database.sh) + # Export to GitHub Actions output + echo "api_key=${API_KEY}" >> $GITHUB_OUTPUT + echo "project_id=${PROJECT_ID}" >> $GITHUB_OUTPUT + + - name: Run API smoke test + working-directory: on-prem + run: | + echo "Running API smoke test..." + ./scripts/smoke-test/api-test.sh "${{ steps.seed.outputs.api_key }}" "${{ steps.seed.outputs.project_id }}" + + - name: Cleanup + if: always() + working-directory: on-prem + run: | + docker compose -f docker-compose.full.yml down -v --remove-orphans diff --git a/on-prem/docker-compose.database.yml b/on-prem/docker-compose.database.yml index 398be59..1b47ef9 100644 --- a/on-prem/docker-compose.database.yml +++ b/on-prem/docker-compose.database.yml @@ -26,9 +26,6 @@ services: CURRENTS_ENV: onprem EMAIL_TRANSPORTER: smtp depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false @@ -52,9 +49,6 @@ services: CLICKHOUSE_PASSWORD: ${CLICKHOUSE_CURRENTS_PASSWORD} EMAIL_TRANSPORTER: smtp depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false @@ -73,15 +67,13 @@ services: CLICKHOUSE_PASSWORD: ${CLICKHOUSE_CURRENTS_PASSWORD} EMAIL_TRANSPORTER: smtp depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false redis: condition: service_started required: false + # Currents services depend on clickhouse write-worker: image: ${DC_CURRENTS_IMAGE_REPOSITORY:-currents-}writer:${DC_CURRENTS_IMAGE_TAG:-staging} restart: unless-stopped @@ -118,9 +110,6 @@ services: volumes: - ${DC_SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false @@ -138,9 +127,6 @@ services: CURRENTS_ENV: onprem EMAIL_TRANSPORTER: smtp depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false diff --git a/on-prem/docker-compose.full.yml b/on-prem/docker-compose.full.yml index f5c3c34..522c22e 100644 --- a/on-prem/docker-compose.full.yml +++ b/on-prem/docker-compose.full.yml @@ -27,9 +27,6 @@ services: EMAIL_TRANSPORTER: smtp FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false @@ -54,9 +51,6 @@ services: EMAIL_TRANSPORTER: smtp FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false @@ -76,15 +70,13 @@ services: EMAIL_TRANSPORTER: smtp FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false redis: condition: service_started required: false + # Currents services depend on clickhouse write-worker: image: ${DC_CURRENTS_IMAGE_REPOSITORY:-currents-}writer:${DC_CURRENTS_IMAGE_TAG:-staging} restart: unless-stopped @@ -123,9 +115,6 @@ services: volumes: - ${DC_SCHEDULER_STARTUP_VOLUME:-./data/startup}:/app/packages/scheduler/dist/.startup depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false @@ -144,9 +133,6 @@ services: EMAIL_TRANSPORTER: smtp FILE_STORAGE_FORCE_PATH_STYLE: "true" depends_on: - clickhouse: - condition: service_healthy - required: false mongodb: condition: service_healthy required: false diff --git a/on-prem/docs/container-images.md b/on-prem/docs/container-images.md index 3b186cf..cafaaef 100644 --- a/on-prem/docs/container-images.md +++ b/on-prem/docs/container-images.md @@ -25,7 +25,18 @@ Create an IAM role in your AWS account with the following policy: "ecr:GetDownloadUrlForLayer" ], "Resource": [ - "arn:aws:ecr:us-east-1:513558712013:repository/currents/on-prem/*" + "arn:aws:ecr:us-east-1:513558712013:currents/on-prem/api/*", + "arn:aws:ecr:us-east-1:513558712013:repository/currents/on-prem/api", + "arn:aws:ecr:us-east-1:513558712013:currents/on-prem/change-streams/*", + "arn:aws:ecr:us-east-1:513558712013:repository/currents/on-prem/change-streams", + "arn:aws:ecr:us-east-1:513558712013:currents/on-prem/director/*", + "arn:aws:ecr:us-east-1:513558712013:repository/currents/on-prem/director", + "arn:aws:ecr:us-east-1:513558712013:currents/on-prem/scheduler/*", + "arn:aws:ecr:us-east-1:513558712013:repository/currents/on-prem/scheduler", + "arn:aws:ecr:us-east-1:513558712013:currents/on-prem/writer/*", + "arn:aws:ecr:us-east-1:513558712013:repository/currents/on-prem/writer", + "arn:aws:ecr:us-east-1:513558712013:currents/on-prem/webhooks/*", + "arn:aws:ecr:us-east-1:513558712013:repository/currents/on-prem/webhooks" ] } ] diff --git a/on-prem/scripts/setup.sh b/on-prem/scripts/setup.sh index 8ccabb3..7b08770 100755 --- a/on-prem/scripts/setup.sh +++ b/on-prem/scripts/setup.sh @@ -3,6 +3,10 @@ # Interactive setup script for Currents on-prem # Generates a docker-compose file based on your infrastructure choices # and sets it as the default docker-compose.yml +# +# Usage: +# ./setup.sh # Interactive setup (profile selection + env generation) +# ./setup.sh --env-only # Only generate .env file with secrets (non-interactive) set -e @@ -16,27 +20,56 @@ BLUE='\033[0;34m' YELLOW='\033[1;33m' NC='\033[0m' # No Color -echo -e "${BLUE}" -echo "╔═══════════════════════════════════════════════════════════╗" -echo "║ Currents On-Prem Setup ║" -echo "╚═══════════════════════════════════════════════════════════╝" -echo -e "${NC}" - -echo "This script will help you generate a docker-compose configuration" -echo "based on which services you want to run locally vs externally." -echo "" +# Parse arguments +ENV_ONLY=false +FORCE_REGEN=false +while [[ $# -gt 0 ]]; do + case $1 in + --env-only) + ENV_ONLY=true + shift + ;; + --force) + FORCE_REGEN=true + shift + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --env-only Only generate .env file with secrets (non-interactive)" + echo " --force Force regenerate secrets even if .env exists" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done # ============================================================================= # Environment file setup # ============================================================================= +# Define setup_env_file function first (used by both modes) setup_env_file() { cd "$ON_PREM_DIR" if [ -f .env ]; then echo -e "${YELLOW}Found existing .env file${NC}" - read -p "Regenerate secrets in .env? [y/N]: " regen_secrets - if [[ ! $regen_secrets =~ ^[Yy] ]]; then + if [ "$FORCE_REGEN" = true ]; then + echo "Force regenerating secrets..." + elif [ "$ENV_ONLY" = true ]; then + # Non-interactive mode: skip if .env exists and --force not set + echo "Using existing .env file (use --force to regenerate secrets)" return + else + read -p "Regenerate secrets in .env? [y/N]: " regen_secrets + if [[ ! $regen_secrets =~ ^[Yy] ]]; then + return + fi fi ENV_FILE=".env" else @@ -106,8 +139,27 @@ setup_env_file() { } # ============================================================================= -# Profile selection +# Handle --env-only mode # ============================================================================= +if [ "$ENV_ONLY" = true ]; then + setup_env_file + echo -e "${GREEN}✓ Environment setup complete!${NC}" + exit 0 +fi + +# ============================================================================= +# Interactive mode: Profile selection +# ============================================================================= +echo -e "${BLUE}" +echo "╔═══════════════════════════════════════════════════════════╗" +echo "║ Currents On-Prem Setup ║" +echo "╚═══════════════════════════════════════════════════════════╝" +echo -e "${NC}" + +echo "This script will help you generate a docker-compose configuration" +echo "based on which services you want to run locally vs externally." +echo "" + echo -e "${YELLOW}Select a configuration profile:${NC}" echo "" echo " 1) full - All services (redis, mongodb, clickhouse, rustfs)" diff --git a/on-prem/scripts/smoke-test/api-test.sh b/on-prem/scripts/smoke-test/api-test.sh new file mode 100755 index 0000000..1b0c42f --- /dev/null +++ b/on-prem/scripts/smoke-test/api-test.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# +# API smoke test - verify the Currents API is working +# Creates an action via the API and verifies it can be retrieved +# +# Usage: ./api-test.sh +# +# Based on Currents API: https://api.currents.dev/v1/docs/ + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ON_PREM_DIR="$SCRIPT_DIR/../.." + +API_KEY="${1:-}" +PROJECT_ID="${2:-}" +API_BASE_URL="${API_BASE_URL:-http://localhost:4000/v1}" + +# Function to show logs on failure +show_logs_on_failure() { + echo "" + echo "==========================================" + echo "API container logs (last 100 lines):" + echo "==========================================" + cd "$ON_PREM_DIR" + docker compose -f docker-compose.full.yml logs --tail=100 api + echo "==========================================" +} + +if [ -z "$API_KEY" ]; then + echo "❌ Error: API key is required" + echo "Usage: $0 " + exit 1 +fi + +if [ -z "$PROJECT_ID" ]; then + echo "❌ Error: Project ID is required" + echo "Usage: $0 " + exit 1 +fi + +echo "Running API smoke test..." +echo "API URL: $API_BASE_URL" +echo "Project ID: $PROJECT_ID" +echo "" + +# Generate unique name for this test run +TEST_NAME="smoke-test-action-$(date +%s)" + +# ============================================================================= +# Step 1: Create an action +# ============================================================================= +echo "Step 1: Creating test action..." + +CREATE_RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${API_BASE_URL}/actions?projectId=${PROJECT_ID}" \ + -H "Authorization: Bearer ${API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "'"${TEST_NAME}"'", + "description": "Smoke test action - safe to delete", + "action": [{"op": "skip"}], + "matcher": { + "op": "AND", + "cond": [ + { + "type": "title", + "op": "eq", + "value": "smoke-test-placeholder" + } + ] + } + }') + +HTTP_CODE=$(echo "$CREATE_RESPONSE" | tail -n1) +RESPONSE_BODY=$(echo "$CREATE_RESPONSE" | sed '$d') + +if [ "$HTTP_CODE" != "201" ]; then + echo "❌ Failed to create action (HTTP $HTTP_CODE)" + echo "Response: $RESPONSE_BODY" + show_logs_on_failure + exit 1 +fi + +ACTION_ID=$(echo "$RESPONSE_BODY" | jq -r '.data.actionId') + +if [ -z "$ACTION_ID" ] || [ "$ACTION_ID" = "null" ]; then + echo "❌ Failed to extract actionId from response" + echo "Response: $RESPONSE_BODY" + show_logs_on_failure + exit 1 +fi + +echo "✅ Created action: $ACTION_ID" + +# ============================================================================= +# Step 2: Fetch the action back +# ============================================================================= +echo "" +echo "Step 2: Fetching action..." + +GET_RESPONSE=$(curl -s -w "\n%{http_code}" "${API_BASE_URL}/actions/${ACTION_ID}" \ + -H "Authorization: Bearer ${API_KEY}") + +HTTP_CODE=$(echo "$GET_RESPONSE" | tail -n1) +RESPONSE_BODY=$(echo "$GET_RESPONSE" | sed '$d') + +if [ "$HTTP_CODE" != "200" ]; then + echo "❌ Failed to fetch action (HTTP $HTTP_CODE)" + echo "Response: $RESPONSE_BODY" + show_logs_on_failure + exit 1 +fi + +FETCHED_NAME=$(echo "$RESPONSE_BODY" | jq -r '.data.name') +FETCHED_STATUS=$(echo "$RESPONSE_BODY" | jq -r '.data.status') + +if [ "$FETCHED_NAME" != "$TEST_NAME" ]; then + echo "❌ Action name mismatch" + echo "Expected: $TEST_NAME" + echo "Got: $FETCHED_NAME" + show_logs_on_failure + exit 1 +fi + +echo "✅ Fetched action successfully" +echo " Name: $FETCHED_NAME" +echo " Status: $FETCHED_STATUS" + +# ============================================================================= +# Step 3: Clean up - delete the action +# ============================================================================= +echo "" +echo "Step 3: Cleaning up (deleting action)..." + +DELETE_RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE "${API_BASE_URL}/actions/${ACTION_ID}" \ + -H "Authorization: Bearer ${API_KEY}") + +HTTP_CODE=$(echo "$DELETE_RESPONSE" | tail -n1) +RESPONSE_BODY=$(echo "$DELETE_RESPONSE" | sed '$d') + +if [ "$HTTP_CODE" != "200" ]; then + echo "⚠️ Warning: Failed to delete action (HTTP $HTTP_CODE)" + echo "Response: $RESPONSE_BODY" + # Don't fail the test for cleanup issues +else + echo "✅ Action deleted (archived)" +fi + +# ============================================================================= +# Summary +# ============================================================================= +echo "" +echo "==========================================" +echo "✅ API smoke test passed!" +echo "==========================================" +echo "" +echo "Verified:" +echo " - POST /actions (create)" +echo " - GET /actions/{actionId} (read)" +echo " - DELETE /actions/{actionId} (delete)" +echo "" diff --git a/on-prem/scripts/smoke-test/seed-database.sh b/on-prem/scripts/smoke-test/seed-database.sh new file mode 100755 index 0000000..6fec434 --- /dev/null +++ b/on-prem/scripts/smoke-test/seed-database.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# +# Seed the database with test data for smoke testing +# Creates: organization, project, and API key +# +# Prerequisites: +# - MongoDB must be running and healthy +# - API must have created the root user (from ON_PREM_EMAIL in .env) +# +# Usage: ./seed-database.sh +# Outputs: API_KEY and PROJECT_ID to stdout (can be eval'd) +# +# Example: +# eval $(./scripts/smoke-test/seed-database.sh) +# echo $API_KEY $PROJECT_ID + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ON_PREM_DIR="$SCRIPT_DIR/../.." + +cd "$ON_PREM_DIR" + +# Source environment variables +echo "[DEBUG] Sourcing .env file..." >&2 +source .env + +echo "Seeding database with test data..." >&2 + +# Generate a random 6-character project ID (alphanumeric) +generate_project_id() { + openssl rand -base64 12 | tr -dc 'a-zA-Z0-9' | head -c 6 +} + +# Generate a 64-character API key +generate_api_key() { + openssl rand -base64 96 | tr -dc 'a-zA-Z0-9' | head -c 64 +} + +echo "[DEBUG] Generating project ID..." >&2 +PROJECT_ID=$(generate_project_id) +echo "[DEBUG] Project ID: $PROJECT_ID" >&2 + +echo "[DEBUG] Generating API key..." >&2 +API_KEY=$(generate_api_key) +echo "[DEBUG] API key generated" >&2 + +ROOT_EMAIL="${ON_PREM_EMAIL:-root@currents.local}" +echo "[DEBUG] Root email: $ROOT_EMAIL" >&2 + +# Create temp file for the MongoDB script +echo "[DEBUG] Creating temp file..." >&2 +TEMP_SCRIPT=$(mktemp) +trap "rm -f $TEMP_SCRIPT" EXIT +echo "[DEBUG] Temp file: $TEMP_SCRIPT" >&2 + +echo "[DEBUG] Writing MongoDB script to temp file..." >&2 +cat > "$TEMP_SCRIPT" <&2 + +# Get the container name +echo "[DEBUG] Getting MongoDB container ID..." >&2 +CONTAINER=$(docker compose -f docker-compose.full.yml ps -q mongodb) +echo "[DEBUG] Container ID: $CONTAINER" >&2 + +if [ -z "$CONTAINER" ]; then + echo "❌ MongoDB container not found" >&2 + exit 1 +fi + +# Copy script into container +echo "[DEBUG] Copying seed script to container..." >&2 +docker cp "$TEMP_SCRIPT" "$CONTAINER:/tmp/seed.js" +echo "[DEBUG] Script copied" >&2 + +# Run the script +echo "[DEBUG] Running MongoDB seed script..." >&2 +RESULT=$(docker exec "$CONTAINER" mongosh \ + -u "$MONGODB_USERNAME" -p "$MONGODB_PASSWORD" --authenticationDatabase admin \ + --quiet --file /tmp/seed.js 2>&1) + +echo "[DEBUG] Script execution completed" >&2 +echo "$RESULT" >&2 + +# Check if MongoDB commands succeeded +if ! echo "$RESULT" | grep -q "SUCCESS"; then + echo "❌ Failed to seed database" >&2 + exit 1 +fi + +# Output variables that can be eval'd by the caller +echo "API_KEY=${API_KEY}" +echo "PROJECT_ID=${PROJECT_ID}" + +echo "✅ Database seeded successfully" >&2 +echo " Project ID: ${PROJECT_ID}" >&2 +echo " API Key: ${API_KEY:0:8}..." >&2 diff --git a/on-prem/templates/compose.clickhouse.yml b/on-prem/templates/compose.clickhouse.yml index 5b1080f..a109f4d 100644 --- a/on-prem/templates/compose.clickhouse.yml +++ b/on-prem/templates/compose.clickhouse.yml @@ -58,38 +58,8 @@ services: - ${DC_CLICKHOUSE_VOLUME:-./data/clickhouse}:/var/lib/clickhouse # Currents services depend on clickhouse - director: - depends_on: - clickhouse: - condition: service_healthy - required: false - - api: - depends_on: - clickhouse: - condition: service_healthy - required: false - - changestreams-worker: - depends_on: - clickhouse: - condition: service_healthy - required: false - write-worker: depends_on: clickhouse: condition: service_healthy required: false - - scheduler: - depends_on: - clickhouse: - condition: service_healthy - required: false - - webhooks: - depends_on: - clickhouse: - condition: service_healthy - required: false