From 65422921240f576a2cb3be8255666fd6bb501047 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 12 Aug 2025 18:12:56 +0200 Subject: [PATCH 01/22] Build and push image to nexus --- .github/workflows/action.yml | 77 ++++------------------ .github/workflows/build-and-push-image.yml | 70 ++++++++++++++++++++ db.sh | 3 - prod.Dockerfile | 6 ++ scripts/fix-permissions.sh | 8 +++ scripts/setup-venv.sh | 4 +- 6 files changed, 98 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/build-and-push-image.yml create mode 100644 prod.Dockerfile create mode 100755 scripts/fix-permissions.sh mode change 100644 => 100755 scripts/setup-venv.sh diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 31e895c..d3b32d6 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -1,73 +1,20 @@ -name: CI +name: Build and Push Docker Image on: push: branches: - - develop - pull_request: - branches: - - develop + - develop # dev trigger (automatic) + workflow_dispatch: # prod trigger (manual) jobs: - lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./requirements/lint.in - - # Rework happend only in ETL folder - - name: Run black - run: black --check ./ - - - name: Run isort - run: isort --check-only ./ - - - name: Install shellcheck and run ShellCheck - run: | - shellcheck ./db.sh - shellcheck ./scripts/*.sh + dev_build: + if: github.event_name == 'push' && github.ref == 'refs/heads/develop' + uses: ./.github/workflows/build-and-push-image.yml + with: + environment: dev - build: - # Only run if the event is a manual trigger + prod_build: if: github.event_name == 'workflow_dispatch' - needs: lint - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./requirements/build.in - - - name: Copy template.env to .env - run: cp template.env .env - - - name: Run the "db.sh" script - env: - GITHUB_CI: "true" - NEXUS_URL: ${{ secrets.NEXUS_URL }} - NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - run: ./db.sh - - # To update: If it is merged to main -> final tag (10.1.0) or (latest?) - # To update: If it is just a PR -> latest tag of the branch (10.0.2025-01-01-xxxx) Git Commit Hash? \ No newline at end of file + uses: ./.github/workflows/build-and-push-image.yml + with: + environment: prod diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml new file mode 100644 index 0000000..2b0ab57 --- /dev/null +++ b/.github/workflows/build-and-push-image.yml @@ -0,0 +1,70 @@ +name: Build and Push Docker Image + +on: + workflow_call: + inputs: + environment: + required: true + type: string + +jobs: + build: + runs-on: ubuntu-latest + env: + GITHUB_CI: true + NEXUS_URL: ${{ secrets.NEXUS_URL }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + DOCKER_DATA_MODEL_CONTAINER_NAME: infocompanies-data-model-postgres + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Makes sure inputs.environment is set + if: ${{ inputs.environment != 'prod' && inputs.environment != 'dev' }} + run: | + echo "Error: 'environment' input is required ('dev' or 'prod')." + exit 1 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r ./requirements/build.in + + - name: Copy template.env to .env + run: cp template.env .env + + - if: ${{ inputs.environment == 'dev' }} + name: Run database setup script + run: ./db.sh + + - name: Log in to Nexus Docker registry + run: echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_URL}" -u "${NEXUS_USERNAME}" --password-stdin + + - name: Generate a tag, commit and push the image to Nexus + run: | + DOCKER_TAG=$(date +%Y%m%d%H%M%S) + IMAGE="${NEXUS_URL}/${DOCKER_DATA_MODEL_CONTAINER_NAME}-${{ inputs.environment }}:${DOCKER_TAG}" + + if [ "${{ inputs.environment }}" = 'dev' ]; then + docker commit $DOCKER_DATA_MODEL_CONTAINER_NAME $IMAGE + else + docker build -t $IMAGE -f prod.Dockerfile . + fi + + docker push $IMAGE + echo "Docker image pushed: $IMAGE" + + - name: Remove Nexus login credentials + run: docker logout ${NEXUS_URL} + + - name: Clean up Docker resources + run: | + docker compose down || echo "No docker-compose file found or no containers running" + docker system prune -f + echo "Cleaned up Docker resources." diff --git a/db.sh b/db.sh index 4f94f13..2cfe476 100755 --- a/db.sh +++ b/db.sh @@ -24,6 +24,3 @@ cd .. ./scripts/load-csv-to-database.sh - -docker compose down - diff --git a/prod.Dockerfile b/prod.Dockerfile new file mode 100644 index 0000000..551cfb7 --- /dev/null +++ b/prod.Dockerfile @@ -0,0 +1,6 @@ +FROM postgres:16.4 + +WORKDIR /data-model/ + +# Copy requirements files and schema +COPY ./requirements/build.in ./schema/ /data-model/ diff --git a/scripts/fix-permissions.sh b/scripts/fix-permissions.sh new file mode 100755 index 0000000..85af9f8 --- /dev/null +++ b/scripts/fix-permissions.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Fix permissions script +# Sets appropriate permissions for files and directories + +chmod -R 755 ./scripts + +echo "Permissions have been fixed successfully!" diff --git a/scripts/setup-venv.sh b/scripts/setup-venv.sh old mode 100644 new mode 100755 index 46b2ee8..c309ee7 --- a/scripts/setup-venv.sh +++ b/scripts/setup-venv.sh @@ -20,8 +20,8 @@ setup_virtual_environment() { local requirements_folder="./requirements" local requirements_files=( - "$requirements_folder/lint.sh" - "$requirements_folder/build.sh" + "$requirements_folder/build.in" + "$requirements_folder/lint.in" ) # Install requirements From 0aa4fc45947f0210b147d8d73132011b64b44eb1 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 12 Aug 2025 18:15:02 +0200 Subject: [PATCH 02/22] fix: add pull request trigger for develop branch in GitHub Actions workflow --- .github/workflows/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index d3b32d6..cd10f3f 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -4,6 +4,9 @@ on: push: branches: - develop # dev trigger (automatic) + pull_request: + branches: + - develop workflow_dispatch: # prod trigger (manual) jobs: From 88b8369c098c7ceca44bb7eed9c2afd8cd41a50b Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 12 Aug 2025 18:17:19 +0200 Subject: [PATCH 03/22] fix: update dev_build job condition to trigger on all push events --- .github/workflows/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index cd10f3f..3c36fee 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -11,7 +11,6 @@ on: jobs: dev_build: - if: github.event_name == 'push' && github.ref == 'refs/heads/develop' uses: ./.github/workflows/build-and-push-image.yml with: environment: dev From 62fc6582869faf1c83c994ae648a384ebcc411aa Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 12 Aug 2025 18:18:45 +0200 Subject: [PATCH 04/22] fix: reorder condition and name for environment check in CI workflow --- .github/workflows/build-and-push-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 2b0ab57..4bbabe8 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -25,8 +25,8 @@ jobs: with: python-version: "3.11" - - name: Makes sure inputs.environment is set - if: ${{ inputs.environment != 'prod' && inputs.environment != 'dev' }} + - if: ${{ inputs.environment != 'prod' && inputs.environment != 'dev' }} + name: Makes sure inputs.environment is set run: | echo "Error: 'environment' input is required ('dev' or 'prod')." exit 1 From 15f1ec80ad3c6a4c39a2368ebf263aad0a99bfd6 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Fri, 15 Aug 2025 14:24:43 +0200 Subject: [PATCH 05/22] feat: refactor Docker setup and database initialization scripts; remove unused scripts and improve CSV handling --- .github/workflows/build-and-push-image.yml | 57 +++++++++------------ db.sh | 26 ---------- dev.Dockerfile | 43 ++++++++++++++++ prod.Dockerfile | 25 ++++++++-- scripts/clean-docker-volumes.sh | 16 ------ scripts/docker/docker-build-init.sh | 57 +++++++++++++++++++++ scripts/docker/run-migrations.sh | 11 ++++ scripts/load-csv-to-database.sh | 23 ++++----- scripts/nexus/pull-csv.sh | 41 +++++++++++++++ scripts/nexus/push-csv.sh | 32 ++++++++++++ scripts/pull-csv.sh | 22 -------- scripts/setup-db.sh | 58 ++++++++++++++++++++++ scripts/{ => user}/backup.sh | 0 scripts/{ => user}/fix-permissions.sh | 0 scripts/{ => user}/setup-venv.sh | 0 scripts/util.sh | 18 +++++++ template.env | 1 + 17 files changed, 315 insertions(+), 115 deletions(-) delete mode 100755 db.sh create mode 100644 dev.Dockerfile delete mode 100755 scripts/clean-docker-volumes.sh create mode 100755 scripts/docker/docker-build-init.sh create mode 100644 scripts/docker/run-migrations.sh create mode 100755 scripts/nexus/pull-csv.sh create mode 100755 scripts/nexus/push-csv.sh delete mode 100755 scripts/pull-csv.sh create mode 100755 scripts/setup-db.sh rename scripts/{ => user}/backup.sh (100%) rename scripts/{ => user}/fix-permissions.sh (100%) rename scripts/{ => user}/setup-venv.sh (100%) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 4bbabe8..fa610ad 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -1,4 +1,4 @@ -name: Build and Push Docker Image +name: Build and Push ImageS on: workflow_call: @@ -11,60 +11,49 @@ jobs: build: runs-on: ubuntu-latest env: - GITHUB_CI: true NEXUS_URL: ${{ secrets.NEXUS_URL }} + NEXUS_DOCKER_URL: ${{ secrets.NEXUS_DOCKER_URL }} NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - DOCKER_DATA_MODEL_CONTAINER_NAME: infocompanies-data-model-postgres + DOCKER_DATA_MODEL_NAME: infocompanies-data-model-postgres + steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - if: ${{ inputs.environment != 'prod' && inputs.environment != 'dev' }} - name: Makes sure inputs.environment is set + name: Validate environment input run: | - echo "Error: 'environment' input is required ('dev' or 'prod')." + echo "Error: 'environment' must be 'dev' or 'prod'." exit 1 - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./requirements/build.in - - - name: Copy template.env to .env - run: cp template.env .env - + - if: ${{ inputs.environment == 'dev' }} - name: Run database setup script - run: ./db.sh - - - name: Log in to Nexus Docker registry - run: echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_URL}" -u "${NEXUS_USERNAME}" --password-stdin + name: Login to Nexus native port for pulling CSVs + run: | + echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_URL}" -u "${NEXUS_USERNAME}" --password-stdin + ./scripts/pull-csv.sh + docker logout "${NEXUS_URL}" - - name: Generate a tag, commit and push the image to Nexus + - name: Build prefilled Postgres image run: | DOCKER_TAG=$(date +%Y%m%d%H%M%S) - IMAGE="${NEXUS_URL}/${DOCKER_DATA_MODEL_CONTAINER_NAME}-${{ inputs.environment }}:${DOCKER_TAG}" - if [ "${{ inputs.environment }}" = 'dev' ]; then - docker commit $DOCKER_DATA_MODEL_CONTAINER_NAME $IMAGE + if [ "${{ inputs.environment }}" == "dev" ]; then + DOCKERFILE="dev.Dockerfile" else - docker build -t $IMAGE -f prod.Dockerfile . + DOCKERFILE="prod.Dockerfile" fi + IMAGE="${NEXUS_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${DOCKER_TAG}" + docker build -f "${DOCKERFILE}" -t $IMAGE . + + echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_DOCKER_URL}" -u "${NEXUS_USERNAME}" --password-stdin docker push $IMAGE - echo "Docker image pushed: $IMAGE" + docker logout "${NEXUS_DOCKER_URL}" - - name: Remove Nexus login credentials - run: docker logout ${NEXUS_URL} - name: Clean up Docker resources run: | - docker compose down || echo "No docker-compose file found or no containers running" + docker rm -f "${DOCKER_DATA_MODEL_NAME}" || true + docker volume rm pgdata || true docker system prune -f - echo "Cleaned up Docker resources." diff --git a/db.sh b/db.sh deleted file mode 100755 index 2cfe476..0000000 --- a/db.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# shellcheck disable=SC1091 -source ./scripts/util.sh - - -# Main script -if [[ "${GITHUB_CI:-}" == "true" ]]; then - ./scripts/pull-csv.sh -fi - -docker compose up -d --quiet-pull -sleep 2 - -log_info "Initializing the database." -cd ./schema -alembic upgrade head - -log_info "Executing Database.py" -PYTHONPATH=. python3 app/database/database.py -cd .. - - -./scripts/load-csv-to-database.sh diff --git a/dev.Dockerfile b/dev.Dockerfile new file mode 100644 index 0000000..2ee056f --- /dev/null +++ b/dev.Dockerfile @@ -0,0 +1,43 @@ +# =============================== +# Builder stage – preload database +# =============================== +FROM postgres:16.4 AS builder + +ENV POSTGRES_USER=postgres +ENV POSTGRES_PASSWORD=root +ENV POSTGRES_DB=postgres +ENV PGDATA=/var/lib/postgresql/data + +# Install Python & dependencies for Alembic +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-pip postgresql-client \ + && python3 -m venv /opt/venv \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/opt/venv/bin:$PATH" + +# Copy requirements and install +COPY requirements/build.in /tmp/requirements.in +RUN pip install --no-cache-dir -r /tmp/requirements.in + +# Copy app code and CSVs +WORKDIR /app +COPY schema ./schema +COPY scripts ./scripts +COPY scripts/setup-db.sh /app/setup-db.sh +COPY final.csv leaders.csv fichier_combine_updated_big_fixed.csv ./ + +# Make ./data writable by postgres user +RUN mkdir -p /app/data && chown -R postgres:postgres /app/data + +# Start Postgres for migrations and CSV loading +USER postgres +RUN /app/setup-db.sh + +# =============================== +# Final image – production-ready +# =============================== +FROM postgres:16.4 + +# Copy preloaded, vacuumed database +COPY --from=builder /var/lib/postgresql/data /var/lib/postgresql/data diff --git a/prod.Dockerfile b/prod.Dockerfile index 551cfb7..29d18bd 100644 --- a/prod.Dockerfile +++ b/prod.Dockerfile @@ -1,6 +1,25 @@ FROM postgres:16.4 -WORKDIR /data-model/ +# Install Python & dependencies for Alembic +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-pip postgresql-client \ + && python3 -m venv /opt/venv \ + && rm -rf /var/lib/apt/lists/* -# Copy requirements files and schema -COPY ./requirements/build.in ./schema/ /data-model/ +ENV PATH="/opt/venv/bin:$PATH" + +# Copy requirements and install as root (postgres user cannot write to venv yet) +COPY requirements/build.in /tmp/requirements.in +RUN pip install --no-cache-dir -r /tmp/requirements.in + +# Copy app code +WORKDIR /app +COPY schema ./schema +COPY scripts ./scripts + +# Make migration script executable +COPY ./scripts/docker/run-migrations.sh /docker-entrypoint-initdb.d/01_run_migrations.sh +RUN chmod +x /docker-entrypoint-initdb.d/01_run_migrations.sh + +# Switch to postgres user (final step) +USER postgres diff --git a/scripts/clean-docker-volumes.sh b/scripts/clean-docker-volumes.sh deleted file mode 100755 index c6c2540..0000000 --- a/scripts/clean-docker-volumes.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# shellcheck disable=SC1091 -source ./scripts/util.sh -volume="$(get_infocompanies_data_model_postgres_volume)" - - -log_info "Stopping Docker containers..." -docker compose down - -log_info "Cleaning up Docker volume..." -docker volume rm -f "$volume" - -log_success "Docker volume cleaned up." \ No newline at end of file diff --git a/scripts/docker/docker-build-init.sh b/scripts/docker/docker-build-init.sh new file mode 100755 index 0000000..d9da14c --- /dev/null +++ b/scripts/docker/docker-build-init.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +echo ">>> Initializing Postgres data at build time..." + +# Init database cluster +initdb -D /var/lib/postgresql/data + +# Prepend Docker subnet access to pg_hba.conf +# This ensures it is checked BEFORE any default scram-sha-256 lines +sed -i "1ihost all all 172.19.0.0/16 md5" /var/lib/postgresql/data/pg_hba.conf + +# Start Postgres on IPv4 localhost +pg_ctl -D /var/lib/postgresql/data \ + -o "-c listen_addresses='*' -c unix_socket_directories='/var/run/postgresql'" \ + -w start + +# Wait until Postgres is ready +until pg_isready -U postgres -h 127.0.0.1 -p 5432; do + echo "Waiting for Postgres to be ready..." + sleep 1 +done + +# Before running migrations / CSV load +psql -v ON_ERROR_STOP=1 --username=postgres <<-EOSQL + ALTER SYSTEM SET max_wal_size = '6GB'; + ALTER SYSTEM SET checkpoint_timeout = '30min'; + ALTER SYSTEM SET synchronous_commit = off; + ALTER SYSTEM SET fsync = off; + ALTER SYSTEM SET full_page_writes = off; +EOSQL +pg_ctl -D /var/lib/postgresql/data reload + +# Apply migrations +export DATABASE_URL=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@127.0.0.1:5432/$POSTGRES_DB +cd /app/schema +PYTHONPATH=/app alembic upgrade head + +# Populate with CSV data +PYTHONPATH=/app python /app/app/database/database.py +cd .. +/app/scripts/load-csv-to-database.sh + +# Restore safer defaults +psql -v ON_ERROR_STOP=1 --username=postgres <<-EOSQL + ALTER SYSTEM RESET max_wal_size; + ALTER SYSTEM RESET checkpoint_timeout; + ALTER SYSTEM RESET synchronous_commit; + ALTER SYSTEM RESET fsync; + ALTER SYSTEM RESET full_page_writes; +EOSQL +pg_ctl -D /var/lib/postgresql/data reload + +# Stop Postgres +pg_ctl -D /var/lib/postgresql/data -m fast -w stop + +echo ">>> Database populated and baked into image." diff --git a/scripts/docker/run-migrations.sh b/scripts/docker/run-migrations.sh new file mode 100644 index 0000000..444bc2b --- /dev/null +++ b/scripts/docker/run-migrations.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source ./scripts/util.sh + +log_info "Initializing the database." +cd ./schema +alembic upgrade head +log_info "Database initialized." \ No newline at end of file diff --git a/scripts/load-csv-to-database.sh b/scripts/load-csv-to-database.sh index 52d7d6b..f2096e6 100755 --- a/scripts/load-csv-to-database.sh +++ b/scripts/load-csv-to-database.sh @@ -27,13 +27,10 @@ transfer_csv_to_database() { exit 1 fi - local postgres_container - postgres_container=$(get_infocompanies_data_model_postgres_container) - log_info "Transferring the CSV file '$csv_file_path' to the PostgreSQL database." - docker cp "$csv_file_path" "$postgres_container:$container_csv_file" - docker exec -u postgres -i "$postgres_container" psql -d postgres -c "COPY $table_name($columns) FROM '$container_csv_file' DELIMITER '$delimiter' CSV HEADER;" + cp "$csv_file_path" "$container_csv_file" + psql -d postgres -c "COPY $table_name($columns) FROM '$container_csv_file' DELIMITER '$delimiter' CSV HEADER;" log_success "Transfer of '$csv_file_path' to the database table '$table_name' completed successfully." } @@ -46,8 +43,6 @@ export_unique_values() { local format="${3:-csv}" # Default format is CSV local base_name base_name=$(basename "$output_file") - local postgres_container - postgres_container=$(get_infocompanies_data_model_postgres_container) mkdir -p "$(dirname "$output_file")" @@ -55,25 +50,25 @@ export_unique_values() { local output_csv="/tmp/$base_name.csv" log_info "Exporting unique values for the $base_name table in CSV format..." - docker exec -u postgres -i "$postgres_container" mkdir -p /tmp - docker exec -u postgres -i "$postgres_container" psql -d postgres -c "\copy ($query) TO '$output_csv' CSV HEADER;" - docker cp "$postgres_container:$output_csv" "$output_file" - + mkdir -p /tmp + psql -d postgres -c "\copy ($query) TO '$output_csv' CSV HEADER;" + cp "$output_csv" "$output_file" log_success "Exported CSV file saved to $output_file" + elif [ "$format" == "sql" ]; then log_info "Exporting unique values for the $base_name table in SQL format..." local temp_table="temp_export" local output_sql="$output_file.sql" - docker exec -u postgres -i "$postgres_container" psql -d postgres -c " + psql -d postgres -c " DROP TABLE IF EXISTS $temp_table; CREATE TABLE $temp_table AS SELECT row_number() OVER () AS id, * FROM ($query) AS subquery; " - docker exec -u postgres -i "$postgres_container" pg_dump -U postgres --data-only --table="$temp_table" postgres >"$output_sql" - docker exec -u postgres -i "$postgres_container" psql -d postgres -c "DROP TABLE IF EXISTS $temp_table;" + pg_dump -U postgres --data-only --table="$temp_table" postgres >"$output_sql" + psql -d postgres -c "DROP TABLE IF EXISTS $temp_table;" log_success "Exported SQL file saved to $output_sql" else diff --git a/scripts/nexus/pull-csv.sh b/scripts/nexus/pull-csv.sh new file mode 100755 index 0000000..bf5c4de --- /dev/null +++ b/scripts/nexus/pull-csv.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source ./scripts/util.sh +# shellcheck disable=SC1091 +source .env + +# Reminder: we use the native Nexus port and not the Docker one +NEXUS_CSV_URL="${NEXUS_URL}/repository/datasets" +USERNAME_WITH_PASSWORD=$(build_nexus_username_with_password "$NEXUS_USERNAME" "$NEXUS_PASSWORD") + +# TODO: Change that to be only done by the CI +FINAL_TAG="0.1" +LEADER_TAG="0.1" +FICHIER_COMBINE_TAG="0.1" + +LEADER_CSV="leaders-$LEADER_TAG.csv.gz" +FINAL_CSV="final-$FINAL_TAG.csv.gz" +FICHIER_COMBINE_CSV="fichier_combine_updated_big_fixed-$FICHIER_COMBINE_TAG.csv.gz" + +# Pull the CSVs to insert +log_info "Pulling the CSVs..." +curl -u "$USERNAME_WITH_PASSWORD" -O "$NEXUS_CSV_URL/$LEADER_CSV" +curl -u "$USERNAME_WITH_PASSWORD" -O "$NEXUS_CSV_URL/$FINAL_CSV" +curl -u "$USERNAME_WITH_PASSWORD" -O "$NEXUS_CSV_URL/$FICHIER_COMBINE_CSV" +log_success "CSV files pulled successfully." + +# Uncompress the CSVs +log_info "Uncompressing the CSVs..." +gunzip "$LEADER_CSV" +gunzip "$FINAL_CSV" +gunzip "$FICHIER_COMBINE_CSV" + +# Rename files to remove version tags +mv "leaders-$LEADER_TAG.csv" "leaders.csv" +mv "final-$FINAL_TAG.csv" "final.csv" +mv "fichier_combine_updated_big_fixed-$FICHIER_COMBINE_TAG.csv" "fichier_combine_updated_big_fixed.csv" + +log_success "CSV files uncompressed successfully." \ No newline at end of file diff --git a/scripts/nexus/push-csv.sh b/scripts/nexus/push-csv.sh new file mode 100755 index 0000000..63cce66 --- /dev/null +++ b/scripts/nexus/push-csv.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source ./scripts/util.sh +# shellcheck disable=SC1091 +source .env + +# Reminder: we use the native Nexus port and not the Docker one +NEXUS_CSV_URL="${NEXUS_URL}/repository/datasets" +USERNAME_WITH_PASSWORD=$(build_nexus_username_with_password "$NEXUS_USERNAME" "$NEXUS_PASSWORD") + + +# TODO: Change that to be only done by the CI +FINAL_TAG="0.1" +LEADER_TAG="0.1" +FICHIER_COMBINE_TAG="0.1" + +echo "Pushing CSVs to Nexus..." +curl -u "$USERNAME_WITH_PASSWORD" \ + --upload-file ./final.csv.gz \ + "$NEXUS_CSV_URL/final-${FINAL_TAG}.csv.gz" + +curl -u "$USERNAME_WITH_PASSWORD" \ + --upload-file ./leaders.csv.gz \ + "$NEXUS_CSV_URL/leaders-${LEADER_TAG}.csv.gz" + +curl -u "$USERNAME_WITH_PASSWORD" \ + --upload-file ./fichier_combine_updated_big_fixed.csv.gz \ + "$NEXUS_CSV_URL/fichier_combine_updated_big_fixed-${FICHIER_COMBINE_TAG}.csv.gz" +echo "CSV files pushed successfully." \ No newline at end of file diff --git a/scripts/pull-csv.sh b/scripts/pull-csv.sh deleted file mode 100755 index 3716788..0000000 --- a/scripts/pull-csv.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# shellcheck disable=SC1091 -source ./scripts/util.sh - - -# Pull the CSVs to insert -log_info "Pulling the CSVs..." -curl -u "$NEXUS_USERNAME:$NEXUS_PASSWORD" -O "$NEXUS_URL/repository/datasets/versions/v1.0/leaders.csv.gz" -curl -u "$NEXUS_USERNAME:$NEXUS_PASSWORD" -O "$NEXUS_URL/repository/datasets/versions/v1.0/final.csv.gz" -curl -u "$NEXUS_USERNAME:$NEXUS_PASSWORD" -O "$NEXUS_URL/repository/datasets/versions/v1.0/fichier_combine_updated_big_fixed.csv.gz" -log_success "CSV files pulled successfully." - -# Uncompress the CSVs -log_info "Uncompressing the CSVs..." -gunzip leaders.csv.gz -gunzip final.csv.gz -gunzip fichier_combine_updated_big_fixed.csv.gz - -log_success "CSV files uncompressed successfully." \ No newline at end of file diff --git a/scripts/setup-db.sh b/scripts/setup-db.sh new file mode 100755 index 0000000..9616c66 --- /dev/null +++ b/scripts/setup-db.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -e + +# Start Postgres in background +docker-entrypoint.sh postgres & + +# Wait for Postgres to be ready +echo "Waiting for Postgres..." +sleep 5 + +export PGHOST=localhost +export PGUSER=postgres +export PGPASSWORD=root + +# Test connection +psql -d postgres -c "SELECT 1;" + +# Run migrations +cd /app/schema +alembic upgrade head +PYTHONPATH=. python ./app/database/database.py +cd /app + +psql -v ON_ERROR_STOP=1 --username="$POSTGRES_USER" < Date: Fri, 15 Aug 2025 14:32:21 +0200 Subject: [PATCH 06/22] fix: clean up workflow files and ensure secrets are properly defined for image build --- .github/workflows/action.yml | 6 ++++-- .github/workflows/build-and-push-image.yml | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 3c36fee..0f99304 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -3,20 +3,22 @@ name: Build and Push Docker Image on: push: branches: - - develop # dev trigger (automatic) + - develop # dev trigger (automatic) pull_request: branches: - develop - workflow_dispatch: # prod trigger (manual) + workflow_dispatch: # prod trigger (manual) jobs: dev_build: uses: ./.github/workflows/build-and-push-image.yml with: environment: dev + secrets: inherit prod_build: if: github.event_name == 'workflow_dispatch' uses: ./.github/workflows/build-and-push-image.yml with: environment: prod + secrets: inherit diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index fa610ad..be48d27 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -6,6 +6,15 @@ on: environment: required: true type: string + secrets: + NEXUS_URL: + required: true + NEXUS_DOCKER_URL: + required: true + NEXUS_USERNAME: + required: true + NEXUS_PASSWORD: + required: true jobs: build: @@ -30,10 +39,21 @@ jobs: - if: ${{ inputs.environment == 'dev' }} name: Login to Nexus native port for pulling CSVs run: | + # Create temporary .env file for the script + cat > .env << EOF + NEXUS_URL=${NEXUS_URL} + NEXUS_DOCKER_URL=${NEXUS_DOCKER_URL} + NEXUS_USERNAME=${NEXUS_USERNAME} + NEXUS_PASSWORD=${NEXUS_PASSWORD} + EOF + echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_URL}" -u "${NEXUS_USERNAME}" --password-stdin - ./scripts/pull-csv.sh + ./scripts/nexus/pull-csv.sh docker logout "${NEXUS_URL}" + # Clean up .env file + rm -f .env + - name: Build prefilled Postgres image run: | DOCKER_TAG=$(date +%Y%m%d%H%M%S) From 28a2cd716fe9a0d1e03693c37dac6e63a7b37a9f Mon Sep 17 00:00:00 2001 From: Matithieu Date: Sat, 16 Aug 2025 17:23:51 +0200 Subject: [PATCH 07/22] fix: streamline CSV push script and improve logging for successful uploads --- .github/workflows/build-and-push-image.yml | 21 +++------------------ scripts/nexus/push-csv.sh | 7 ++++++- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index be48d27..7759bcd 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -35,24 +35,10 @@ jobs: run: | echo "Error: 'environment' must be 'dev' or 'prod'." exit 1 - - - if: ${{ inputs.environment == 'dev' }} - name: Login to Nexus native port for pulling CSVs - run: | - # Create temporary .env file for the script - cat > .env << EOF - NEXUS_URL=${NEXUS_URL} - NEXUS_DOCKER_URL=${NEXUS_DOCKER_URL} - NEXUS_USERNAME=${NEXUS_USERNAME} - NEXUS_PASSWORD=${NEXUS_PASSWORD} - EOF - - echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_URL}" -u "${NEXUS_USERNAME}" --password-stdin - ./scripts/nexus/pull-csv.sh - docker logout "${NEXUS_URL}" - # Clean up .env file - rm -f .env + - if: ${{ inputs.environment == 'dev' }} + name: Pull CSV files from Nexus + run: ./scripts/nexus/pull-csv.sh - name: Build prefilled Postgres image run: | @@ -71,7 +57,6 @@ jobs: docker push $IMAGE docker logout "${NEXUS_DOCKER_URL}" - - name: Clean up Docker resources run: | docker rm -f "${DOCKER_DATA_MODEL_NAME}" || true diff --git a/scripts/nexus/push-csv.sh b/scripts/nexus/push-csv.sh index 63cce66..3726e5a 100755 --- a/scripts/nexus/push-csv.sh +++ b/scripts/nexus/push-csv.sh @@ -8,9 +8,10 @@ source ./scripts/util.sh source .env # Reminder: we use the native Nexus port and not the Docker one -NEXUS_CSV_URL="${NEXUS_URL}/repository/datasets" +NEXUS_CSV_URL="$NEXUS_URL/repository/datasets" USERNAME_WITH_PASSWORD=$(build_nexus_username_with_password "$NEXUS_USERNAME" "$NEXUS_PASSWORD") +echo "Nexus CSV URL: $NEXUS_CSV_URL" # TODO: Change that to be only done by the CI FINAL_TAG="0.1" @@ -21,12 +22,16 @@ echo "Pushing CSVs to Nexus..." curl -u "$USERNAME_WITH_PASSWORD" \ --upload-file ./final.csv.gz \ "$NEXUS_CSV_URL/final-${FINAL_TAG}.csv.gz" +echo "CSV 'final' pushed successfully." curl -u "$USERNAME_WITH_PASSWORD" \ --upload-file ./leaders.csv.gz \ "$NEXUS_CSV_URL/leaders-${LEADER_TAG}.csv.gz" +echo "CSV 'leaders' pushed successfully." curl -u "$USERNAME_WITH_PASSWORD" \ --upload-file ./fichier_combine_updated_big_fixed.csv.gz \ "$NEXUS_CSV_URL/fichier_combine_updated_big_fixed-${FICHIER_COMBINE_TAG}.csv.gz" +echo "CSV 'fichier_combine_updated_big_fixed' pushed successfully." + echo "CSV files pushed successfully." \ No newline at end of file From 89d8adf591f720586ad54864bff707f42c33d4c1 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Sat, 16 Aug 2025 17:24:34 +0200 Subject: [PATCH 08/22] fix: create temporary .env file for Nexus CSV pull script and ensure cleanup --- .github/workflows/build-and-push-image.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 7759bcd..ab9203b 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -38,7 +38,19 @@ jobs: - if: ${{ inputs.environment == 'dev' }} name: Pull CSV files from Nexus - run: ./scripts/nexus/pull-csv.sh + run: | + # Create temporary .env file for the script + cat > .env << EOF + NEXUS_URL=${NEXUS_URL} + NEXUS_DOCKER_URL=${NEXUS_DOCKER_URL} + NEXUS_USERNAME=${NEXUS_USERNAME} + NEXUS_PASSWORD=${NEXUS_PASSWORD} + EOF + + ./scripts/nexus/pull-csv.sh + + # Clean up .env file + rm -f .env - name: Build prefilled Postgres image run: | From f921003fb0af7357ffc368b1ea510a991cd5b1c7 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Sat, 16 Aug 2025 17:30:13 +0200 Subject: [PATCH 09/22] fix: correct Docker image URL in build step to use NEXUS_DOCKER_URL --- .github/workflows/build-and-push-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index ab9203b..31eb6d6 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -62,7 +62,7 @@ jobs: DOCKERFILE="prod.Dockerfile" fi - IMAGE="${NEXUS_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${DOCKER_TAG}" + IMAGE="${NEXUS_DOCKER_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${DOCKER_TAG}" docker build -f "${DOCKERFILE}" -t $IMAGE . echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_DOCKER_URL}" -u "${NEXUS_USERNAME}" --password-stdin From 616de8781f37564a164023deea58c0c4f2eb253a Mon Sep 17 00:00:00 2001 From: Matithieu Date: Wed, 20 Aug 2025 18:24:13 +0200 Subject: [PATCH 10/22] fix: Column ID type to BIGINT + free up disk space on runner --- .github/workflows/build-and-push-image.yml | 7 +++++++ schema/alembic/versions/57a5a84e79d0_init_db.py | 14 +++++++------- schema/app/models/autocomplete.py | 10 +++++----- schema/app/models/config.py | 4 ++-- schema/app/models/user_company_status.py | 6 +++--- scripts/load-csv-to-database.sh | 2 +- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 31eb6d6..6f722c1 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -36,6 +36,13 @@ jobs: echo "Error: 'environment' must be 'dev' or 'prod'." exit 1 + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + df -h + - if: ${{ inputs.environment == 'dev' }} name: Pull CSV files from Nexus run: | diff --git a/schema/alembic/versions/57a5a84e79d0_init_db.py b/schema/alembic/versions/57a5a84e79d0_init_db.py index 1e3d1be..d052a9f 100644 --- a/schema/alembic/versions/57a5a84e79d0_init_db.py +++ b/schema/alembic/versions/57a5a84e79d0_init_db.py @@ -237,20 +237,20 @@ def upgrade() -> None: op.create_index("idx_leader_siren", "leaders", ["siren"], unique=False) op.create_table( "config", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("last_reset_quota_date", sa.Date(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_table( "city", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_index("ix_city_name", "city", ["name"], unique=False) op.create_table( "industry_sector", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) @@ -259,14 +259,14 @@ def upgrade() -> None: ) op.create_table( "legal_form", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_index("ix_legal_form_name", "legal_form", ["name"], unique=False) op.create_table( "region", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) @@ -287,12 +287,12 @@ def upgrade() -> None: op.create_index("ix_user_quota_user_id", "user_quota", ["user_id"], unique=False) op.create_table( "user_company_status", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("user_id", sa.String(), nullable=True), sa.Column( "status", sa.Enum("NOT_DONE", "TO_DO", "DONE", name="status"), nullable=True ), - sa.Column("company_id", sa.BigInteger(), nullable=True), + sa.Column("company_id", sa.Integer(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_index( diff --git a/schema/app/models/autocomplete.py b/schema/app/models/autocomplete.py index c2cf99d..9f0dd16 100644 --- a/schema/app/models/autocomplete.py +++ b/schema/app/models/autocomplete.py @@ -1,4 +1,4 @@ -from sqlalchemy import BigInteger, Column, Index, String +from sqlalchemy import Column, Index, Integer, String from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -7,7 +7,7 @@ class City(Base): __tablename__ = "city" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_city_name", "name"),) @@ -16,7 +16,7 @@ class City(Base): class IndustrySector(Base): __tablename__ = "industry_sector" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_industry_sector_name", "name"),) @@ -25,7 +25,7 @@ class IndustrySector(Base): class LegalForm(Base): __tablename__ = "legal_form" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_legal_form_name", "name"),) @@ -34,7 +34,7 @@ class LegalForm(Base): class Region(Base): __tablename__ = "region" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_region_name", "name"),) diff --git a/schema/app/models/config.py b/schema/app/models/config.py index f5d7136..e66b787 100644 --- a/schema/app/models/config.py +++ b/schema/app/models/config.py @@ -1,4 +1,4 @@ -from sqlalchemy import BigInteger, Column, Date +from sqlalchemy import Column, Date, Integer from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -7,5 +7,5 @@ class Config(Base): __tablename__ = "config" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) last_reset_quota_date = Column(Date) diff --git a/schema/app/models/user_company_status.py b/schema/app/models/user_company_status.py index ee92442..3d49bc1 100644 --- a/schema/app/models/user_company_status.py +++ b/schema/app/models/user_company_status.py @@ -1,6 +1,6 @@ import enum -from sqlalchemy import BigInteger, Column, Enum, Index, String +from sqlalchemy import Column, Enum, Index, Integer, String from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -15,10 +15,10 @@ class Status(enum.Enum): class UserCompanyStatus(Base): __tablename__ = "user_company_status" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) user_id = Column(String) status = Column(Enum(Status)) - company_id = Column(BigInteger) + company_id = Column(Integer) __table_args__ = ( Index("ix_user_company_status_user_id", "user_id"), diff --git a/scripts/load-csv-to-database.sh b/scripts/load-csv-to-database.sh index f2096e6..5be354a 100755 --- a/scripts/load-csv-to-database.sh +++ b/scripts/load-csv-to-database.sh @@ -32,7 +32,7 @@ transfer_csv_to_database() { cp "$csv_file_path" "$container_csv_file" psql -d postgres -c "COPY $table_name($columns) FROM '$container_csv_file' DELIMITER '$delimiter' CSV HEADER;" - log_success "Transfer of '$csv_file_path' to the database table '$table_name' completed successfully." + log_success "Transfer of '$container_csv_file' to the database table '$table_name' completed successfully." } From 2fa041d73b54fc27991d5cf7a309cb34aa372623 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Wed, 20 Aug 2025 19:39:43 +0200 Subject: [PATCH 11/22] fix: optimize Docker image build process by adjusting variable scope and limiting concurrent uploads --- .github/workflows/build-and-push-image.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 6f722c1..80affc1 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -61,19 +61,18 @@ jobs: - name: Build prefilled Postgres image run: | - DOCKER_TAG=$(date +%Y%m%d%H%M%S) - if [ "${{ inputs.environment }}" == "dev" ]; then - DOCKERFILE="dev.Dockerfile" + DOCKERFILE="dev.Dockerfile" else - DOCKERFILE="prod.Dockerfile" + DOCKERFILE="prod.Dockerfile" fi + DOCKER_TAG=$(date +%Y%m%d%H%M%S) IMAGE="${NEXUS_DOCKER_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${DOCKER_TAG}" docker build -f "${DOCKERFILE}" -t $IMAGE . echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_DOCKER_URL}" -u "${NEXUS_USERNAME}" --password-stdin - docker push $IMAGE + docker push --max-concurrent-uploads=1 $IMAGE docker logout "${NEXUS_DOCKER_URL}" - name: Clean up Docker resources From abf02564cbb36ab00054daf1cccf5ade8219b723 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:14:39 +0200 Subject: [PATCH 12/22] fix: automatic tagging + automatic prod build --- .github/workflows/action.yml | 3 +- .github/workflows/build-and-push-image.yml | 57 ++++++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 0f99304..d4029d5 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -11,13 +11,14 @@ on: jobs: dev_build: + if: ${{ github.event_name == 'push' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev secrets: inherit prod_build: - if: github.event_name == 'workflow_dispatch' + if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: prod diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 80affc1..1586fc3 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -1,6 +1,12 @@ name: Build and Push ImageS on: + workflow_dispatch: + inputs: + MANUAL_TAG: + description: 'Optional manual tag to override versioning' + required: false + default: '' workflow_call: inputs: environment: @@ -16,6 +22,9 @@ on: NEXUS_PASSWORD: required: true +permissions: + contents: write + jobs: build: runs-on: ubuntu-latest @@ -43,6 +52,36 @@ jobs: sudo rm -rf /opt/ghc df -h + - name: Get the last Git tag + id: get_last_tag + run: | + git fetch --tags --force + LAST_TAG=$(git tag --sort=-creatordate | head -n 1) + if [ -z "$LAST_TAG" ]; then + LAST_TAG="0.0.0" + fi + echo "LAST_TAG=${LAST_TAG}" >> $GITHUB_ENV + + - name: Calculate new version + id: calculate_version + run: | + LAST_TAG=${{ env.LAST_TAG }} + IFS='.' read -r MAJOR MINOR PATCH <<< "$LAST_TAG" + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + PATCH=$((PATCH + 1)) + else + MINOR=$((MINOR + 1)) + PATCH=0 + fi + + NEW_VERSION="${MAJOR}.${MINOR}.${PATCH}" + + if [[ "${{ github.event.inputs.MANUAL_TAG }}" != "" ]]; then + NEW_VERSION="${{ github.event.inputs.MANUAL_TAG}}" + fi + echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV + - if: ${{ inputs.environment == 'dev' }} name: Pull CSV files from Nexus run: | @@ -67,12 +106,11 @@ jobs: DOCKERFILE="prod.Dockerfile" fi - DOCKER_TAG=$(date +%Y%m%d%H%M%S) - IMAGE="${NEXUS_DOCKER_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${DOCKER_TAG}" + IMAGE="${NEXUS_DOCKER_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${NEW_VERSION}" docker build -f "${DOCKERFILE}" -t $IMAGE . echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_DOCKER_URL}" -u "${NEXUS_USERNAME}" --password-stdin - docker push --max-concurrent-uploads=1 $IMAGE + docker push $IMAGE docker logout "${NEXUS_DOCKER_URL}" - name: Clean up Docker resources @@ -80,3 +118,16 @@ jobs: docker rm -f "${DOCKER_DATA_MODEL_NAME}" || true docker volume rm pgdata || true docker system prune -f + + - name: Create and push Git tag + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + git tag -a "${NEW_VERSION}" -m "Release ${NEW_VERSION}" + git push origin "${NEW_VERSION}" --no-verify + + - name: Output new version + run: echo "New version is ${{ env.NEW_VERSION }}" From c2711a608fefc0253417653222d4b08dacad745e Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:15:31 +0200 Subject: [PATCH 13/22] remove push trigger --- .github/workflows/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index d4029d5..9f93d8d 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -11,7 +11,6 @@ on: jobs: dev_build: - if: ${{ github.event_name == 'push' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev From 93c937d5d59ccdc260de61d96aa32965899642dd Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:18:19 +0200 Subject: [PATCH 14/22] fix: update workflow_dispatch inputs for manual tagging in build process --- .github/workflows/action.yml | 8 +++++++- .github/workflows/build-and-push-image.yml | 14 ++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 9f93d8d..1bc68b1 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -7,10 +7,16 @@ on: pull_request: branches: - develop - workflow_dispatch: # prod trigger (manual) + workflow_dispatch: + inputs: + MANUAL_TAG: + description: 'Optional manual tag to override versioning' + required: false + default: '' jobs: dev_build: + if: ${{ github.event_name == 'pull_request' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 1586fc3..d9cd6dd 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -1,17 +1,15 @@ name: Build and Push ImageS on: - workflow_dispatch: - inputs: - MANUAL_TAG: - description: 'Optional manual tag to override versioning' - required: false - default: '' workflow_call: inputs: environment: required: true type: string + manual_tag: + required: false + type: string + default: '' secrets: NEXUS_URL: required: true @@ -77,8 +75,8 @@ jobs: NEW_VERSION="${MAJOR}.${MINOR}.${PATCH}" - if [[ "${{ github.event.inputs.MANUAL_TAG }}" != "" ]]; then - NEW_VERSION="${{ github.event.inputs.MANUAL_TAG}}" + if [[ "${{ inputs.manual_tag }}" != "" ]]; then + NEW_VERSION="${{ inputs.manual_tag}}" fi echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV From 98be226788683c9e02f290e80767230280b08cff Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:19:41 +0200 Subject: [PATCH 15/22] fix: remove pull_request condition for dev_build job in workflow --- .github/workflows/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 1bc68b1..0778178 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -16,7 +16,6 @@ on: jobs: dev_build: - if: ${{ github.event_name == 'pull_request' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev From 945dcf53b61301ca3ec59d521c794bc96712ed5b Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:22:49 +0200 Subject: [PATCH 16/22] fix: standardize manual_tag casing and improve version calculation handling --- .github/workflows/action.yml | 9 +++++++-- .github/workflows/build-and-push-image.yml | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 0778178..bbe94f1 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -5,25 +5,30 @@ on: branches: - develop # dev trigger (automatic) pull_request: + types: + - closed branches: - develop workflow_dispatch: inputs: - MANUAL_TAG: + manual_tag: description: 'Optional manual tag to override versioning' required: false default: '' jobs: dev_build: + if: ${{ github.event_name == 'push' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev + manual_tag: ${{ inputs.manual_tag }} secrets: inherit prod_build: - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) }} uses: ./.github/workflows/build-and-push-image.yml with: environment: prod + manual_tag: ${{ inputs.manual_tag }} secrets: inherit diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index d9cd6dd..de2c399 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -63,7 +63,7 @@ jobs: - name: Calculate new version id: calculate_version run: | - LAST_TAG=${{ env.LAST_TAG }} + LAST_TAG="${LAST_TAG}" IFS='.' read -r MAJOR MINOR PATCH <<< "$LAST_TAG" if [[ "${{ github.event_name }}" == "pull_request" ]]; then @@ -76,7 +76,7 @@ jobs: NEW_VERSION="${MAJOR}.${MINOR}.${PATCH}" if [[ "${{ inputs.manual_tag }}" != "" ]]; then - NEW_VERSION="${{ inputs.manual_tag}}" + NEW_VERSION="${{ inputs.manual_tag }}" fi echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV @@ -128,4 +128,4 @@ jobs: git push origin "${NEW_VERSION}" --no-verify - name: Output new version - run: echo "New version is ${{ env.NEW_VERSION }}" + run: echo "New version is ${NEW_VERSION}" From ee1d52ce2c00a35ff233a850e95d24a1485804b9 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:24:22 +0200 Subject: [PATCH 17/22] fix: adjust permissions for workflow files and clean up redundant entries --- .github/workflows/action.yml | 3 +++ .github/workflows/build-and-push-image.yml | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index bbe94f1..c723f4d 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -16,6 +16,9 @@ on: required: false default: '' +permissions: + contents: write + jobs: dev_build: if: ${{ github.event_name == 'push' }} diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index de2c399..400bb75 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -20,9 +20,6 @@ on: NEXUS_PASSWORD: required: true -permissions: - contents: write - jobs: build: runs-on: ubuntu-latest From 9f97f01892750852f81bad94c6794a51beb217da Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:26:02 +0200 Subject: [PATCH 18/22] fix: remove push condition for dev_build job in workflow --- .github/workflows/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index c723f4d..de55827 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -21,7 +21,6 @@ permissions: jobs: dev_build: - if: ${{ github.event_name == 'push' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev From 9e7ebce2a130e6dca47748ea7865a81ec576c238 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:27:41 +0200 Subject: [PATCH 19/22] fix: remove closed pull_request type and clarify dev_build job condition --- .github/workflows/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index de55827..d4c79f1 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -5,8 +5,6 @@ on: branches: - develop # dev trigger (automatic) pull_request: - types: - - closed branches: - develop workflow_dispatch: @@ -21,6 +19,7 @@ permissions: jobs: dev_build: + if: ${{ github.event_name == 'push' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev From a004df4d641a53891c9e816e34ed71da8e1319fb Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:28:39 +0200 Subject: [PATCH 20/22] fix: update dev_build job condition to exclude pull_request events --- .github/workflows/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index d4c79f1..5c4a27b 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -19,7 +19,7 @@ permissions: jobs: dev_build: - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev From abfa40ad904d1403163898ab72409fd152407987 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Tue, 2 Sep 2025 22:29:17 +0200 Subject: [PATCH 21/22] fix: update dev_build job condition to trigger only on pull_request events --- .github/workflows/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 5c4a27b..4cf972c 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -19,7 +19,7 @@ permissions: jobs: dev_build: - if: ${{ github.event_name != 'pull_request' }} + if: ${{ github.event_name == 'pull_request' }} uses: ./.github/workflows/build-and-push-image.yml with: environment: dev From 6ae451d9611573f8f355cf7015302e86c5821264 Mon Sep 17 00:00:00 2001 From: Matithieu Date: Sun, 7 Sep 2025 17:52:50 +0200 Subject: [PATCH 22/22] fix: update manual_tag reference in prod_build job to use github.event.inputs --- .github/workflows/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 4cf972c..761988d 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -31,5 +31,5 @@ jobs: uses: ./.github/workflows/build-and-push-image.yml with: environment: prod - manual_tag: ${{ inputs.manual_tag }} + manual_tag: ${{ github.event.inputs.manual_tag }} secrets: inherit