Build and push image to nexus

.github/workflows/action.yml

@@ -1,73 +1,35 @@
-name: CI
+name: Build and Push Docker Image
 
 on:
   push:
     branches:
-      - develop
+      - develop # dev trigger (automatic)
   pull_request:
     branches:
       - develop
+  workflow_dispatch:
+    inputs:
+      manual_tag:
+        description: 'Optional manual tag to override versioning'
+        required: false
+        default: ''
 
-jobs:
-  lint:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r ./requirements/lint.in
-
-      # Rework happend only in ETL folder
-      - name: Run black
-        run: black --check ./
-
-      - name: Run isort
-        run: isort --check-only ./
-
-      - name: Install shellcheck and run ShellCheck
-        run: |
-          shellcheck ./db.sh
-          shellcheck ./scripts/*.sh
+permissions:
+  contents: write
 
-  build:
-    # Only run if the event is a manual trigger
-    if: github.event_name == 'workflow_dispatch'
-    needs: lint
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r ./requirements/build.in
-
-      - name: Copy template.env to .env
-        run: cp template.env .env
-
-      - name: Run the "db.sh" script
-        env:
-          GITHUB_CI: "true"
-          NEXUS_URL: ${{ secrets.NEXUS_URL }}
-          NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
-          NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
-        run: ./db.sh
-
-      # To update: If it is merged to main -> final tag (10.1.0) or (latest?)
-      # To update: If it is just a PR -> latest tag of the branch (10.0.2025-01-01-xxxx) Git Commit Hash?
+jobs:
+  dev_build:
+    if: ${{ github.event_name == 'pull_request' }}
+    uses: ./.github/workflows/build-and-push-image.yml
+    with:
+      environment: dev
+      manual_tag: ${{ inputs.manual_tag }}
+    secrets: inherit
+
+  prod_build:
+    if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) }}
+    uses: ./.github/workflows/build-and-push-image.yml
+    with:
+      environment: prod
+      manual_tag: ${{ github.event.inputs.manual_tag }}
+    secrets: inherit

.github/workflows/build-and-push-image.yml

@@ -0,0 +1,128 @@
+name: Build and Push ImageS
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      manual_tag:
+        required: false
+        type: string
+        default: ''
+    secrets:
+      NEXUS_URL:
+        required: true
+      NEXUS_DOCKER_URL:
+        required: true
+      NEXUS_USERNAME:
+        required: true
+      NEXUS_PASSWORD:
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      NEXUS_URL: ${{ secrets.NEXUS_URL }}
+      NEXUS_DOCKER_URL: ${{ secrets.NEXUS_DOCKER_URL }}
+      NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
+      NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+      DOCKER_DATA_MODEL_NAME: infocompanies-data-model-postgres
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - if: ${{ inputs.environment != 'prod' && inputs.environment != 'dev' }}
+        name: Validate environment input
+        run: |
+          echo "Error: 'environment' must be 'dev' or 'prod'."
+          exit 1
+
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          df -h
+
+      - name: Get the last Git tag
+        id: get_last_tag
+        run: |
+          git fetch --tags --force
+          LAST_TAG=$(git tag --sort=-creatordate | head -n 1)
+          if [ -z "$LAST_TAG" ]; then
+            LAST_TAG="0.0.0"
+          fi
+          echo "LAST_TAG=${LAST_TAG}" >> $GITHUB_ENV
+
+      - name: Calculate new version
+        id: calculate_version
+        run: |
+          LAST_TAG="${LAST_TAG}"
+          IFS='.' read -r MAJOR MINOR PATCH <<< "$LAST_TAG"
+
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            PATCH=$((PATCH + 1))
+          else
+            MINOR=$((MINOR + 1))
+            PATCH=0
+          fi
+
+          NEW_VERSION="${MAJOR}.${MINOR}.${PATCH}"
+
+          if [[ "${{ inputs.manual_tag }}" != "" ]]; then
+            NEW_VERSION="${{ inputs.manual_tag }}"
+          fi
+          echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV
+
+      - if: ${{ inputs.environment == 'dev' }}
+        name: Pull CSV files from Nexus
+        run: |
+          # Create temporary .env file for the script
+          cat > .env << EOF
+          NEXUS_URL=${NEXUS_URL}
+          NEXUS_DOCKER_URL=${NEXUS_DOCKER_URL}
+          NEXUS_USERNAME=${NEXUS_USERNAME}
+          NEXUS_PASSWORD=${NEXUS_PASSWORD}
+          EOF
+
+          ./scripts/nexus/pull-csv.sh
+
+          # Clean up .env file
+          rm -f .env
+
+      - name: Build prefilled Postgres image
+        run: |
+          if [ "${{ inputs.environment }}" == "dev" ]; then
+          DOCKERFILE="dev.Dockerfile"
+          else
+          DOCKERFILE="prod.Dockerfile"
+          fi
+
+          IMAGE="${NEXUS_DOCKER_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${NEW_VERSION}"
+          docker build -f "${DOCKERFILE}" -t $IMAGE .
+
+          echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_DOCKER_URL}" -u "${NEXUS_USERNAME}" --password-stdin
+          docker push $IMAGE
+          docker logout "${NEXUS_DOCKER_URL}"
+
+      - name: Clean up Docker resources
+        run: |
+          docker rm -f "${DOCKER_DATA_MODEL_NAME}" || true
+          docker volume rm pgdata || true
+          docker system prune -f
+
+      - name: Create and push Git tag
+        if: github.event_name != 'pull_request'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "actions@github.com"
+          git tag -a "${NEW_VERSION}" -m "Release ${NEW_VERSION}"
+          git push origin "${NEW_VERSION}" --no-verify
+
+      - name: Output new version
+        run: echo "New version is ${NEW_VERSION}"

dev.Dockerfile

@@ -0,0 +1,43 @@
+# ===============================
+# Builder stage – preload database
+# ===============================
+FROM postgres:16.4 AS builder
+
+ENV POSTGRES_USER=postgres
+ENV POSTGRES_PASSWORD=root
+ENV POSTGRES_DB=postgres
+ENV PGDATA=/var/lib/postgresql/data
+
+# Install Python & dependencies for Alembic
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    python3 python3-venv python3-pip postgresql-client \
+ && python3 -m venv /opt/venv \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Copy requirements and install
+COPY requirements/build.in /tmp/requirements.in
+RUN pip install --no-cache-dir -r /tmp/requirements.in
+
+# Copy app code and CSVs
+WORKDIR /app
+COPY schema ./schema
+COPY scripts ./scripts
+COPY scripts/setup-db.sh /app/setup-db.sh
+COPY final.csv leaders.csv fichier_combine_updated_big_fixed.csv ./
+
+# Make ./data writable by postgres user
+RUN mkdir -p /app/data && chown -R postgres:postgres /app/data
+
+# Start Postgres for migrations and CSV loading
+USER postgres
+RUN /app/setup-db.sh
+
+# ===============================
+# Final image – production-ready
+# ===============================
+FROM postgres:16.4
+
+# Copy preloaded, vacuumed database
+COPY --from=builder /var/lib/postgresql/data /var/lib/postgresql/data

prod.Dockerfile

@@ -0,0 +1,25 @@
+FROM postgres:16.4
+
+# Install Python & dependencies for Alembic
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python3 python3-venv python3-pip postgresql-client \
+    && python3 -m venv /opt/venv \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Copy requirements and install as root (postgres user cannot write to venv yet)
+COPY requirements/build.in /tmp/requirements.in
+RUN pip install --no-cache-dir -r /tmp/requirements.in
+
+# Copy app code
+WORKDIR /app
+COPY schema ./schema
+COPY scripts ./scripts
+
+# Make migration script executable
+COPY ./scripts/docker/run-migrations.sh /docker-entrypoint-initdb.d/01_run_migrations.sh
+RUN chmod +x /docker-entrypoint-initdb.d/01_run_migrations.sh
+
+# Switch to postgres user (final step)
+USER postgres

schema/alembic/versions/57a5a84e79d0_init_db.py

@@ -237,20 +237,20 @@ def upgrade() -> None:
     op.create_index("idx_leader_siren", "leaders", ["siren"], unique=False)
     op.create_table(
         "config",
-        sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("last_reset_quota_date", sa.Date(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
     op.create_table(
         "city",
-        sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("name", sa.String(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
     op.create_index("ix_city_name", "city", ["name"], unique=False)
     op.create_table(
         "industry_sector",
-        sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("name", sa.String(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
@@ -259,14 +259,14 @@ def upgrade() -> None:
     )
     op.create_table(
         "legal_form",
-        sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("name", sa.String(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
     op.create_index("ix_legal_form_name", "legal_form", ["name"], unique=False)
     op.create_table(
         "region",
-        sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("name", sa.String(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
@@ -287,12 +287,12 @@ def upgrade() -> None:
     op.create_index("ix_user_quota_user_id", "user_quota", ["user_id"], unique=False)
     op.create_table(
         "user_company_status",
-        sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
         sa.Column("user_id", sa.String(), nullable=True),
         sa.Column(
             "status", sa.Enum("NOT_DONE", "TO_DO", "DONE", name="status"), nullable=True
         ),
-        sa.Column("company_id", sa.BigInteger(), nullable=True),
+        sa.Column("company_id", sa.Integer(), nullable=True),
         sa.PrimaryKeyConstraint("id"),
     )
     op.create_index(