diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 31e895c..761988d 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -1,73 +1,35 @@ -name: CI +name: Build and Push Docker Image on: push: branches: - - develop + - develop # dev trigger (automatic) pull_request: branches: - develop + workflow_dispatch: + inputs: + manual_tag: + description: 'Optional manual tag to override versioning' + required: false + default: '' -jobs: - lint: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./requirements/lint.in - - # Rework happend only in ETL folder - - name: Run black - run: black --check ./ - - - name: Run isort - run: isort --check-only ./ - - - name: Install shellcheck and run ShellCheck - run: | - shellcheck ./db.sh - shellcheck ./scripts/*.sh +permissions: + contents: write - build: - # Only run if the event is a manual trigger - if: github.event_name == 'workflow_dispatch' - needs: lint - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./requirements/build.in - - - name: Copy template.env to .env - run: cp template.env .env - - - name: Run the "db.sh" script - env: - GITHUB_CI: "true" - NEXUS_URL: ${{ secrets.NEXUS_URL }} - NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} - NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} - run: ./db.sh - - # To update: If it is merged to main -> final tag (10.1.0) or (latest?) - # To update: If it is just a PR -> latest tag of the branch (10.0.2025-01-01-xxxx) Git Commit Hash? \ No newline at end of file +jobs: + dev_build: + if: ${{ github.event_name == 'pull_request' }} + uses: ./.github/workflows/build-and-push-image.yml + with: + environment: dev + manual_tag: ${{ inputs.manual_tag }} + secrets: inherit + + prod_build: + if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.merged == true) }} + uses: ./.github/workflows/build-and-push-image.yml + with: + environment: prod + manual_tag: ${{ github.event.inputs.manual_tag }} + secrets: inherit diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml new file mode 100644 index 0000000..400bb75 --- /dev/null +++ b/.github/workflows/build-and-push-image.yml @@ -0,0 +1,128 @@ +name: Build and Push ImageS + +on: + workflow_call: + inputs: + environment: + required: true + type: string + manual_tag: + required: false + type: string + default: '' + secrets: + NEXUS_URL: + required: true + NEXUS_DOCKER_URL: + required: true + NEXUS_USERNAME: + required: true + NEXUS_PASSWORD: + required: true + +jobs: + build: + runs-on: ubuntu-latest + env: + NEXUS_URL: ${{ secrets.NEXUS_URL }} + NEXUS_DOCKER_URL: ${{ secrets.NEXUS_DOCKER_URL }} + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + DOCKER_DATA_MODEL_NAME: infocompanies-data-model-postgres + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - if: ${{ inputs.environment != 'prod' && inputs.environment != 'dev' }} + name: Validate environment input + run: | + echo "Error: 'environment' must be 'dev' or 'prod'." + exit 1 + + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + df -h + + - name: Get the last Git tag + id: get_last_tag + run: | + git fetch --tags --force + LAST_TAG=$(git tag --sort=-creatordate | head -n 1) + if [ -z "$LAST_TAG" ]; then + LAST_TAG="0.0.0" + fi + echo "LAST_TAG=${LAST_TAG}" >> $GITHUB_ENV + + - name: Calculate new version + id: calculate_version + run: | + LAST_TAG="${LAST_TAG}" + IFS='.' read -r MAJOR MINOR PATCH <<< "$LAST_TAG" + + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + PATCH=$((PATCH + 1)) + else + MINOR=$((MINOR + 1)) + PATCH=0 + fi + + NEW_VERSION="${MAJOR}.${MINOR}.${PATCH}" + + if [[ "${{ inputs.manual_tag }}" != "" ]]; then + NEW_VERSION="${{ inputs.manual_tag }}" + fi + echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV + + - if: ${{ inputs.environment == 'dev' }} + name: Pull CSV files from Nexus + run: | + # Create temporary .env file for the script + cat > .env << EOF + NEXUS_URL=${NEXUS_URL} + NEXUS_DOCKER_URL=${NEXUS_DOCKER_URL} + NEXUS_USERNAME=${NEXUS_USERNAME} + NEXUS_PASSWORD=${NEXUS_PASSWORD} + EOF + + ./scripts/nexus/pull-csv.sh + + # Clean up .env file + rm -f .env + + - name: Build prefilled Postgres image + run: | + if [ "${{ inputs.environment }}" == "dev" ]; then + DOCKERFILE="dev.Dockerfile" + else + DOCKERFILE="prod.Dockerfile" + fi + + IMAGE="${NEXUS_DOCKER_URL}/${DOCKER_DATA_MODEL_NAME}-${{ inputs.environment }}:${NEW_VERSION}" + docker build -f "${DOCKERFILE}" -t $IMAGE . + + echo "${NEXUS_PASSWORD}" | docker login "${NEXUS_DOCKER_URL}" -u "${NEXUS_USERNAME}" --password-stdin + docker push $IMAGE + docker logout "${NEXUS_DOCKER_URL}" + + - name: Clean up Docker resources + run: | + docker rm -f "${DOCKER_DATA_MODEL_NAME}" || true + docker volume rm pgdata || true + docker system prune -f + + - name: Create and push Git tag + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + git tag -a "${NEW_VERSION}" -m "Release ${NEW_VERSION}" + git push origin "${NEW_VERSION}" --no-verify + + - name: Output new version + run: echo "New version is ${NEW_VERSION}" diff --git a/db.sh b/db.sh deleted file mode 100755 index 4f94f13..0000000 --- a/db.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# shellcheck disable=SC1091 -source ./scripts/util.sh - - -# Main script -if [[ "${GITHUB_CI:-}" == "true" ]]; then - ./scripts/pull-csv.sh -fi - -docker compose up -d --quiet-pull -sleep 2 - -log_info "Initializing the database." -cd ./schema -alembic upgrade head - -log_info "Executing Database.py" -PYTHONPATH=. python3 app/database/database.py -cd .. - - -./scripts/load-csv-to-database.sh - -docker compose down - diff --git a/dev.Dockerfile b/dev.Dockerfile new file mode 100644 index 0000000..2ee056f --- /dev/null +++ b/dev.Dockerfile @@ -0,0 +1,43 @@ +# =============================== +# Builder stage – preload database +# =============================== +FROM postgres:16.4 AS builder + +ENV POSTGRES_USER=postgres +ENV POSTGRES_PASSWORD=root +ENV POSTGRES_DB=postgres +ENV PGDATA=/var/lib/postgresql/data + +# Install Python & dependencies for Alembic +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-pip postgresql-client \ + && python3 -m venv /opt/venv \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/opt/venv/bin:$PATH" + +# Copy requirements and install +COPY requirements/build.in /tmp/requirements.in +RUN pip install --no-cache-dir -r /tmp/requirements.in + +# Copy app code and CSVs +WORKDIR /app +COPY schema ./schema +COPY scripts ./scripts +COPY scripts/setup-db.sh /app/setup-db.sh +COPY final.csv leaders.csv fichier_combine_updated_big_fixed.csv ./ + +# Make ./data writable by postgres user +RUN mkdir -p /app/data && chown -R postgres:postgres /app/data + +# Start Postgres for migrations and CSV loading +USER postgres +RUN /app/setup-db.sh + +# =============================== +# Final image – production-ready +# =============================== +FROM postgres:16.4 + +# Copy preloaded, vacuumed database +COPY --from=builder /var/lib/postgresql/data /var/lib/postgresql/data diff --git a/prod.Dockerfile b/prod.Dockerfile new file mode 100644 index 0000000..29d18bd --- /dev/null +++ b/prod.Dockerfile @@ -0,0 +1,25 @@ +FROM postgres:16.4 + +# Install Python & dependencies for Alembic +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-venv python3-pip postgresql-client \ + && python3 -m venv /opt/venv \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/opt/venv/bin:$PATH" + +# Copy requirements and install as root (postgres user cannot write to venv yet) +COPY requirements/build.in /tmp/requirements.in +RUN pip install --no-cache-dir -r /tmp/requirements.in + +# Copy app code +WORKDIR /app +COPY schema ./schema +COPY scripts ./scripts + +# Make migration script executable +COPY ./scripts/docker/run-migrations.sh /docker-entrypoint-initdb.d/01_run_migrations.sh +RUN chmod +x /docker-entrypoint-initdb.d/01_run_migrations.sh + +# Switch to postgres user (final step) +USER postgres diff --git a/schema/alembic/versions/57a5a84e79d0_init_db.py b/schema/alembic/versions/57a5a84e79d0_init_db.py index 1e3d1be..d052a9f 100644 --- a/schema/alembic/versions/57a5a84e79d0_init_db.py +++ b/schema/alembic/versions/57a5a84e79d0_init_db.py @@ -237,20 +237,20 @@ def upgrade() -> None: op.create_index("idx_leader_siren", "leaders", ["siren"], unique=False) op.create_table( "config", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("last_reset_quota_date", sa.Date(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_table( "city", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_index("ix_city_name", "city", ["name"], unique=False) op.create_table( "industry_sector", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) @@ -259,14 +259,14 @@ def upgrade() -> None: ) op.create_table( "legal_form", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_index("ix_legal_form_name", "legal_form", ["name"], unique=False) op.create_table( "region", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("name", sa.String(), nullable=True), sa.PrimaryKeyConstraint("id"), ) @@ -287,12 +287,12 @@ def upgrade() -> None: op.create_index("ix_user_quota_user_id", "user_quota", ["user_id"], unique=False) op.create_table( "user_company_status", - sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), sa.Column("user_id", sa.String(), nullable=True), sa.Column( "status", sa.Enum("NOT_DONE", "TO_DO", "DONE", name="status"), nullable=True ), - sa.Column("company_id", sa.BigInteger(), nullable=True), + sa.Column("company_id", sa.Integer(), nullable=True), sa.PrimaryKeyConstraint("id"), ) op.create_index( diff --git a/schema/app/models/autocomplete.py b/schema/app/models/autocomplete.py index c2cf99d..9f0dd16 100644 --- a/schema/app/models/autocomplete.py +++ b/schema/app/models/autocomplete.py @@ -1,4 +1,4 @@ -from sqlalchemy import BigInteger, Column, Index, String +from sqlalchemy import Column, Index, Integer, String from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -7,7 +7,7 @@ class City(Base): __tablename__ = "city" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_city_name", "name"),) @@ -16,7 +16,7 @@ class City(Base): class IndustrySector(Base): __tablename__ = "industry_sector" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_industry_sector_name", "name"),) @@ -25,7 +25,7 @@ class IndustrySector(Base): class LegalForm(Base): __tablename__ = "legal_form" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_legal_form_name", "name"),) @@ -34,7 +34,7 @@ class LegalForm(Base): class Region(Base): __tablename__ = "region" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) name = Column(String) __table_args__ = (Index("ix_region_name", "name"),) diff --git a/schema/app/models/config.py b/schema/app/models/config.py index f5d7136..e66b787 100644 --- a/schema/app/models/config.py +++ b/schema/app/models/config.py @@ -1,4 +1,4 @@ -from sqlalchemy import BigInteger, Column, Date +from sqlalchemy import Column, Date, Integer from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -7,5 +7,5 @@ class Config(Base): __tablename__ = "config" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) last_reset_quota_date = Column(Date) diff --git a/schema/app/models/user_company_status.py b/schema/app/models/user_company_status.py index ee92442..3d49bc1 100644 --- a/schema/app/models/user_company_status.py +++ b/schema/app/models/user_company_status.py @@ -1,6 +1,6 @@ import enum -from sqlalchemy import BigInteger, Column, Enum, Index, String +from sqlalchemy import Column, Enum, Index, Integer, String from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -15,10 +15,10 @@ class Status(enum.Enum): class UserCompanyStatus(Base): __tablename__ = "user_company_status" - id = Column(BigInteger, primary_key=True, autoincrement=True) + id = Column(Integer, primary_key=True, autoincrement=True) user_id = Column(String) status = Column(Enum(Status)) - company_id = Column(BigInteger) + company_id = Column(Integer) __table_args__ = ( Index("ix_user_company_status_user_id", "user_id"), diff --git a/scripts/clean-docker-volumes.sh b/scripts/clean-docker-volumes.sh deleted file mode 100755 index c6c2540..0000000 --- a/scripts/clean-docker-volumes.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# shellcheck disable=SC1091 -source ./scripts/util.sh -volume="$(get_infocompanies_data_model_postgres_volume)" - - -log_info "Stopping Docker containers..." -docker compose down - -log_info "Cleaning up Docker volume..." -docker volume rm -f "$volume" - -log_success "Docker volume cleaned up." \ No newline at end of file diff --git a/scripts/docker/docker-build-init.sh b/scripts/docker/docker-build-init.sh new file mode 100755 index 0000000..d9da14c --- /dev/null +++ b/scripts/docker/docker-build-init.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +echo ">>> Initializing Postgres data at build time..." + +# Init database cluster +initdb -D /var/lib/postgresql/data + +# Prepend Docker subnet access to pg_hba.conf +# This ensures it is checked BEFORE any default scram-sha-256 lines +sed -i "1ihost all all 172.19.0.0/16 md5" /var/lib/postgresql/data/pg_hba.conf + +# Start Postgres on IPv4 localhost +pg_ctl -D /var/lib/postgresql/data \ + -o "-c listen_addresses='*' -c unix_socket_directories='/var/run/postgresql'" \ + -w start + +# Wait until Postgres is ready +until pg_isready -U postgres -h 127.0.0.1 -p 5432; do + echo "Waiting for Postgres to be ready..." + sleep 1 +done + +# Before running migrations / CSV load +psql -v ON_ERROR_STOP=1 --username=postgres <<-EOSQL + ALTER SYSTEM SET max_wal_size = '6GB'; + ALTER SYSTEM SET checkpoint_timeout = '30min'; + ALTER SYSTEM SET synchronous_commit = off; + ALTER SYSTEM SET fsync = off; + ALTER SYSTEM SET full_page_writes = off; +EOSQL +pg_ctl -D /var/lib/postgresql/data reload + +# Apply migrations +export DATABASE_URL=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@127.0.0.1:5432/$POSTGRES_DB +cd /app/schema +PYTHONPATH=/app alembic upgrade head + +# Populate with CSV data +PYTHONPATH=/app python /app/app/database/database.py +cd .. +/app/scripts/load-csv-to-database.sh + +# Restore safer defaults +psql -v ON_ERROR_STOP=1 --username=postgres <<-EOSQL + ALTER SYSTEM RESET max_wal_size; + ALTER SYSTEM RESET checkpoint_timeout; + ALTER SYSTEM RESET synchronous_commit; + ALTER SYSTEM RESET fsync; + ALTER SYSTEM RESET full_page_writes; +EOSQL +pg_ctl -D /var/lib/postgresql/data reload + +# Stop Postgres +pg_ctl -D /var/lib/postgresql/data -m fast -w stop + +echo ">>> Database populated and baked into image." diff --git a/scripts/docker/run-migrations.sh b/scripts/docker/run-migrations.sh new file mode 100644 index 0000000..444bc2b --- /dev/null +++ b/scripts/docker/run-migrations.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source ./scripts/util.sh + +log_info "Initializing the database." +cd ./schema +alembic upgrade head +log_info "Database initialized." \ No newline at end of file diff --git a/scripts/load-csv-to-database.sh b/scripts/load-csv-to-database.sh index 52d7d6b..5be354a 100755 --- a/scripts/load-csv-to-database.sh +++ b/scripts/load-csv-to-database.sh @@ -27,15 +27,12 @@ transfer_csv_to_database() { exit 1 fi - local postgres_container - postgres_container=$(get_infocompanies_data_model_postgres_container) - log_info "Transferring the CSV file '$csv_file_path' to the PostgreSQL database." - docker cp "$csv_file_path" "$postgres_container:$container_csv_file" - docker exec -u postgres -i "$postgres_container" psql -d postgres -c "COPY $table_name($columns) FROM '$container_csv_file' DELIMITER '$delimiter' CSV HEADER;" + cp "$csv_file_path" "$container_csv_file" + psql -d postgres -c "COPY $table_name($columns) FROM '$container_csv_file' DELIMITER '$delimiter' CSV HEADER;" - log_success "Transfer of '$csv_file_path' to the database table '$table_name' completed successfully." + log_success "Transfer of '$container_csv_file' to the database table '$table_name' completed successfully." } @@ -46,8 +43,6 @@ export_unique_values() { local format="${3:-csv}" # Default format is CSV local base_name base_name=$(basename "$output_file") - local postgres_container - postgres_container=$(get_infocompanies_data_model_postgres_container) mkdir -p "$(dirname "$output_file")" @@ -55,25 +50,25 @@ export_unique_values() { local output_csv="/tmp/$base_name.csv" log_info "Exporting unique values for the $base_name table in CSV format..." - docker exec -u postgres -i "$postgres_container" mkdir -p /tmp - docker exec -u postgres -i "$postgres_container" psql -d postgres -c "\copy ($query) TO '$output_csv' CSV HEADER;" - docker cp "$postgres_container:$output_csv" "$output_file" - + mkdir -p /tmp + psql -d postgres -c "\copy ($query) TO '$output_csv' CSV HEADER;" + cp "$output_csv" "$output_file" log_success "Exported CSV file saved to $output_file" + elif [ "$format" == "sql" ]; then log_info "Exporting unique values for the $base_name table in SQL format..." local temp_table="temp_export" local output_sql="$output_file.sql" - docker exec -u postgres -i "$postgres_container" psql -d postgres -c " + psql -d postgres -c " DROP TABLE IF EXISTS $temp_table; CREATE TABLE $temp_table AS SELECT row_number() OVER () AS id, * FROM ($query) AS subquery; " - docker exec -u postgres -i "$postgres_container" pg_dump -U postgres --data-only --table="$temp_table" postgres >"$output_sql" - docker exec -u postgres -i "$postgres_container" psql -d postgres -c "DROP TABLE IF EXISTS $temp_table;" + pg_dump -U postgres --data-only --table="$temp_table" postgres >"$output_sql" + psql -d postgres -c "DROP TABLE IF EXISTS $temp_table;" log_success "Exported SQL file saved to $output_sql" else diff --git a/scripts/nexus/pull-csv.sh b/scripts/nexus/pull-csv.sh new file mode 100755 index 0000000..bf5c4de --- /dev/null +++ b/scripts/nexus/pull-csv.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source ./scripts/util.sh +# shellcheck disable=SC1091 +source .env + +# Reminder: we use the native Nexus port and not the Docker one +NEXUS_CSV_URL="${NEXUS_URL}/repository/datasets" +USERNAME_WITH_PASSWORD=$(build_nexus_username_with_password "$NEXUS_USERNAME" "$NEXUS_PASSWORD") + +# TODO: Change that to be only done by the CI +FINAL_TAG="0.1" +LEADER_TAG="0.1" +FICHIER_COMBINE_TAG="0.1" + +LEADER_CSV="leaders-$LEADER_TAG.csv.gz" +FINAL_CSV="final-$FINAL_TAG.csv.gz" +FICHIER_COMBINE_CSV="fichier_combine_updated_big_fixed-$FICHIER_COMBINE_TAG.csv.gz" + +# Pull the CSVs to insert +log_info "Pulling the CSVs..." +curl -u "$USERNAME_WITH_PASSWORD" -O "$NEXUS_CSV_URL/$LEADER_CSV" +curl -u "$USERNAME_WITH_PASSWORD" -O "$NEXUS_CSV_URL/$FINAL_CSV" +curl -u "$USERNAME_WITH_PASSWORD" -O "$NEXUS_CSV_URL/$FICHIER_COMBINE_CSV" +log_success "CSV files pulled successfully." + +# Uncompress the CSVs +log_info "Uncompressing the CSVs..." +gunzip "$LEADER_CSV" +gunzip "$FINAL_CSV" +gunzip "$FICHIER_COMBINE_CSV" + +# Rename files to remove version tags +mv "leaders-$LEADER_TAG.csv" "leaders.csv" +mv "final-$FINAL_TAG.csv" "final.csv" +mv "fichier_combine_updated_big_fixed-$FICHIER_COMBINE_TAG.csv" "fichier_combine_updated_big_fixed.csv" + +log_success "CSV files uncompressed successfully." \ No newline at end of file diff --git a/scripts/nexus/push-csv.sh b/scripts/nexus/push-csv.sh new file mode 100755 index 0000000..3726e5a --- /dev/null +++ b/scripts/nexus/push-csv.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -euo pipefail + +# shellcheck disable=SC1091 +source ./scripts/util.sh +# shellcheck disable=SC1091 +source .env + +# Reminder: we use the native Nexus port and not the Docker one +NEXUS_CSV_URL="$NEXUS_URL/repository/datasets" +USERNAME_WITH_PASSWORD=$(build_nexus_username_with_password "$NEXUS_USERNAME" "$NEXUS_PASSWORD") + +echo "Nexus CSV URL: $NEXUS_CSV_URL" + +# TODO: Change that to be only done by the CI +FINAL_TAG="0.1" +LEADER_TAG="0.1" +FICHIER_COMBINE_TAG="0.1" + +echo "Pushing CSVs to Nexus..." +curl -u "$USERNAME_WITH_PASSWORD" \ + --upload-file ./final.csv.gz \ + "$NEXUS_CSV_URL/final-${FINAL_TAG}.csv.gz" +echo "CSV 'final' pushed successfully." + +curl -u "$USERNAME_WITH_PASSWORD" \ + --upload-file ./leaders.csv.gz \ + "$NEXUS_CSV_URL/leaders-${LEADER_TAG}.csv.gz" +echo "CSV 'leaders' pushed successfully." + +curl -u "$USERNAME_WITH_PASSWORD" \ + --upload-file ./fichier_combine_updated_big_fixed.csv.gz \ + "$NEXUS_CSV_URL/fichier_combine_updated_big_fixed-${FICHIER_COMBINE_TAG}.csv.gz" +echo "CSV 'fichier_combine_updated_big_fixed' pushed successfully." + +echo "CSV files pushed successfully." \ No newline at end of file diff --git a/scripts/pull-csv.sh b/scripts/pull-csv.sh deleted file mode 100755 index 3716788..0000000 --- a/scripts/pull-csv.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# shellcheck disable=SC1091 -source ./scripts/util.sh - - -# Pull the CSVs to insert -log_info "Pulling the CSVs..." -curl -u "$NEXUS_USERNAME:$NEXUS_PASSWORD" -O "$NEXUS_URL/repository/datasets/versions/v1.0/leaders.csv.gz" -curl -u "$NEXUS_USERNAME:$NEXUS_PASSWORD" -O "$NEXUS_URL/repository/datasets/versions/v1.0/final.csv.gz" -curl -u "$NEXUS_USERNAME:$NEXUS_PASSWORD" -O "$NEXUS_URL/repository/datasets/versions/v1.0/fichier_combine_updated_big_fixed.csv.gz" -log_success "CSV files pulled successfully." - -# Uncompress the CSVs -log_info "Uncompressing the CSVs..." -gunzip leaders.csv.gz -gunzip final.csv.gz -gunzip fichier_combine_updated_big_fixed.csv.gz - -log_success "CSV files uncompressed successfully." \ No newline at end of file diff --git a/scripts/setup-db.sh b/scripts/setup-db.sh new file mode 100755 index 0000000..9616c66 --- /dev/null +++ b/scripts/setup-db.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -e + +# Start Postgres in background +docker-entrypoint.sh postgres & + +# Wait for Postgres to be ready +echo "Waiting for Postgres..." +sleep 5 + +export PGHOST=localhost +export PGUSER=postgres +export PGPASSWORD=root + +# Test connection +psql -d postgres -c "SELECT 1;" + +# Run migrations +cd /app/schema +alembic upgrade head +PYTHONPATH=. python ./app/database/database.py +cd /app + +psql -v ON_ERROR_STOP=1 --username="$POSTGRES_USER" <