diff --git a/.github/workflows/linting-checks.yml b/.github/workflows/linting-checks.yml index 9e06621..09020b3 100644 --- a/.github/workflows/linting-checks.yml +++ b/.github/workflows/linting-checks.yml @@ -1,15 +1,15 @@ -name: Post-Merge Checks +name: PR Linting Checks on: - push: - branches: - - main + pull_request: + types: [opened, synchronize, reopened] permissions: contents: read + pull-requests: write jobs: - pre-commit-checks: + linting-checks: runs-on: ubuntu-latest steps: @@ -21,14 +21,17 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.11.14' cache: 'pip' - - name: Install dependencies + - name: Install requirements run: | python -m pip install --upgrade pip pip install -r requirements.txt + - name: Install dbt dependencies + run: dbt deps + - name: Set up Terraform uses: hashicorp/setup-terraform@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5812b7c..3387c0b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,12 +8,21 @@ repos: include: \.(yaml|yml)$ - repo: https://github.com/sqlfluff/sqlfluff - rev: 3.2.3 + rev: 4.2.0 hooks: - id: sqlfluff-lint - additional_dependencies: ['dbt-bigquery==1.8.1', 'sqlfluff-templater-dbt==3.2.3'] + additional_dependencies: + - dbt-core==1.11.2 + - dbt-bigquery==1.11.0 + - sqlfluff==4.2.0 + - sqlfluff-templater-dbt==4.2.0 + - id: sqlfluff-fix - additional_dependencies: ['dbt-bigquery==1.8.1', 'sqlfluff-templater-dbt==3.2.3'] + additional_dependencies: + - dbt-core==1.11.2 + - dbt-bigquery==1.11.0 + - sqlfluff==4.2.0 + - sqlfluff-templater-dbt==4.2.0 - repo: https://github.com/dbt-checkpoint/dbt-checkpoint rev: v1.2.1 diff --git a/.sqlfluff b/.sqlfluff index 52a5187..c1dd1ce 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -25,15 +25,15 @@ align_scope = bracketed # AS should be there in table aliases [sqlfluff:rules:aliasing.table] -aliasing.table = explicit +aliasing = explicit # AS should be there for renamed columns [sqlfluff:rules:aliasing.column] -aliasing.column = explicit +aliasing = explicit # AS should be there after SUM(), MAX() etc [sqlfluff:rules:aliasing.expression] -aliasing.expression = explicit +aliasing = explicit # use ALL numbers in GROUP BY, ORDER BY [sqlfluff:rules:ambiguous.column_references] diff --git a/.sqlfluffignore b/.sqlfluffignore new file mode 100644 index 0000000..6d2113b --- /dev/null +++ b/.sqlfluffignore @@ -0,0 +1,3 @@ +target +dbt_packages +logs diff --git a/dbt/analyses/compare_column_values.sql b/dbt/analyses/compare_column_values.sql deleted file mode 100644 index 2a067a4..0000000 --- a/dbt/analyses/compare_column_values.sql +++ /dev/null @@ -1,40 +0,0 @@ -{{ - config( - materialized='table', - alias='compare_single_query_column' - ) -}} - - -{% set old_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs` -{% endset %} - -{% set new_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs_copy` -{% endset %} - -{{ audit_helper.compare_column_values( - a_query = old_query, - b_query = new_query, - primary_key = 'unique_identifier', - column_to_compare = 'job_title' - ) -}} - - -{# - [{ - "column_name": "job_title", - "match_status": "✅: perfect match", - "count_records": "2830", - "percent_of_total": "45.88" - }, { - "column_name": "job_title", - "match_status": "❌: ‍values do not match", - "count_records": "3338", - "percent_of_total": "54.12" - }] -#} diff --git a/dbt/analyses/quick_are_queries_identical.sql b/dbt/analyses/quick_are_queries_identical.sql deleted file mode 100644 index cfb5999..0000000 --- a/dbt/analyses/quick_are_queries_identical.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ - config( - materialized='table', - alias='compare_multiple_query_columns' - ) -}} - -{% set old_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs` -{% endset %} - -{% set new_query %} - SELECT *, CONCAT(company_name, location) AS unique_identifier - FROM `data-geeking-gcp.the_data_challenge.data_engineer_jobs_copy` -{% endset %} - -{{ audit_helper.quick_are_queries_identical( - query_a = old_query, - query_b = new_query, - columns = ['job_title', 'salary_estimate', 'job_description'] - ) -}} diff --git a/dbt/analyses/quick_are_relations_identical.sql b/dbt/analyses/quick_are_relations_identical.sql deleted file mode 100644 index 447fa42..0000000 --- a/dbt/analyses/quick_are_relations_identical.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ - config( - materialized='table', - alias='compare_multiple_table_columns' - ) -}} - -{% set old_table = adapter.get_relation( - database = "data-geeking-gcp", - schema = "the_data_challenge", - identifier = "data_engineer_jobs" -) -%} - -{% set new_table = "data-geeking-gcp.the_data_challenge.data_engineer_jobs_copy" %} -- noqa: LT05 - -{{ audit_helper.quick_are_relations_identical( - a_relation = old_table, - b_relation = new_table, - columns = ['job_title', 'salary_estimate', 'job_description'] -) }} diff --git a/requirements.txt b/requirements.txt index 99e1ec1..628fd5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ dbt-core==1.11.2 dbt-bigquery==1.11.0 -sqlfluff==4.0.0 -sqlfluff-templater-dbt==4.0.0 +sqlfluff==4.2.0 +sqlfluff-templater-dbt==4.2.0 yamllint==1.35.1 pylint==3.3.1 pymarkdownlnt==0.9.24